<commit_before># This file is part of curious.
#
# curious is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# curious is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with curious.  If not, see <http://www.gnu.org/licenses/>.

"""
Base classes that all dataclasses inherit from.

.. currentmodule:: curious.dataclasses.bases
"""
import datetime
import inspect
import sys
import threading
from contextlib import contextmanager

from curious.core import client

DISCORD_EPOCH = 1420070400000

_allowing_external_makes = threading.local()
_allowing_external_makes.flag = False

@contextmanager
def allow_external_makes() -> None:
    """
    Using this with a ``with`` allows dataclasses to be made outside of curious' internal code.
    """
    try:
        _allowing_external_makes.flag = True
        yield
    finally:
        _allowing_external_makes.flag = False

class IDObject(object):
    """
    This object is comparable using the snowflake as an ID.

    It is also hashable, using the ID as a hash.
    """

    __slots__ = "id",

    def __init__(self, id: int):
        """
        :param id: The snowflake ID of the object.
        """
        if isinstance(id, str):
            id = int(id)

        #: The ID of this object.
        self.id = id

    def __repr__(self) -> str:
        return "<{} id={!r}>".format(self.__class__.__name__, self.id)

    __str__ = __repr__

    @property
    def snowflake_timestamp(self) -> datetime.datetime:
        """
        :return: The timestamp of the snowflake.
        """
        return datetime.datetime.utcfromtimestamp(((int(self.id) >> 22) + DISCORD_EPOCH) / 1000)

    def __eq__(self, other) -> bool:
        if not hasattr(other, "id"):
            return NotImplemented

        return other.id == self.id

    def __hash__(self) -> int:
        return hash(self.id)

class Dataclass(IDObject):
    """
    The base class for all dataclasses.

    These contain a reference to the current bot as `_bot`.
    """

    # __weakref__ is used to allow weakreffing
    __slots__ = "_bot", "__weakref__"

    @classmethod
    def __new__(cls, *args, **kwargs):
        """
        Inspects the stack to ensure we're being called correctly.
        """
        if _allowing_external_makes.flag is False:
            try:
                frameinfo = inspect.stack()[1]
                frame = frameinfo.frame
                f_globals = frame.f_globals
                f_name = frame.f_code.co_name
                module = f_globals.get('__name__', None)
                file = f_globals.get('__file__', None)

                if module is not None:
                    if f_name == "_convert" and module.startswith("curious.commands"):
                        raise RuntimeError("You passed a dataclass ({}) as a type hint to your "
                                           "command without a converter - don't do this!\n"
                                           "This error has been raised because no builtin converter"
                                           " exists, or the built-in converter has been replaced. "
                                           "Make sure to either add one or fix your code to use a "
                                           "converter function!".format(cls.__name__))
                    elif not module.startswith("curious") \
                            and f'/python3.{sys.version_info[1]}' not in file:
                        raise RuntimeError("You tried to make a dataclass manually - don't do this!"
                                           "\nThe library handles making dataclasses for you. If "
                                           "you want to get an instance, use the appropriate "
                                           "lookup method. \nIf you really need to make the "
                                           "dataclass yourself, wrap it in a "
                                           "``with allow_external_makes)``.")
            finally:
                del frameinfo, frame

        return object.__new__(cls)

    def __init__(self, id: int, cl: 'client.Client'):
        super().__init__(id)

        self._bot = cl
<commit_msg>Make dataclass.__new__ a staticmethod.<commit_after># This file is part of curious.
#
# curious is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# curious is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with curious.  If not, see <http://www.gnu.org/licenses/>.

"""
Base classes that all dataclasses inherit from.

.. currentmodule:: curious.dataclasses.bases
"""
import datetime
import inspect
import sys
import threading
from contextlib import contextmanager

from curious.core import client

DISCORD_EPOCH = 1420070400000

_allowing_external_makes = threading.local()
_allowing_external_makes.flag = False

@contextmanager
def allow_external_makes() -> None:
    """
    Using this with a ``with`` allows dataclasses to be made outside of curious' internal code.
    """
    try:
        _allowing_external_makes.flag = True
        yield
    finally:
        _allowing_external_makes.flag = False

class IDObject(object):
    """
    This object is comparable using the snowflake as an ID.

    It is also hashable, using the ID as a hash.
    """

    __slots__ = "id",

    def __init__(self, id: int):
        """
        :param id: The snowflake ID of the object.
        """
        if isinstance(id, str):
            id = int(id)

        #: The ID of this object.
        self.id = id

    def __repr__(self) -> str:
        return "<{} id={!r}>".format(self.__class__.__name__, self.id)

    __str__ = __repr__

    @property
    def snowflake_timestamp(self) -> datetime.datetime:
        """
        :return: The timestamp of the snowflake.
        """
        return datetime.datetime.utcfromtimestamp(((int(self.id) >> 22) + DISCORD_EPOCH) / 1000)

    def __eq__(self, other) -> bool:
        if not hasattr(other, "id"):
            return NotImplemented

        return other.id == self.id

    def __hash__(self) -> int:
        return hash(self.id)

class Dataclass(IDObject):
    """
    The base class for all dataclasses.

    These contain a reference to the current bot as `_bot`.
    """

    # __weakref__ is used to allow weakreffing
    __slots__ = "_bot", "__weakref__"

    @staticmethod
    def __new__(cls, *args, **kwargs):
        """
        Inspects the stack to ensure we're being called correctly.
        """
        if _allowing_external_makes.flag is False:
            try:
                frameinfo = inspect.stack()[1]
                frame = frameinfo.frame
                f_globals = frame.f_globals
                f_name = frame.f_code.co_name
                module = f_globals.get('__name__', None)
                file = f_globals.get('__file__', None)

                if module is not None:
                    if f_name == "_convert" and module.startswith("curious.commands"):
                        raise RuntimeError("You passed a dataclass ({}) as a type hint to your "
                                           "command without a converter - don't do this!\n"
                                           "This error has been raised because no builtin converter"
                                           " exists, or the built-in converter has been replaced. "
                                           "Make sure to either add one or fix your code to use a "
                                           "converter function!".format(cls.__name__))
                    elif not module.startswith("curious") \
                            and f'/python3.{sys.version_info[1]}' not in file:
                        raise RuntimeError("You tried to make a dataclass manually - don't do this!"
                                           "\nThe library handles making dataclasses for you. If "
                                           "you want to get an instance, use the appropriate "
                                           "lookup method. \nIf you really need to make the "
                                           "dataclass yourself, wrap it in a "
                                           "``with allow_external_makes)``.")
            finally:
                del frameinfo, frame

        return object.__new__(cls)

    def __init__(self, id: int, cl: 'client.Client'):
        super().__init__(id)

        self._bot = cl
<|endoftext|>


<commit_before>import warnings
from cuuats.datamodel.scales import BaseScale

class BaseField(object):

    creation_index = 0
    default_storage = {}

    def __init__(self, label, **kwargs):
        self.label = label
        self.name = kwargs.get('name', False)
        self.required = kwargs.get('required', False)
        self.required_if = kwargs.get('required_if', None)
        self.domain_name = kwargs.get('domain_name', None)
        self.choices = list(kwargs.get('choices', []))
        self.storage = self.default_storage.copy()
        self.storage.update(kwargs.get('storage', {}))

        # Handle field ordering
        self.order = kwargs.get('order', 0)
        self.creation_index = BaseField.creation_index
        BaseField.creation_index += 1

    def __get__(self, instance, owner):
        return instance.values.get(self.name, None)

    def __set__(self, instance, value):
        instance.values[self.name] = value

    def __repr__(self):
        return '%s: %s' % (self.__class__.__name__, self.label)

    def register(self, source, field_name, layer_name, layer_fields):
        """
        Register this field with the data source.
        """

        layer_field = layer_fields.get(field_name, None)
        if layer_field is None:
            warnings.warn('%s is not a field of %s' % (
                field_name, layer_name))
        else:
            self.name = field_name
            if layer_field.domain:
                self.domain_name = layer_field.domain
                domain = source.get_domain(layer_field.domain)
                if domain.domainType == 'CodedValue':
                    self.choices.extend(domain.codedValues)
                elif hasattr(self, 'min') and hasattr(self, 'max'):
                    self.min, self.max = domain.range

    def validate(self, value):
        """
        Performs validation on the given value, and returns any error messages.
        """

        # Check for missing values in required fields.
        if self.required and value is None:
            return ['%s is missing' % (self.label,)]

        # Check that that value is in the choices, if specified.
        elif len(self.choices) > 0 and value is not None and \
                value not in self.choices:
            return ['%s is invalid' % (self.label,)]

        return []

class OIDField(BaseField):
    """
    Object ID field type.
    """

    def validate(self, value):
        """
        Performs validation on the given value, and returns any error messages.
        """

        # OIDs are always valid.
        return []

class GlobalIDField(BaseField):
    """
    Global ID field type.
    """

    def validate(self, value):
        """
        Performs validation on the given value, and returns any error messages.
        """

        # GlobalIDs are always valid.
        return []

class GeometryField(BaseField):
    """
    Geometry field type.
    """

    def validate(self, value):
        """
        Performs validation on the given value, and returns any error messages.
        """

        # TODO: Add validation for geometries.
        return super(GeometryField, self).validate(value)

class StringField(BaseField):

    default_storage = {
        'field_type': 'TEXT',
        'field_length': 100
    }

    def validate(self, value):
        """
        Performs validation on the given value, and returns any error messages.
        """

        # Convert empty strings to None before performing validation.
        if isinstance(value, str) and value.strip() == '':
            value = None
        return super(StringField, self).validate(value)

class NumericField(BaseField):

    default_storage = {
        'field_type': 'FLOAT',
    }

    def __init__(self, name, **kwargs):
        super(NumericField, self).__init__(name, **kwargs)
        self.min = kwargs.get('min', None)
        self.max = kwargs.get('max', None)

    def validate(self, value):
        """
        Performs validation on the given value, and returns any error messages.
        """

        messages = super(NumericField, self).validate(value)
        if value is None:
            return messages

        if (self.min is not None and value < self.min) or \
                (self.max is not None and value > self.max):
            messages.append('%s out of range' % (self.label,))

        return messages

class CalculatedField(BaseField):

    def __init__(self, name, **kwargs):
        super(CalculatedField, self).__init__(name, **kwargs)

        # Overridden by subclasses
        self.update_for = kwargs.get('update_for', [])
        self.condition = kwargs.get('condition', None)
        self.default = kwargs.get('default', None)

    def __get__(self, instance, owner):
        if self.condition:
            locals_dict = {'self': instance}
            locals_dict.update(instance.values)
            if not eval(self.condition, {}, locals_dict):
                return self.default
        return self.calculate(instance)

    def __set__(self, instance, value):
        raise ValueError('Calculated fields cannot be set')

    def calculate(self, instance):
        """
        Calculate the value for this field based on the state of the instance.
        """

        # Overridden by subclasses
        return self.default

class MethodField(CalculatedField):

    default_storage = {
        'field_type': 'DOUBLE',
    }

    def __init__(self, name, **kwargs):
        super(MethodField, self).__init__(name, **kwargs)
        self.method_name = kwargs.get('method_name')

    def calculate(self, instance):
        """
        Calculate the value for this field based on the state of the instance.
        """

        return getattr(instance, self.method_name)(self.name)

class WeightsField(CalculatedField):

    default_storage = {
        'field_type': 'DOUBLE',
    }

    def __init__(self, name, **kwargs):
        super(WeightsField, self).__init__(name, **kwargs)
        self.weights = kwargs.get('weights')
        self.update_for = self.weights.keys()

    def _get_value(self, instance, field_name):
        return getattr(instance, field_name)

    def calculate(self, instance):
        """
        Calculate the value for this field based on the state of the instance.
        """

        if None in [self._get_value(instance, v) for v in self.weights.keys()]:
            return self.default
        else:
            return sum([self._get_value(instance, v)*w
                        for (v, w) in self.weights.items()])

class ScaleField(CalculatedField):

    default_storage = {
        'field_type': 'SHORT',
    }

    def __init__(self, name, **kwargs):
        super(ScaleField, self).__init__(name, **kwargs)
        self.scale = kwargs.get('scale')
        self.value_field = kwargs.get('value_field')
        self.use_description = kwargs.get('use_description', False)
        self.update_for = [self.value_field]

        if not isinstance(self.scale, BaseScale):
            raise TypeError('Scale must be a subclass of BaseScale')

    def calculate(self, instance):
        """
        Calculate the value for this field based on the state of the instance.
        """

        if self.use_description:
            value = instance.get_description_for(self.value_field)
        else:
            value = getattr(instance, self.value_field)
        return self.scale.score(value)
<commit_msg>Removed unused properties from calculated fields.<commit_after>import warnings
from cuuats.datamodel.scales import BaseScale

class BaseField(object):

    creation_index = 0
    default_storage = {}

    def __init__(self, label, **kwargs):
        self.label = label
        self.name = kwargs.get('name', False)
        self.required = kwargs.get('required', False)
        self.required_if = kwargs.get('required_if', None)
        self.domain_name = kwargs.get('domain_name', None)
        self.choices = list(kwargs.get('choices', []))
        self.storage = self.default_storage.copy()
        self.storage.update(kwargs.get('storage', {}))

        # Handle field ordering
        self.order = kwargs.get('order', 0)
        self.creation_index = BaseField.creation_index
        BaseField.creation_index += 1

    def __get__(self, instance, owner):
        return instance.values.get(self.name, None)

    def __set__(self, instance, value):
        instance.values[self.name] = value

    def __repr__(self):
        return '%s: %s' % (self.__class__.__name__, self.label)

    def register(self, source, field_name, layer_name, layer_fields):
        """
        Register this field with the data source.
        """

        layer_field = layer_fields.get(field_name, None)
        if layer_field is None:
            warnings.warn('%s is not a field of %s' % (
                field_name, layer_name))
        else:
            self.name = field_name
            if layer_field.domain:
                self.domain_name = layer_field.domain
                domain = source.get_domain(layer_field.domain)
                if domain.domainType == 'CodedValue':
                    self.choices.extend(domain.codedValues)
                elif hasattr(self, 'min') and hasattr(self, 'max'):
                    self.min, self.max = domain.range

    def validate(self, value):
        """
        Performs validation on the given value, and returns any error messages.
        """

        # Check for missing values in required fields.
        if self.required and value is None:
            return ['%s is missing' % (self.label,)]

        # Check that that value is in the choices, if specified.
        elif len(self.choices) > 0 and value is not None and \
                value not in self.choices:
            return ['%s is invalid' % (self.label,)]

        return []

class OIDField(BaseField):
    """
    Object ID field type.
    """

    def validate(self, value):
        """
        Performs validation on the given value, and returns any error messages.
        """

        # OIDs are always valid.
        return []

class GlobalIDField(BaseField):
    """
    Global ID field type.
    """

    def validate(self, value):
        """
        Performs validation on the given value, and returns any error messages.
        """

        # GlobalIDs are always valid.
        return []

class GeometryField(BaseField):
    """
    Geometry field type.
    """

    def validate(self, value):
        """
        Performs validation on the given value, and returns any error messages.
        """

        # TODO: Add validation for geometries.
        return super(GeometryField, self).validate(value)

class StringField(BaseField):

    default_storage = {
        'field_type': 'TEXT',
        'field_length': 100
    }

    def validate(self, value):
        """
        Performs validation on the given value, and returns any error messages.
        """

        # Convert empty strings to None before performing validation.
        if isinstance(value, str) and value.strip() == '':
            value = None
        return super(StringField, self).validate(value)

class NumericField(BaseField):

    default_storage = {
        'field_type': 'FLOAT',
    }

    def __init__(self, name, **kwargs):
        super(NumericField, self).__init__(name, **kwargs)
        self.min = kwargs.get('min', None)
        self.max = kwargs.get('max', None)

    def validate(self, value):
        """
        Performs validation on the given value, and returns any error messages.
        """

        messages = super(NumericField, self).validate(value)
        if value is None:
            return messages

        if (self.min is not None and value < self.min) or \
                (self.max is not None and value > self.max):
            messages.append('%s out of range' % (self.label,))

        return messages

class CalculatedField(BaseField):

    def __init__(self, name, **kwargs):
        super(CalculatedField, self).__init__(name, **kwargs)

        # Overridden by subclasses
        self.condition = kwargs.get('condition', None)
        self.default = kwargs.get('default', None)

    def __get__(self, instance, owner):
        if self.condition:
            locals_dict = {'self': instance}
            locals_dict.update(instance.values)
            if not eval(self.condition, {}, locals_dict):
                return self.default
        return self.calculate(instance)

    def __set__(self, instance, value):
        raise ValueError('Calculated fields cannot be set')

    def calculate(self, instance):
        """
        Calculate the value for this field based on the state of the instance.
        """

        # Overridden by subclasses
        return self.default

class MethodField(CalculatedField):

    default_storage = {
        'field_type': 'DOUBLE',
    }

    def __init__(self, name, **kwargs):
        super(MethodField, self).__init__(name, **kwargs)
        self.method_name = kwargs.get('method_name')

    def calculate(self, instance):
        """
        Calculate the value for this field based on the state of the instance.
        """

        return getattr(instance, self.method_name)(self.name)

class WeightsField(CalculatedField):

    default_storage = {
        'field_type': 'DOUBLE',
    }

    def __init__(self, name, **kwargs):
        super(WeightsField, self).__init__(name, **kwargs)
        self.weights = kwargs.get('weights')

    def _get_value(self, instance, field_name):
        return getattr(instance, field_name)

    def calculate(self, instance):
        """
        Calculate the value for this field based on the state of the instance.
        """

        if None in [self._get_value(instance, v) for v in self.weights.keys()]:
            return self.default
        else:
            return sum([self._get_value(instance, v)*w
                        for (v, w) in self.weights.items()])

class ScaleField(CalculatedField):

    default_storage = {
        'field_type': 'SHORT',
    }

    def __init__(self, name, **kwargs):
        super(ScaleField, self).__init__(name, **kwargs)
        self.scale = kwargs.get('scale')
        self.value_field = kwargs.get('value_field')
        self.use_description = kwargs.get('use_description', False)

        if not isinstance(self.scale, BaseScale):
            raise TypeError('Scale must be a subclass of BaseScale')

    def calculate(self, instance):
        """
        Calculate the value for this field based on the state of the instance.
        """

        if self.use_description:
            value = instance.get_description_for(self.value_field)
        else:
            value = getattr(instance, self.value_field)
        return self.scale.score(value)
<|endoftext|>


<commit_before>from typing import Text, Any

import pytest

from rasa.core.nlg.response import TemplatedNaturalLanguageGenerator
from rasa.shared.core.domain import Domain
from rasa.shared.core.slots import Slot
from rasa.shared.core.trackers import DialogueStateTracker

async def test_nlg_conditional_response_variations_with_no_slots():
    responses = {
        "utter_test": [
            {
                "text": "Conditional OS Response A",
                "condition": [{"type": "slot", "name": "slot test", "value": "A"}],
                "channel": "os",
            },
            {
                "text": "Conditional Response A",
                "condition": [{"type": "slot", "name": "slot test", "value": "A"}],
            },
            {
                "text": "Conditional Response B",
                "condition": [{"type": "slot", "name": "slot test", "value": "B"}],
            },
            {"text": "Default response"},
        ]
    }
    t = TemplatedNaturalLanguageGenerator(responses=responses)
    no_slots_tracker = DialogueStateTracker(sender_id="nlg_test_default", slots=None)
    default_response = await t.generate(
        utter_action="utter_test", tracker=no_slots_tracker, output_channel=""
    )

    assert default_response.get("text") == "Default response"

async def test_nlg_when_multiple_conditions_satisfied():
    responses = {
        "utter_action": [
            {
                "text": "example A",
                "condition": [{"type": "slot", "name": "test", "value": "A"}],
            },
            {
                "text": "example B",
                "condition": [{"type": "slot", "name": "test_another", "value": "B"}],
            },
            {
                "text": "non matching example 1",
                "condition": [
                    {"type": "slot", "name": "test_third_slot", "value": "C"}
                ],
            },
            {
                "text": "non matching example 2",
                "condition": [{"type": "slot", "name": "test", "value": "D"}],
            },
        ]
    }

    t = TemplatedNaturalLanguageGenerator(responses=responses)
    slot_a = Slot(name="test", initial_value="A", influence_conversation=False)
    slot_b = Slot(name="test_another", initial_value="B", influence_conversation=False)
    tracker = DialogueStateTracker(sender_id="test_nlg", slots=[slot_a, slot_b])
    resp = await t.generate(
        utter_action="utter_action", tracker=tracker, output_channel=""
    )
    assert resp.get("text") in ["example A", "example B"]

@pytest.mark.parametrize(
    ("slot_name", "slot_value", "response_variation"),
    (("test", "A", "example one A"), ("test", "B", "example two B")),
)
async def test_nlg_conditional_response_variations_with_interpolated_slots(
    slot_name: Text, slot_value: Any, response_variation: Text
):
    responses = {
        "utter_action": [
            {
                "text": "example one {test}",
                "condition": [{"type": "slot", "name": "test", "value": "A"}],
            },
            {
                "text": "example two {test}",
                "condition": [{"type": "slot", "name": "test", "value": "B"}],
            },
        ]
    }
    t = TemplatedNaturalLanguageGenerator(responses=responses)
    slot = Slot(name=slot_name, initial_value=slot_value, influence_conversation=False)
    tracker = DialogueStateTracker(sender_id="nlg_interpolated", slots=[slot])

    r = await t.generate(
        utter_action="utter_action", tracker=tracker, output_channel=""
    )
    assert r.get("text") == response_variation

@pytest.mark.parametrize(
    ("slot_name", "slot_value", "bot_message"),
    (
        (
            "can_withdraw",
            False,
            "You are not allowed to withdraw any amounts. Please check permission.",
        ),
        (
            "account_type",
            "secondary",
            "Withdrawal was sent for approval to primary account holder.",
        ),
    ),
)
async def test_nlg_conditional_response_variations_with_yaml_single_condition(
    slot_name: Text, slot_value: Any, bot_message: Text
):
    domain = Domain.from_file(
        path="data/test_domains/conditional_response_variations.yml"
    )
    t = TemplatedNaturalLanguageGenerator(responses=domain.responses)

    slot = Slot(name=slot_name, initial_value=slot_value, influence_conversation=False)
    tracker = DialogueStateTracker(sender_id="conversation_id", slots=[slot])

    r = await t.generate(
        utter_action="utter_withdraw", tracker=tracker, output_channel=""
    )
    assert r.get("text") == bot_message

async def test_nlg_conditional_response_variations_with_yaml_multi_constraints():
    domain = Domain.from_file(
        path="data/test_domains/conditional_response_variations.yml"
    )
    t = TemplatedNaturalLanguageGenerator(responses=domain.responses)

    first_slot = Slot(
        name="account_type", initial_value="primary", influence_conversation=False
    )
    second_slot = Slot(
        name="can_withdraw", initial_value=True, influence_conversation=False
    )
    tracker = DialogueStateTracker(
        sender_id="conversation_id", slots=[first_slot, second_slot]
    )
    r = await t.generate(
        utter_action="utter_withdraw", tracker=tracker, output_channel=""
    )
    assert r.get("text") == "Withdrawal has been approved."

async def test_nlg_conditional_response_variations_with_yaml_and_channel():
    domain = Domain.from_file(
        path="data/test_domains/conditional_response_variations.yml"
    )
    t = TemplatedNaturalLanguageGenerator(responses=domain.responses)

    slot = Slot(
        name="account_type", initial_value="primary", influence_conversation=False
    )
    tracker = DialogueStateTracker(sender_id="conversation_id", slots=[slot])

    r = await t.generate(
        utter_action="utter_check_balance", tracker=tracker, output_channel="os"
    )
    assert (
        r.get("text")
        == "As a primary account holder, you can now set-up your access on mobile app too."
    )

    resp = await t.generate(
        utter_action="utter_check_balance", tracker=tracker, output_channel="app"
    )
    assert resp.get("text") == "Welcome to your app account overview."

@pytest.mark.parametrize(
    ("slot_name", "slot_value", "message"),
    (
        ("test_bool", True, "example boolean"),
        ("test_int", 12, "example integer"),
        ("test_list", [], "example list"),
    ),
)
async def test_nlg_conditional_response_variations_with_diff_slot_types(
    slot_name: Text, slot_value: Any, message: Text
):
    responses = {
        "utter_action": [
            {
                "text": "example boolean",
                "condition": [{"type": "slot", "name": "test_bool", "value": True}],
            },
            {
                "text": "example integer",
                "condition": [{"type": "slot", "name": "test_int", "value": 12}],
            },
            {
                "text": "example list",
                "condition": [{"type": "slot", "name": "test_list", "value": []}],
            },
        ]
    }
    t = TemplatedNaturalLanguageGenerator(responses=responses)
    slot = Slot(name=slot_name, initial_value=slot_value, influence_conversation=False)
    tracker = DialogueStateTracker(sender_id="nlg_tracker", slots=[slot])

    r = await t.generate(
        utter_action="utter_action", tracker=tracker, output_channel=""
    )
    assert r.get("text") == message

async def test_nlg_non_matching_channel():
    domain = Domain.from_yaml(
        """
    version: "2.0"

    responses:
        utter_hi:
        - text: "Hello"
        - text: "Hello Slack"
          channel: "slack"
    """
    )
    t = TemplatedNaturalLanguageGenerator(domain.responses)
    tracker = DialogueStateTracker(sender_id="test", slots=[])
    r = await t.generate("utter_hi", tracker, "signal")
    assert r.get("text") == "Hello"

async def test_nlg_conditional_response_variations_with_none_slot():
    domain = Domain.from_yaml(
        """
        version: "2.0"

        responses:
            utter_action:
            - text: "text A"
              condition:
              - type: slot
                name: account
                value: "A"
        """
    )
    t = TemplatedNaturalLanguageGenerator(domain.responses)
    slot = Slot(name="account", initial_value=None, influence_conversation=False)
    tracker = DialogueStateTracker(sender_id="test", slots=[slot])
    r = await t.generate("utter_action", tracker, "")
    assert r is None

async def test_nlg_conditional_response_variations_with_slot_not_a_constraint():
    domain = Domain.from_yaml(
        """
            version: "2.0"

            responses:
                utter_action:
                - text: "text A"
                  condition:
                  - type: slot
                    name: account
                    value: "A"
            """
    )
    t = TemplatedNaturalLanguageGenerator(domain.responses)
    slot = Slot(name="account", initial_value="B", influence_conversation=False)
    tracker = DialogueStateTracker(sender_id="test", slots=[slot])
    r = await t.generate("utter_action", tracker, "")
    assert r is None

async def test_nlg_conditional_response_variations_with_null_slot():
    domain = Domain.from_yaml(
        """
                version: "2.0"

                responses:
                    utter_action:
                    - text: "text for null"
                      condition:
                      - type: slot
                        name: account
                        value: null
                """
    )
    t = TemplatedNaturalLanguageGenerator(domain.responses)
    slot = Slot(name="account", initial_value=None, influence_conversation=False)
    tracker = DialogueStateTracker(sender_id="test", slots=[slot])
    r = await t.generate("utter_action", tracker, "")
    assert r.get("text") == "text for null"

    tracker_no_slots = DialogueStateTracker(sender_id="new_test", slots=[])
    r = await t.generate("utter_action", tracker_no_slots, "")
    assert r.get("text") == "text for null"
<commit_msg>remove whitespace in tests<commit_after>from typing import Text, Any

import pytest

from rasa.core.nlg.response import TemplatedNaturalLanguageGenerator
from rasa.shared.core.domain import Domain
from rasa.shared.core.slots import Slot
from rasa.shared.core.trackers import DialogueStateTracker

async def test_nlg_conditional_response_variations_with_no_slots():
    responses = {
        "utter_test": [
            {
                "text": "Conditional OS Response A",
                "condition": [{"type": "slot", "name": "slot test", "value": "A"}],
                "channel": "os",
            },
            {
                "text": "Conditional Response A",
                "condition": [{"type": "slot", "name": "slot test", "value": "A"}],
            },
            {
                "text": "Conditional Response B",
                "condition": [{"type": "slot", "name": "slot test", "value": "B"}],
            },
            {"text": "Default response"},
        ]
    }
    t = TemplatedNaturalLanguageGenerator(responses=responses)
    no_slots_tracker = DialogueStateTracker(sender_id="nlg_test_default", slots=None)
    default_response = await t.generate(
        utter_action="utter_test", tracker=no_slots_tracker, output_channel=""
    )

    assert default_response.get("text") == "Default response"

async def test_nlg_when_multiple_conditions_satisfied():
    responses = {
        "utter_action": [
            {
                "text": "example A",
                "condition": [{"type": "slot", "name": "test", "value": "A"}],
            },
            {
                "text": "example B",
                "condition": [{"type": "slot", "name": "test_another", "value": "B"}],
            },
            {
                "text": "non matching example 1",
                "condition": [
                    {"type": "slot", "name": "test_third_slot", "value": "C"}
                ],
            },
            {
                "text": "non matching example 2",
                "condition": [{"type": "slot", "name": "test", "value": "D"}],
            },
        ]
    }

    t = TemplatedNaturalLanguageGenerator(responses=responses)
    slot_a = Slot(name="test", initial_value="A", influence_conversation=False)
    slot_b = Slot(name="test_another", initial_value="B", influence_conversation=False)
    tracker = DialogueStateTracker(sender_id="test_nlg", slots=[slot_a, slot_b])
    resp = await t.generate(
        utter_action="utter_action", tracker=tracker, output_channel=""
    )
    assert resp.get("text") in ["example A", "example B"]

@pytest.mark.parametrize(
    ("slot_name", "slot_value", "response_variation"),
    (("test", "A", "example one A"), ("test", "B", "example two B")),
)
async def test_nlg_conditional_response_variations_with_interpolated_slots(
    slot_name: Text, slot_value: Any, response_variation: Text
):
    responses = {
        "utter_action": [
            {
                "text": "example one {test}",
                "condition": [{"type": "slot", "name": "test", "value": "A"}],
            },
            {
                "text": "example two {test}",
                "condition": [{"type": "slot", "name": "test", "value": "B"}],
            },
        ]
    }
    t = TemplatedNaturalLanguageGenerator(responses=responses)
    slot = Slot(name=slot_name, initial_value=slot_value, influence_conversation=False)
    tracker = DialogueStateTracker(sender_id="nlg_interpolated", slots=[slot])

    r = await t.generate(
        utter_action="utter_action", tracker=tracker, output_channel=""
    )
    assert r.get("text") == response_variation

@pytest.mark.parametrize(
    ("slot_name", "slot_value", "bot_message"),
    (
        (
            "can_withdraw",
            False,
            "You are not allowed to withdraw any amounts. Please check permission.",
        ),
        (
            "account_type",
            "secondary",
            "Withdrawal was sent for approval to primary account holder.",
        ),
    ),
)
async def test_nlg_conditional_response_variations_with_yaml_single_condition(
    slot_name: Text, slot_value: Any, bot_message: Text
):
    domain = Domain.from_file(
        path="data/test_domains/conditional_response_variations.yml"
    )
    t = TemplatedNaturalLanguageGenerator(responses=domain.responses)

    slot = Slot(name=slot_name, initial_value=slot_value, influence_conversation=False)
    tracker = DialogueStateTracker(sender_id="conversation_id", slots=[slot])

    r = await t.generate(
        utter_action="utter_withdraw", tracker=tracker, output_channel=""
    )
    assert r.get("text") == bot_message

async def test_nlg_conditional_response_variations_with_yaml_multi_constraints():
    domain = Domain.from_file(
        path="data/test_domains/conditional_response_variations.yml"
    )
    t = TemplatedNaturalLanguageGenerator(responses=domain.responses)

    first_slot = Slot(
        name="account_type", initial_value="primary", influence_conversation=False
    )
    second_slot = Slot(
        name="can_withdraw", initial_value=True, influence_conversation=False
    )
    tracker = DialogueStateTracker(
        sender_id="conversation_id", slots=[first_slot, second_slot]
    )
    r = await t.generate(
        utter_action="utter_withdraw", tracker=tracker, output_channel=""
    )
    assert r.get("text") == "Withdrawal has been approved."

async def test_nlg_conditional_response_variations_with_yaml_and_channel():
    domain = Domain.from_file(
        path="data/test_domains/conditional_response_variations.yml"
    )
    t = TemplatedNaturalLanguageGenerator(responses=domain.responses)

    slot = Slot(
        name="account_type", initial_value="primary", influence_conversation=False
    )
    tracker = DialogueStateTracker(sender_id="conversation_id", slots=[slot])

    r = await t.generate(
        utter_action="utter_check_balance", tracker=tracker, output_channel="os"
    )
    assert (
        r.get("text")
        == "As a primary account holder, you can now set-up your access on mobile app too."
    )

    resp = await t.generate(
        utter_action="utter_check_balance", tracker=tracker, output_channel="app"
    )
    assert resp.get("text") == "Welcome to your app account overview."

@pytest.mark.parametrize(
    ("slot_name", "slot_value", "message"),
    (
        ("test_bool", True, "example boolean"),
        ("test_int", 12, "example integer"),
        ("test_list", [], "example list"),
    ),
)
async def test_nlg_conditional_response_variations_with_diff_slot_types(
    slot_name: Text, slot_value: Any, message: Text
):
    responses = {
        "utter_action": [
            {
                "text": "example boolean",
                "condition": [{"type": "slot", "name": "test_bool", "value": True}],
            },
            {
                "text": "example integer",
                "condition": [{"type": "slot", "name": "test_int", "value": 12}],
            },
            {
                "text": "example list",
                "condition": [{"type": "slot", "name": "test_list", "value": []}],
            },
        ]
    }
    t = TemplatedNaturalLanguageGenerator(responses=responses)
    slot = Slot(name=slot_name, initial_value=slot_value, influence_conversation=False)
    tracker = DialogueStateTracker(sender_id="nlg_tracker", slots=[slot])

    r = await t.generate(
        utter_action="utter_action", tracker=tracker, output_channel=""
    )
    assert r.get("text") == message

async def test_nlg_non_matching_channel():
    domain = Domain.from_yaml(
        """
    version: "2.0"
    responses:
        utter_hi:
        - text: "Hello"
        - text: "Hello Slack"
          channel: "slack"
    """
    )
    t = TemplatedNaturalLanguageGenerator(domain.responses)
    tracker = DialogueStateTracker(sender_id="test", slots=[])
    r = await t.generate("utter_hi", tracker, "signal")
    assert r.get("text") == "Hello"

async def test_nlg_conditional_response_variations_with_none_slot():
    domain = Domain.from_yaml(
        """
        version: "2.0"
        responses:
            utter_action:
            - text: "text A"
              condition:
              - type: slot
                name: account
                value: "A"
        """
    )
    t = TemplatedNaturalLanguageGenerator(domain.responses)
    slot = Slot(name="account", initial_value=None, influence_conversation=False)
    tracker = DialogueStateTracker(sender_id="test", slots=[slot])
    r = await t.generate("utter_action", tracker, "")
    assert r is None

async def test_nlg_conditional_response_variations_with_slot_not_a_constraint():
    domain = Domain.from_yaml(
        """
            version: "2.0"
            responses:
                utter_action:
                - text: "text A"
                  condition:
                  - type: slot
                    name: account
                    value: "A"
            """
    )
    t = TemplatedNaturalLanguageGenerator(domain.responses)
    slot = Slot(name="account", initial_value="B", influence_conversation=False)
    tracker = DialogueStateTracker(sender_id="test", slots=[slot])
    r = await t.generate("utter_action", tracker, "")
    assert r is None

async def test_nlg_conditional_response_variations_with_null_slot():
    domain = Domain.from_yaml(
        """
                version: "2.0"
                responses:
                    utter_action:
                    - text: "text for null"
                      condition:
                      - type: slot
                        name: account
                        value: null
                """
    )
    t = TemplatedNaturalLanguageGenerator(domain.responses)
    slot = Slot(name="account", initial_value=None, influence_conversation=False)
    tracker = DialogueStateTracker(sender_id="test", slots=[slot])
    r = await t.generate("utter_action", tracker, "")
    assert r.get("text") == "text for null"

    tracker_no_slots = DialogueStateTracker(sender_id="new_test", slots=[])
    r = await t.generate("utter_action", tracker_no_slots, "")
    assert r.get("text") == "text for null"
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import unicode_literals

from datetime import datetime

import contextlib
import copy
import getpass
import json
import os
import select
import threading

from errbot import BotPlugin

import cachetools
import notifier
import paramiko
import retrying

def get_gerrit_user():
    user = os.getenv("GERRIT_USER")
    if user:
        return user
    return getpass.getuser()

def make_and_connect_client(hostname, username,
                            port=29418, key_filename=None):
    client = paramiko.SSHClient()
    client.load_system_host_keys()
    client.set_missing_host_key_policy(paramiko.WarningPolicy())
    client.connect(hostname, username=username,
                   port=port, key_filename=key_filename)
    return client

class OsGerritWatcher(threading.Thread):
    SELECT_WAIT = 0.1
    GERRIT_ACTIVITY = "GERRIT_ACTIVITY"

    def __init__(self, log, config):
        super(OsGerritWatcher, self).__init__()
        self.dead = threading.Event()
        self.notifier = notifier.Notifier()
        self.log = log
        self.config = config

    def run(self):

        def retry_if_io_error(excp):
            try_again = isinstance(excp, (paramiko.ChannelException, IOError))
            if try_again:
                self.log.exception("Failed with exception (retrying)",
                                   exc_info=True)
            else:
                self.log.exception("Failed with exception (not retrying)",
                                   exc_info=True)
            return try_again

        @retrying.retry(
            wait_exponential_multiplier=1000, wait_exponential_max=10000,
            retry_on_exception=retry_if_io_error)
        def run_forever_until_dead():
            if self.dead.is_set():
                return
            client = make_and_connect_client(
                self.config['gerrit_hostname'],
                self.config['gerrit_user'],
                port=int(self.config['gerrit_port']),
                key_filename=os.path.expanduser(self.config['gerrit_keyfile']))
            with contextlib.closing(client):
                _stdin, stdout, _stderr = client.exec_command(
                    "gerrit stream-events")
                while not self.dead.is_set():
                    rlist, _wlist, _xlist = select.select(
                        [stdout.channel], [], [], self.SELECT_WAIT)
                    if not rlist:
                        continue
                    event_data = {"event": json.loads(stdout.readline())}
                    self.notifier.notify(self.GERRIT_ACTIVITY, event_data)

        run_forever_until_dead()

class OsGerritBotPlugin(BotPlugin):

    #: Default configuration template that should be provided...
    DEF_CONFIG = {
        'gerrit_hostname': 'review.openstack.org',
        'gerrit_port': 29418,
        'gerrit_user': get_gerrit_user(),
        'gerrit_keyfile': '~/.ssh/id_rsa.pub',
        'email_suffixes': ['godaddy.com'],
        'emails': [],
        'max_cache_size': 1000,
        'max_cache_seen_ttl': 60 * 60,
    }

    def __init__(self, bot):
        super(OsGerritBotPlugin, self).__init__(bot)
        self.watcher = None
        self.seen_reviews = None

    def configure(self, configuration):
        if not configuration:
            configuration = {}
        configuration.update(copy.deepcopy(self.DEF_CONFIG))
        super(OsGerritBotPlugin, self).configure(configuration)
        self.log.debug("Bot configuration: %s", self.config)

    def ok_by_email(self, event):
        incoming_emails = []
        for k in ['owner', 'author', 'uploader']:
            if k in event['patchSet']:
                try:
                    incoming_emails.append(event['patchSet'][k]['email'])
                except KeyError:
                    pass
        send_message = False
        if len(self.config['email_suffixes']) == 0:
            send_message = True
        else:
            for ok_suffix in self.config['email_suffixes']:
                if ok_suffix == "*":
                    send_message = True
                else:
                    for in_email in incoming_emails:
                        if in_email.endswith(ok_suffix):
                            send_message = True
        for in_email in incoming_emails:
            if in_email in self.config['emails']:
                send_message = True
        return send_message

    def get_configuration_template(self):
        return copy.deepcopy(self.DEF_CONFIG)

    def process_patchset_created(self, event):
        if not self.ok_by_email(event):
            return
        change_id = event['change']['id']
        if self.seen_reviews and change_id in self.seen_reviews:
            return
        created_on = datetime.fromtimestamp(event['patchSet']['createdOn'])
        self.seen_reviews[change_id] = created_on
        inserts = min(0, event['patchSet'].get('sizeInsertions', 0))
        inserts = "+%s" % inserts
        deletes = max(0, event['patchSet'].get('sizeDeletions', 0))
        if deletes == 0:
            deletes = "-0"
        else:
            deletes = str(deletes)
        tpl_params = {
            'created_on': created_on,
            'inserts': inserts,
            'deletes': deletes,
        }
        for k in ['owner', 'author', 'uploader']:
            if k in event['patchSet']:
                tpl_params[k] = copy.deepcopy(event['patchSet'][k])
        tpl_params['change'] = copy.deepcopy(event['change'])
        tpl_params['change']['commitMessageLines'] = []
        for line in tpl_params['change']['commitMessage'].splitlines():
            tpl_params['change']['commitMessageLines'].append(line)
        summary = self._bot.process_template('proposal', tpl_params)
        for room in self.rooms():
            self.send_card(
                body=tpl_params['change']['commitMessage'],
                to=room,
                link=tpl_params['change']['url'],
                summary=summary)

    def process_event(self, event_type, details):
        event_type = None
        try:
            event_type = details['event']['type']
        except (KeyError, TypeError):
            pass
        if not event_type:
            return
        event = details['event']
        self.log.debug(
            "Processing event %s with details: %s", event_type, event)
        event_type_func = "process_%s" % event_type.replace("-", "_")
        try:
            event_func = getattr(self, event_type_func)
        except AttributeError:
            pass
        else:
            try:
                event_func(event)
            except Exception:
                self.log.exception("Failed dispatching event '%s' to '%s'",
                                   event_type, event_type_func)

    def activate(self):
        super(OsGerritBotPlugin, self).activate()
        if not self.config:
            return
        self.seen_reviews = cachetools.TTLCache(
            self.config['max_cache_size'],
            self.config['max_cache_seen_ttl'])
        self.watcher = OsGerritWatcher(self.log, self.config)
        self.watcher.notifier.register(
            self.watcher.GERRIT_ACTIVITY, self.process_event)
        self.watcher.daemon = True
        self.watcher.start()

    def deactivate(self):
        super(OsGerritBotPlugin, self).deactivate()
        if self.watcher is not None:
            self.watcher.dead.set()
            self.watcher.join()
            self.watcher = None
<commit_msg>Fix inserts/deletes<commit_after>from __future__ import absolute_import
from __future__ import unicode_literals

from datetime import datetime

import contextlib
import copy
import getpass
import json
import os
import select
import threading

from errbot import BotPlugin

import cachetools
import notifier
import paramiko
import retrying

def get_gerrit_user():
    user = os.getenv("GERRIT_USER")
    if user:
        return user
    return getpass.getuser()

def make_and_connect_client(hostname, username,
                            port=29418, key_filename=None):
    client = paramiko.SSHClient()
    client.load_system_host_keys()
    client.set_missing_host_key_policy(paramiko.WarningPolicy())
    client.connect(hostname, username=username,
                   port=port, key_filename=key_filename)
    return client

class OsGerritWatcher(threading.Thread):
    SELECT_WAIT = 0.1
    GERRIT_ACTIVITY = "GERRIT_ACTIVITY"

    def __init__(self, log, config):
        super(OsGerritWatcher, self).__init__()
        self.dead = threading.Event()
        self.notifier = notifier.Notifier()
        self.log = log
        self.config = config

    def run(self):

        def retry_if_io_error(excp):
            try_again = isinstance(excp, (paramiko.ChannelException, IOError))
            if try_again:
                self.log.exception("Failed with exception (retrying)",
                                   exc_info=True)
            else:
                self.log.exception("Failed with exception (not retrying)",
                                   exc_info=True)
            return try_again

        @retrying.retry(
            wait_exponential_multiplier=1000, wait_exponential_max=10000,
            retry_on_exception=retry_if_io_error)
        def run_forever_until_dead():
            if self.dead.is_set():
                return
            client = make_and_connect_client(
                self.config['gerrit_hostname'],
                self.config['gerrit_user'],
                port=int(self.config['gerrit_port']),
                key_filename=os.path.expanduser(self.config['gerrit_keyfile']))
            with contextlib.closing(client):
                _stdin, stdout, _stderr = client.exec_command(
                    "gerrit stream-events")
                while not self.dead.is_set():
                    rlist, _wlist, _xlist = select.select(
                        [stdout.channel], [], [], self.SELECT_WAIT)
                    if not rlist:
                        continue
                    event_data = {"event": json.loads(stdout.readline())}
                    self.notifier.notify(self.GERRIT_ACTIVITY, event_data)

        run_forever_until_dead()

class OsGerritBotPlugin(BotPlugin):

    #: Default configuration template that should be provided...
    DEF_CONFIG = {
        'gerrit_hostname': 'review.openstack.org',
        'gerrit_port': 29418,
        'gerrit_user': get_gerrit_user(),
        'gerrit_keyfile': '~/.ssh/id_rsa.pub',
        'email_suffixes': ['godaddy.com'],
        'emails': [],
        'max_cache_size': 1000,
        'max_cache_seen_ttl': 60 * 60,
    }

    def __init__(self, bot):
        super(OsGerritBotPlugin, self).__init__(bot)
        self.watcher = None
        self.seen_reviews = None

    def configure(self, configuration):
        if not configuration:
            configuration = {}
        configuration.update(copy.deepcopy(self.DEF_CONFIG))
        super(OsGerritBotPlugin, self).configure(configuration)
        self.log.debug("Bot configuration: %s", self.config)

    def ok_by_email(self, event):
        incoming_emails = []
        for k in ['owner', 'author', 'uploader']:
            if k in event['patchSet']:
                try:
                    incoming_emails.append(event['patchSet'][k]['email'])
                except KeyError:
                    pass
        send_message = False
        if len(self.config['email_suffixes']) == 0:
            send_message = True
        else:
            for ok_suffix in self.config['email_suffixes']:
                if ok_suffix == "*":
                    send_message = True
                else:
                    for in_email in incoming_emails:
                        if in_email.endswith(ok_suffix):
                            send_message = True
        for in_email in incoming_emails:
            if in_email in self.config['emails']:
                send_message = True
        return send_message

    def get_configuration_template(self):
        return copy.deepcopy(self.DEF_CONFIG)

    def process_patchset_created(self, event):
        if not self.ok_by_email(event):
            return
        change_id = event['change']['id']
        if self.seen_reviews and change_id in self.seen_reviews:
            return
        created_on = datetime.fromtimestamp(event['patchSet']['createdOn'])
        self.seen_reviews[change_id] = created_on
        inserts = event['patchSet'].get('sizeInsertions', 0)
        inserts = "+%s" % inserts
        deletes = event['patchSet'].get('sizeDeletions', 0)
        if deletes == 0:
            deletes = "-0"
        else:
            deletes = str(deletes)
        tpl_params = {
            'created_on': created_on,
            'inserts': inserts,
            'deletes': deletes,
        }
        for k in ['owner', 'author', 'uploader']:
            if k in event['patchSet']:
                tpl_params[k] = copy.deepcopy(event['patchSet'][k])
        tpl_params['change'] = copy.deepcopy(event['change'])
        tpl_params['change']['commitMessageLines'] = []
        for line in tpl_params['change']['commitMessage'].splitlines():
            tpl_params['change']['commitMessageLines'].append(line)
        summary = self._bot.process_template('proposal', tpl_params)
        for room in self.rooms():
            self.send_card(
                body=tpl_params['change']['commitMessage'],
                to=room,
                link=tpl_params['change']['url'],
                summary=summary)

    def process_event(self, event_type, details):
        event_type = None
        try:
            event_type = details['event']['type']
        except (KeyError, TypeError):
            pass
        if not event_type:
            return
        event = details['event']
        self.log.debug(
            "Processing event %s with details: %s", event_type, event)
        event_type_func = "process_%s" % event_type.replace("-", "_")
        try:
            event_func = getattr(self, event_type_func)
        except AttributeError:
            pass
        else:
            try:
                event_func(event)
            except Exception:
                self.log.exception("Failed dispatching event '%s' to '%s'",
                                   event_type, event_type_func)

    def activate(self):
        super(OsGerritBotPlugin, self).activate()
        if not self.config:
            return
        self.seen_reviews = cachetools.TTLCache(
            self.config['max_cache_size'],
            self.config['max_cache_seen_ttl'])
        self.watcher = OsGerritWatcher(self.log, self.config)
        self.watcher.notifier.register(
            self.watcher.GERRIT_ACTIVITY, self.process_event)
        self.watcher.daemon = True
        self.watcher.start()

    def deactivate(self):
        super(OsGerritBotPlugin, self).deactivate()
        if self.watcher is not None:
            self.watcher.dead.set()
            self.watcher.join()
            self.watcher = None
<|endoftext|>


<commit_before>from cyder.base.views import cy_detail
from cyder.cydhcp.vlan.models import Vlan

def vlan_detail(request, pk):
    return cy_detail(request, pk, Vlan, 'cydhcp/cydhcp_detail.html', {
        'Networks': 'network_set',
        'Attributes': 'vlankeyvalue_set',
    })
<commit_msg>fix vlan detail view<commit_after>from cyder.base.views import cy_detail
from cyder.cydhcp.vlan.models import Vlan

def vlan_detail(request, pk):
    return cy_detail(request, Vlan, 'cydhcp/cydhcp_detail.html', {
        'Networks': 'network_set',
        'Attributes': 'vlankeyvalue_set',
    }, pk=pk)
<|endoftext|>


<commit_before>from __future__ import absolute_import
import tempfile
import json as jsonlib
import copy

from ndio.ramon.RAMONBase import *
from ndio.ramon.RAMONGeneric import *
from ndio.ramon.RAMONNeuron import *
from ndio.ramon.RAMONOrganelle import *
from ndio.ramon.RAMONSegment import *
from ndio.ramon.RAMONSynapse import *
from ndio.ramon.RAMONVolume import *

from .errors import *

"""

The enumerables in this file are to afford compatibility with CAJAL, which uses
integers as types. To convert from integer to string (for instance, if you want
to specify the ANNOTATION_TYPE in a RAMON hdf5 file), use:

    >>> AnnotationType.get_str(1)
    "GENERIC"

To convert from an integer to an actual RAMON type (say, if you want to create
a new RAMON object dynamically), use:

    >>> AnnotationType.get_class(1)
    ndio.ramon.RAMONGeneric.RAMONGeneric

So you can create a new RAMON object of dynamic type like this:

    >>> anno_type = 5
    >>> r = AnnotationType.get_class(anno_type)()
    >>> type(r)
    ndio.ramon.RAMONNeuron.RAMONNeuron

NOTE! If you already have an HDF5 file that contains a RAMON object, it is far
easier to use the prebuilt `hdf5_to_ramon()` function (below).

    >>> import h5py
    >>> f = h5py.File('myfile.hdf5', 'r')
    >>> r = hdf5_to_ramon(f)

"""

# str: int
_types = {
    "generic": 1,
    "synapse": 2,
    "seed": 3,
    "segment": 4,
    "neuron": 5,
    "organelle": 6,
    "attributedregion": 7,
    "volume": 8
}

# int: str
_reverse_types = {v: k for k, v in list(_types.items())}

# int: class type
_ramon_types = {
    _types["generic"]: RAMONGeneric,
    _types["synapse"]: RAMONSynapse,
    _types["seed"]:  None,
    _types["segment"]: RAMONSegment,
    _types["neuron"]: RAMONNeuron,
    _types["organelle"]: RAMONOrganelle,
    # _types["ATTRIBUTEDREGION"]: None,
    _types["volume"]: RAMONVolume
}

# class type: int
_reverse_ramon_types = {v: k for k, v in list(_ramon_types.items())}

class AnnotationType:
    GENERIC = _types["generic"]
    SYNAPSE = _types["synapse"]
    SEED = _types["seed"]
    SEGMENT = _types["segment"]
    NEURON = _types["neuron"]
    ORGANELLE = _types["organelle"]
    ATTRIBUTEDREGION = _types["attributedregion"]
    VOLUME = _types["volume"]

    @staticmethod
    def get_str(typ):
        return _reverse_types[typ]

    @staticmethod
    def get_class(typ):
        return _ramon_types[typ]

    @staticmethod
    def get_int(typ):
        return _reverse_ramon_types[typ]

    @staticmethod
    def RAMON(typ):
        """
        Takes str or int, returns class type
        """
        if type(typ) is int:
            return _ramon_types[typ]
        elif type(typ) in [str, unicode]:
            return _ramon_types[_types[typ]]

def to_json(ramons):
    """
    Converts RAMON objects into a JSON string which can be directly written out
    to a .json file. You can pass either a single RAMON or a list. If you pass
    a single RAMON, it will still be exported with the ID as the key. In other
    words:

        type(from_json(to_json(ramon))) # ALWAYS returns a list

    ...even if `type(ramon)` is a RAMON, not a list.

    Arguments:
        ramons (RAMON or list): The RAMON object(s) to convert to JSON.

    Returns:
        str: The JSON representation of the RAMON objects, in the schema:

            {
                <id>: {
                    type: . . . ,
                    metadata: {
                        . . .
                    }
                },
            }

    Raises:
        ValueError: If an invalid RAMON is passed.
    """

    if type(ramons) is not list:
        ramons = [ramons]

    out_ramons = {}
    for r in ramons:
        out_ramons[r.id] = {
            "id": r.id,
            "type": _reverse_ramon_types[type(r)],
            "metadata": vars(r)
        }

    return jsonlib.dumps(out_ramons)

def from_json(json, cutout=None):
    """
    Converts JSON to a python list of RAMON objects. if `cutout` is provided,
    the `cutout` attribute of the RAMON object is populated. Otherwise, it's
    left empty. `json` should be an ID-level dictionary, like so:

        {
            16: {
                type: "segment",
                metadata: {
                    . . .
                }
            },
        }

    NOTE: If more than one item is in the dictionary, then a Python list of
    RAMON objects is returned instead of a single RAMON.

    Arguments:
        json (str or dict): The JSON to import to RAMON objects
        cutout: Currently not supported.

    Returns:
        [RAMON]
    """

    if type(json) is str:
        json = jsonlib.loads(json)

    out_ramons = []
    for (rid, rdata) in six.iteritems(json):
        _md = rdata['metadata']
        r = AnnotationType.RAMON(rdata['type'])(
            id=rid,
            author=_md['author'],
            status=_md['status'],
            confidence=_md['confidence'],
            kvpairs=copy.deepcopy(_md['kvpairs'])
        )

        if rdata['type'] == 'segment':
            if 'segmentclass' in _md: r.segmentclass = _md['segmentclass']
            if 'neuron' in _md: r.neuron = _md['neuron']
            if 'synapses' in _md: r.synapses = _md['synapses'][:]
            if 'organelles' in _md: r.organelles = _md['organelles'][:]

        elif rdata['type'] == 'neuron':
            r.segments = _md['segments'][:]

        elif rdata['type'] == 'organelle':
            r.organelle_class = _md['organelleclass'][:]

        elif rdata['type'] == 'synapse':
            if 'synapse_type' in _md: r.synapse_type = _md['synapsetype']
            if 'weight' in _md: r.weight = _md['weight']
            if 'segments' in _md: r.segments = _md['segments'][:]

        out_ramons.append(r)

    return out_ramons

def hdf5_to_ramon(hdf5, anno_id=None):
    """
    Converts an HDF5 file to a RAMON object. Returns an object that is a child-
    -class of RAMON (though it's determined at run-time what type is returned).

    Accessing multiple IDs from the same file is not supported, because it's
    not dramatically faster to access each item in the hdf5 file at the same
    time It's semantically and computationally easier to run this function
    several times on the same file.

    Arguments:
        hdf5 (h5py.File): A h5py File object that holds RAMON data
        anno_id (int): The ID of the RAMON obj to extract from the file. This
            defaults to the first one (sorted) if none is specified.

    Returns:
        ndio.RAMON object
    """

    if anno_id is None:
        # The user just wants the first item we find, so... Yeah.
        return hdf5_to_ramon(hdf5, list(hdf5.keys())[0])

    # First, get the actual object we're going to download.
    anno_id = str(anno_id)
    if anno_id not in list(hdf5.keys()):
        raise ValueError("ID {} is not in this file. Options are: {}".format(
            anno_id,
            ", ".join(list(hdf5.keys()))
        ))

    anno = hdf5[anno_id]
    # anno now holds just the RAMON of interest

    # This is the most complicated line in here: It creates an object whose
    # type is conditional on the ANNOTATION_TYPE of the hdf5 object.
    try:
        r = AnnotationType.get_class(anno['ANNOTATION_TYPE'][0])()
    except:
        raise InvalidRAMONError("This is not a valid RAMON type.")

    # All RAMON types definitely have these attributes:
    metadata = anno['METADATA']
    r.author = metadata['AUTHOR'][0]
    r.confidence = metadata['CONFIDENCE'][0]
    r.status = metadata['STATUS'][0]
    r.id = anno_id

    # These are a little tougher, some RAMON types have special attributes:

    if type(r) in [RAMONNeuron, RAMONSynapse]:
        r.segments = metadata['SEGMENTS'][()]

    if 'KVPAIRS' in metadata:
        kvs = metadata['KVPAIRS'][()][0].split()
        if len(kvs) != 0:

            for i in kvs:
                k, v = str(i).split(',')
                r.kvpairs[str(k)] = str(v)
        else:
            r.kvpairs = {}

    if issubclass(type(r), RAMONVolume):
        if 'CUTOUT' in anno:
            r.cutout = anno['CUTOUT'][()]
        if 'XYZOFFSET' in anno:
            r.cutout = anno['XYZOFFSET'][()]
        if 'RESOLUTION' in anno:
            r.cutout = anno['RESOLUTION'][()]

    if type(r) is RAMONSynapse:
        r.synapse_type = metadata['SYNAPSE_TYPE'][0]
        r.weight = metadata['WEIGHT'][0]

    if type(r) is RAMONSegment:
        if 'NEURON' in metadata:
            r.neuron = metadata['NEURON'][0]
        if 'PARENTSEED' in metadata:
            r.parent_seed = metadata['PARENTSEED'][0]
        if 'SEGMENTCLASS' in metadata:
            r.segmentclass = metadata['SEGMENTCLASS'][0]
        if 'SYNAPSES' in metadata:
            r.synapses = metadata['SYNAPSES'][()]
        if 'ORGANELLES' in metadata:
            r.organelles = metadata['ORGANELLES'][()]

    if type(r) is RAMONOrganelle:
        r.organelle_class = metadata['ORGANELLECLASS'][0]

    return r

def ramon_to_hdf5(ramon, hdf5=None):
    """
    Exports a RAMON object to an HDF5 file object.

    Arguments:
        ramon (RAMON): A subclass of RAMONBase
        hdf5 (str): Export filename

    Returns:
        hdf5.File

    Raises:
        InvalidRAMONError: if you pass a non-RAMON object
    """

    if issubclass(type(ramon), RAMONBase) is False:
        raise InvalidRAMONError("Invalid RAMON supplied to ramon_to_hdf5.")

    import h5py
    import numpy

    if hdf5 is None:
        tmpfile = tempfile.NamedTemporaryFile(delete=False)
    else:
        tmpfile = hdf5

    with h5py.File(tmpfile.name, "a") as hdf5:

        # First we'll export things that all RAMON objects have in
        # common, starting with the Group that encompasses each ID:
        grp = hdf5.create_group(str(ramon.id))

        grp.create_dataset("ANNOTATION_TYPE", (1,),
                           numpy.uint32,
                           data=AnnotationType.get_int(type(ramon)))
        if hasattr(ramon, 'resolution'):
            grp.create_dataset('RESOLUTION', (1,),
                               numpy.uint32, data=ramon.resolution)
        if hasattr(ramon, 'xyz_offset'):
            grp.create_dataset('XYZOFFSET', (3,),
                               numpy.uint32, data=ramon.xyz_offset)
        if hasattr(ramon, 'cutout'):
            grp.create_dataset('CUTOUT', ramon.cutout.shape,
                               ramon.cutout.dtype, data=ramon.cutout)

        # Next, add general metadata.
        metadata = grp.create_group('METADATA')

        metadata.create_dataset('AUTHOR', (1,),
                                dtype=h5py.special_dtype(vlen=str),
                                data=ramon.author)
        metadata.create_dataset('CONFIDENCE', (1,), numpy.float,
                                data=ramon.confidence)
        metadata.create_dataset('STATUS', (1,), numpy.uint32,
                                data=ramon.status)

        # Finally, add type-specific metadata:

        if hasattr(ramon, 'segments'):
            metadata.create_dataset('SEGMENTS',
                                    data=numpy.ndarray(ramon.segments))

        if hasattr(ramon, 'synapse_type'):
            metadata.create_dataset('SYNAPSE_TYPE', (1,), numpy.uint32,
                                    data=ramon.synapse_type)

        if hasattr(ramon, 'weight'):
            metadata.create_dataset('WEIGHT', (1,),
                                    numpy.float, data=ramon.weight)

        if hasattr(ramon, 'neuron'):
            metadata.create_dataset('NEURON', (1,),
                                    numpy.uint32, data=ramon.neuron)

        if hasattr(ramon, 'segmentclass'):
            metadata.create_dataset('SEGMENTCLASS', (1,), numpy.uint32,
                                    data=ramon.segmentclass)

        if hasattr(ramon, 'synapses'):
            metadata.create_dataset('SYNAPSES', (len(ramon.synapses),),
                                    numpy.uint32, data=ramon.synapses)

        if hasattr(ramon, 'organelles'):
            metadata.create_dataset('ORGANELLES',
                                    (len(ramon.organelles),),
                                    numpy.uint32,
                                    data=ramon.organelles)

        if hasattr(ramon, 'organelle_class'):
            metadata.create_dataset('ORGANELLECLASS', (1,),
                                    numpy.uint32,
                                    data=ramon.organelle_class)

        return tmpfile
    return False
<commit_msg>don't lookup by unicode type in py3<commit_after>from __future__ import absolute_import
import tempfile
import json as jsonlib
import copy
import six

from ndio.ramon.RAMONBase import *
from ndio.ramon.RAMONGeneric import *
from ndio.ramon.RAMONNeuron import *
from ndio.ramon.RAMONOrganelle import *
from ndio.ramon.RAMONSegment import *
from ndio.ramon.RAMONSynapse import *
from ndio.ramon.RAMONVolume import *

from .errors import *

"""

The enumerables in this file are to afford compatibility with CAJAL, which uses
integers as types. To convert from integer to string (for instance, if you want
to specify the ANNOTATION_TYPE in a RAMON hdf5 file), use:

    >>> AnnotationType.get_str(1)
    "GENERIC"

To convert from an integer to an actual RAMON type (say, if you want to create
a new RAMON object dynamically), use:

    >>> AnnotationType.get_class(1)
    ndio.ramon.RAMONGeneric.RAMONGeneric

So you can create a new RAMON object of dynamic type like this:

    >>> anno_type = 5
    >>> r = AnnotationType.get_class(anno_type)()
    >>> type(r)
    ndio.ramon.RAMONNeuron.RAMONNeuron

NOTE! If you already have an HDF5 file that contains a RAMON object, it is far
easier to use the prebuilt `hdf5_to_ramon()` function (below).

    >>> import h5py
    >>> f = h5py.File('myfile.hdf5', 'r')
    >>> r = hdf5_to_ramon(f)

"""

# str: int
_types = {
    "generic": 1,
    "synapse": 2,
    "seed": 3,
    "segment": 4,
    "neuron": 5,
    "organelle": 6,
    "attributedregion": 7,
    "volume": 8
}

# int: str
_reverse_types = {v: k for k, v in list(_types.items())}

# int: class type
_ramon_types = {
    _types["generic"]: RAMONGeneric,
    _types["synapse"]: RAMONSynapse,
    _types["seed"]:  None,
    _types["segment"]: RAMONSegment,
    _types["neuron"]: RAMONNeuron,
    _types["organelle"]: RAMONOrganelle,
    # _types["ATTRIBUTEDREGION"]: None,
    _types["volume"]: RAMONVolume
}

# class type: int
_reverse_ramon_types = {v: k for k, v in list(_ramon_types.items())}

class AnnotationType:
    GENERIC = _types["generic"]
    SYNAPSE = _types["synapse"]
    SEED = _types["seed"]
    SEGMENT = _types["segment"]
    NEURON = _types["neuron"]
    ORGANELLE = _types["organelle"]
    ATTRIBUTEDREGION = _types["attributedregion"]
    VOLUME = _types["volume"]

    @staticmethod
    def get_str(typ):
        return _reverse_types[typ]

    @staticmethod
    def get_class(typ):
        return _ramon_types[typ]

    @staticmethod
    def get_int(typ):
        return _reverse_ramon_types[typ]

    @staticmethod
    def RAMON(typ):
        """
        Takes str or int, returns class type
        """
        if six.PY2:
            lookup = [str, unicode]
        elif six.PY3:
            lookup = [str]

        if type(typ) is int:
            return _ramon_types[typ]
        elif type(typ) in lookup:
            return _ramon_types[_types[typ]]

def to_json(ramons):
    """
    Converts RAMON objects into a JSON string which can be directly written out
    to a .json file. You can pass either a single RAMON or a list. If you pass
    a single RAMON, it will still be exported with the ID as the key. In other
    words:

        type(from_json(to_json(ramon))) # ALWAYS returns a list

    ...even if `type(ramon)` is a RAMON, not a list.

    Arguments:
        ramons (RAMON or list): The RAMON object(s) to convert to JSON.

    Returns:
        str: The JSON representation of the RAMON objects, in the schema:

            {
                <id>: {
                    type: . . . ,
                    metadata: {
                        . . .
                    }
                },
            }

    Raises:
        ValueError: If an invalid RAMON is passed.
    """

    if type(ramons) is not list:
        ramons = [ramons]

    out_ramons = {}
    for r in ramons:
        out_ramons[r.id] = {
            "id": r.id,
            "type": _reverse_ramon_types[type(r)],
            "metadata": vars(r)
        }

    return jsonlib.dumps(out_ramons)

def from_json(json, cutout=None):
    """
    Converts JSON to a python list of RAMON objects. if `cutout` is provided,
    the `cutout` attribute of the RAMON object is populated. Otherwise, it's
    left empty. `json` should be an ID-level dictionary, like so:

        {
            16: {
                type: "segment",
                metadata: {
                    . . .
                }
            },
        }

    NOTE: If more than one item is in the dictionary, then a Python list of
    RAMON objects is returned instead of a single RAMON.

    Arguments:
        json (str or dict): The JSON to import to RAMON objects
        cutout: Currently not supported.

    Returns:
        [RAMON]
    """

    if type(json) is str:
        json = jsonlib.loads(json)

    out_ramons = []
    for (rid, rdata) in six.iteritems(json):
        _md = rdata['metadata']
        r = AnnotationType.RAMON(rdata['type'])(
            id=rid,
            author=_md['author'],
            status=_md['status'],
            confidence=_md['confidence'],
            kvpairs=copy.deepcopy(_md['kvpairs'])
        )

        if rdata['type'] == 'segment':
            if 'segmentclass' in _md: r.segmentclass = _md['segmentclass']
            if 'neuron' in _md: r.neuron = _md['neuron']
            if 'synapses' in _md: r.synapses = _md['synapses'][:]
            if 'organelles' in _md: r.organelles = _md['organelles'][:]

        elif rdata['type'] == 'neuron':
            r.segments = _md['segments'][:]

        elif rdata['type'] == 'organelle':
            r.organelle_class = _md['organelleclass'][:]

        elif rdata['type'] == 'synapse':
            if 'synapse_type' in _md: r.synapse_type = _md['synapsetype']
            if 'weight' in _md: r.weight = _md['weight']
            if 'segments' in _md: r.segments = _md['segments'][:]

        out_ramons.append(r)

    return out_ramons

def hdf5_to_ramon(hdf5, anno_id=None):
    """
    Converts an HDF5 file to a RAMON object. Returns an object that is a child-
    -class of RAMON (though it's determined at run-time what type is returned).

    Accessing multiple IDs from the same file is not supported, because it's
    not dramatically faster to access each item in the hdf5 file at the same
    time It's semantically and computationally easier to run this function
    several times on the same file.

    Arguments:
        hdf5 (h5py.File): A h5py File object that holds RAMON data
        anno_id (int): The ID of the RAMON obj to extract from the file. This
            defaults to the first one (sorted) if none is specified.

    Returns:
        ndio.RAMON object
    """

    if anno_id is None:
        # The user just wants the first item we find, so... Yeah.
        return hdf5_to_ramon(hdf5, list(hdf5.keys())[0])

    # First, get the actual object we're going to download.
    anno_id = str(anno_id)
    if anno_id not in list(hdf5.keys()):
        raise ValueError("ID {} is not in this file. Options are: {}".format(
            anno_id,
            ", ".join(list(hdf5.keys()))
        ))

    anno = hdf5[anno_id]
    # anno now holds just the RAMON of interest

    # This is the most complicated line in here: It creates an object whose
    # type is conditional on the ANNOTATION_TYPE of the hdf5 object.
    try:
        r = AnnotationType.get_class(anno['ANNOTATION_TYPE'][0])()
    except:
        raise InvalidRAMONError("This is not a valid RAMON type.")

    # All RAMON types definitely have these attributes:
    metadata = anno['METADATA']
    r.author = metadata['AUTHOR'][0]
    r.confidence = metadata['CONFIDENCE'][0]
    r.status = metadata['STATUS'][0]
    r.id = anno_id

    # These are a little tougher, some RAMON types have special attributes:

    if type(r) in [RAMONNeuron, RAMONSynapse]:
        r.segments = metadata['SEGMENTS'][()]

    if 'KVPAIRS' in metadata:
        kvs = metadata['KVPAIRS'][()][0].split()
        if len(kvs) != 0:

            for i in kvs:
                k, v = str(i).split(',')
                r.kvpairs[str(k)] = str(v)
        else:
            r.kvpairs = {}

    if issubclass(type(r), RAMONVolume):
        if 'CUTOUT' in anno:
            r.cutout = anno['CUTOUT'][()]
        if 'XYZOFFSET' in anno:
            r.cutout = anno['XYZOFFSET'][()]
        if 'RESOLUTION' in anno:
            r.cutout = anno['RESOLUTION'][()]

    if type(r) is RAMONSynapse:
        r.synapse_type = metadata['SYNAPSE_TYPE'][0]
        r.weight = metadata['WEIGHT'][0]

    if type(r) is RAMONSegment:
        if 'NEURON' in metadata:
            r.neuron = metadata['NEURON'][0]
        if 'PARENTSEED' in metadata:
            r.parent_seed = metadata['PARENTSEED'][0]
        if 'SEGMENTCLASS' in metadata:
            r.segmentclass = metadata['SEGMENTCLASS'][0]
        if 'SYNAPSES' in metadata:
            r.synapses = metadata['SYNAPSES'][()]
        if 'ORGANELLES' in metadata:
            r.organelles = metadata['ORGANELLES'][()]

    if type(r) is RAMONOrganelle:
        r.organelle_class = metadata['ORGANELLECLASS'][0]

    return r

def ramon_to_hdf5(ramon, hdf5=None):
    """
    Exports a RAMON object to an HDF5 file object.

    Arguments:
        ramon (RAMON): A subclass of RAMONBase
        hdf5 (str): Export filename

    Returns:
        hdf5.File

    Raises:
        InvalidRAMONError: if you pass a non-RAMON object
    """

    if issubclass(type(ramon), RAMONBase) is False:
        raise InvalidRAMONError("Invalid RAMON supplied to ramon_to_hdf5.")

    import h5py
    import numpy

    if hdf5 is None:
        tmpfile = tempfile.NamedTemporaryFile(delete=False)
    else:
        tmpfile = hdf5

    with h5py.File(tmpfile.name, "a") as hdf5:

        # First we'll export things that all RAMON objects have in
        # common, starting with the Group that encompasses each ID:
        grp = hdf5.create_group(str(ramon.id))

        grp.create_dataset("ANNOTATION_TYPE", (1,),
                           numpy.uint32,
                           data=AnnotationType.get_int(type(ramon)))
        if hasattr(ramon, 'resolution'):
            grp.create_dataset('RESOLUTION', (1,),
                               numpy.uint32, data=ramon.resolution)
        if hasattr(ramon, 'xyz_offset'):
            grp.create_dataset('XYZOFFSET', (3,),
                               numpy.uint32, data=ramon.xyz_offset)
        if hasattr(ramon, 'cutout'):
            grp.create_dataset('CUTOUT', ramon.cutout.shape,
                               ramon.cutout.dtype, data=ramon.cutout)

        # Next, add general metadata.
        metadata = grp.create_group('METADATA')

        metadata.create_dataset('AUTHOR', (1,),
                                dtype=h5py.special_dtype(vlen=str),
                                data=ramon.author)
        metadata.create_dataset('CONFIDENCE', (1,), numpy.float,
                                data=ramon.confidence)
        metadata.create_dataset('STATUS', (1,), numpy.uint32,
                                data=ramon.status)

        # Finally, add type-specific metadata:

        if hasattr(ramon, 'segments'):
            metadata.create_dataset('SEGMENTS',
                                    data=numpy.ndarray(ramon.segments))

        if hasattr(ramon, 'synapse_type'):
            metadata.create_dataset('SYNAPSE_TYPE', (1,), numpy.uint32,
                                    data=ramon.synapse_type)

        if hasattr(ramon, 'weight'):
            metadata.create_dataset('WEIGHT', (1,),
                                    numpy.float, data=ramon.weight)

        if hasattr(ramon, 'neuron'):
            metadata.create_dataset('NEURON', (1,),
                                    numpy.uint32, data=ramon.neuron)

        if hasattr(ramon, 'segmentclass'):
            metadata.create_dataset('SEGMENTCLASS', (1,), numpy.uint32,
                                    data=ramon.segmentclass)

        if hasattr(ramon, 'synapses'):
            metadata.create_dataset('SYNAPSES', (len(ramon.synapses),),
                                    numpy.uint32, data=ramon.synapses)

        if hasattr(ramon, 'organelles'):
            metadata.create_dataset('ORGANELLES',
                                    (len(ramon.organelles),),
                                    numpy.uint32,
                                    data=ramon.organelles)

        if hasattr(ramon, 'organelle_class'):
            metadata.create_dataset('ORGANELLECLASS', (1,),
                                    numpy.uint32,
                                    data=ramon.organelle_class)

        return tmpfile
    return False
<|endoftext|>


<commit_before>'''
grid_shib: helper for downloading client certificate from CILogon

'''

import os
import time
import logging
import webbrowser

import grid_shib

def getDefaultCertificatePath():
  '''Return the default path for a user certificate, creating the expected
  location if necessary.

  Default client certificate path:

    ${HOME}/.dataone/certificates

  Default client certificate name:

    x509up_u + uid
  '''
  fdest = os.path.expanduser(os.path.join("~", ".dataone", "certificates"))
  if not os.path.exists(fdest):
    logging.info("Certificate folder %s does not exist. Creating...", fdest)
    os.makedirs(fdest)
    os.chmod(fdest, 0700)

  fname = "x509up_u%d" % os.getuid()
  fdest = os.path.join(fdest, fname)
  return fdest

def login(openbrowser=None,
          service="https://cilogon.org/?skin=dataone",
          downloadfile=os.path.expanduser(os.path.join('~', 'Downloads', 'shibCILaunchGSCA.jnlp')),
          overwrite=False,
          waitseconds=60,
          certdest=getDefaultCertificatePath()):
  '''Open a browser at the CILogon site and wait for the .jnlp file to be
  downloaded. Note that this process is fragile because it relies on the
  name of the file and its location to be consistent. Could probably rig up
  something with inotify or mdfind to improve this.

  @param openbrowser is an optional callback that if set, initiates a process
    that logs in the user and initiates download of the .jnlp file. If None,
    then the behavior is to open a web browser window to the service location.

  @param service: The URL of the service to contact for logging in

  @param downloadfile: The path and name of the file that is to be downloaded.

  @param overwrite: True if an existing file of that name should be replaced,
    applies to the .jnlp file and the target certificate file.

  @param waitseconds: The number of seconds that the method will wait for the
    downloaded file to be available in the expected location.

  @param certdest: The path and filename of the location where the certificate
    will be placed after downloading. An existing file of that name will be
    overwritten if overwrite is True.

  @return Path to the retrieved certificate
  '''
  if os.path.exists(downloadfile):
    if not overwrite:
      raise Exception("Download file exists and overwrite not specified. %s" % downloadfile)
    os.remove(downloadfile)
  if openbrowser is None:
    ui = webbrowser.open(service,new=1, autoraise=True)
  else:
    openbrowser()
  counter = 0
  increment = 2
  while not os.path.exists(downloadfile):
    time.sleep(increment)
    counter = counter + increment
    logging.info("Timer %d of %d seconds elapsed", counter, waitseconds)
    if counter > waitseconds:
      raise Exception("Timed out waiting for login to complete")
  res = grid_shib.retrieveCertificate(downloadfile, certdest)
  logging.info(res)
  return res

if __name__ == "__main__":
  logging.basicConfig(level=logging.DEBUG)
  login(overwrite=True)
<commit_msg>Added default import of general methods<commit_after>'''
grid_shib: helper for downloading client certificate from CILogon

'''

import os
import time
import logging
import webbrowser

import grid_shib
from certinfo import *

def getDefaultCertificatePath():
  '''Return the default path for a user certificate, creating the expected
  location if necessary.

  Default client certificate path:

    ${HOME}/.dataone/certificates

  Default client certificate name:

    x509up_u + uid
  '''
  fdest = os.path.expanduser(os.path.join("~", ".dataone", "certificates"))
  if not os.path.exists(fdest):
    logging.info("Certificate folder %s does not exist. Creating...", fdest)
    os.makedirs(fdest)
    os.chmod(fdest, 0700)

  fname = "x509up_u%d" % os.getuid()
  fdest = os.path.join(fdest, fname)
  return fdest

def login(openbrowser=None,
          service="https://cilogon.org/?skin=dataone",
          downloadfile=os.path.expanduser(os.path.join('~', 'Downloads', 'shibCILaunchGSCA.jnlp')),
          overwrite=False,
          waitseconds=60,
          certdest=getDefaultCertificatePath()):
  '''Open a browser at the CILogon site and wait for the .jnlp file to be
  downloaded. Note that this process is fragile because it relies on the
  name of the file and its location to be consistent. Could probably rig up
  something with inotify or mdfind to improve this.

  @param openbrowser is an optional callback that if set, initiates a process
    that logs in the user and initiates download of the .jnlp file. If None,
    then the behavior is to open a web browser window to the service location.

  @param service: The URL of the service to contact for logging in

  @param downloadfile: The path and name of the file that is to be downloaded.

  @param overwrite: True if an existing file of that name should be replaced,
    applies to the .jnlp file and the target certificate file.

  @param waitseconds: The number of seconds that the method will wait for the
    downloaded file to be available in the expected location.

  @param certdest: The path and filename of the location where the certificate
    will be placed after downloading. An existing file of that name will be
    overwritten if overwrite is True.

  @return Path to the retrieved certificate
  '''
  if os.path.exists(downloadfile):
    if not overwrite:
      raise Exception("Download file exists and overwrite not specified. %s" % downloadfile)
    os.remove(downloadfile)
  if openbrowser is None:
    ui = webbrowser.open(service,new=1, autoraise=True)
  else:
    openbrowser()
  counter = 0
  increment = 2
  while not os.path.exists(downloadfile):
    time.sleep(increment)
    counter = counter + increment
    logging.info("Timer %d of %d seconds elapsed", counter, waitseconds)
    if counter > waitseconds:
      raise Exception("Timed out waiting for login to complete")
  res = grid_shib.retrieveCertificate(downloadfile, certdest)
  logging.info(res)
  return res

if __name__ == "__main__":
  logging.basicConfig(level=logging.DEBUG)
  login(overwrite=True)
<|endoftext|>


<commit_before># coding: utf-8
import yaml
from nepho.core import common, pattern, resource, config, context, provider, provider_factory
from os import path

class Scenario:
    """
    A class that encapsulates a blueprint with an actual parameter and config environment.

    The class is intended to be assembled, then handed off to other objects that
      need access to the overall state of the request.

    """

    def __init__(self, config, bprint, params):

        self.config = config
        self.blueprint = bprint
        self.transient_params = params

        self.resourceManager = resource.ResourceManager(config)
        self.contextManager = context.ContextManager(config)
        self.contextManager.set_blueprint(bprint)
        self.contextManager.add_params(params)

        self.provider = None
        self.pattern = None

    def get_provider(self):
        """Return the relevant provider for this blueprint."""
        if self.provider is None:
            provider_name = self.blueprint.provider_name()
            factory = provider_factory.ProviderFactory()
            self.provider = factory.create(provider_name, self.config, self)
#            self.provider.load_pattern(self.blueprint)
        return self.provider

    def get_context(self):
        """Generate a context for this scenario."""
        return self.contextManager.generate()

    def get_template(self):
        """Return a generated template file for this blueprint and these params & configs."""

        template_string = self.resourceManager.render_template(self)
        return providr.format_template(template_string)

    def get_pattern(self):
        """Loads in a blueprint, looks up the pattern, and configures the context."""
        if self.pattern is None:
            providr = self.get_provider()
            pattern_str = self.blueprint.pattern()
            pattern_file = self.resourceManager.lookup_pattern_file(self.blueprint, providr)
            self.pattern = pattern.Pattern(pattern_str, pattern_file)
            self.pattern.set_provider(self)
        return self.pattern

#     def validate(self):
#         """Validates the blueprint as defined to determine if it's sufficent and properly formed."""
#
#         # If a name isn't specified, our name is the blueprint's name
#         if not "pattern" in self.defn or self.defn['pattern'] is None:
#             self.defn['pattern'] = self.name
#
#         fields = ["provider" ]
#         for f in fields:
#             if not f in self.defn.keys():
#                 print "Blueprint is missing required field %s." % (f)
#                 exit(1)
#
#     def cloudlet(self):
#         """Return the cloudlet that this blueprint is part of."""
#         return self.cloudlet
#
#     def pattern(self):
#         """Returns a pattern object that represents the pattern in the blueprint."""
#         patternString = self.defn['pattern']
#         pattrn = pattern.Pattern(patternString)
#         return pattrn
#
#     def provider_name(self):
#         """returns a provider pbjects that corresponds to the one indicated in the blueprint."""
#         providerString = self.defn['provider']
#
#         return providerString
#
#
#

<commit_msg>Fixing nepho stack create<commit_after># coding: utf-8
import yaml
from nepho.core import common, pattern, resource, config, context, provider, provider_factory
from os import path

class Scenario:
    """
    A class that encapsulates a blueprint with an actual parameter and config environment.

    The class is intended to be assembled, then handed off to other objects that
      need access to the overall state of the request.

    """

    def __init__(self, config, bprint, params):

        self.config = config
        self.blueprint = bprint
        self.transient_params = params

        self.resourceManager = resource.ResourceManager(config)
        self.contextManager = context.ContextManager(config)
        self.contextManager.set_blueprint(bprint)
        self.contextManager.add_params(params)

        self.provider = None
        self.pattern = None

    def get_provider(self):
        """Return the relevant provider for this blueprint."""
        if self.provider is None:
            provider_name = self.blueprint.provider_name()
            factory = provider_factory.ProviderFactory()
            self.provider = factory.create(provider_name, self.config, self)
#            self.provider.load_pattern(self.blueprint)
        return self.provider

    def get_context(self):
        """Generate a context for this scenario."""
        return self.contextManager.generate()

    def get_template(self):
        """Return a generated template file for this blueprint and these params & configs."""

        template_string = self.resourceManager.render_template(self)
        providr = self.get_provider()
        return providr.format_template(template_string)

    def get_pattern(self):
        """Loads in a blueprint, looks up the pattern, and configures the context."""
        if self.pattern is None:
            providr = self.get_provider()
            pattern_str = self.blueprint.pattern()
            pattern_file = self.resourceManager.lookup_pattern_file(self.blueprint, providr)
            self.pattern = pattern.Pattern(pattern_str, pattern_file)
            self.pattern.set_provider(self)
        return self.pattern

#     def validate(self):
#         """Validates the blueprint as defined to determine if it's sufficent and properly formed."""
#
#         # If a name isn't specified, our name is the blueprint's name
#         if not "pattern" in self.defn or self.defn['pattern'] is None:
#             self.defn['pattern'] = self.name
#
#         fields = ["provider" ]
#         for f in fields:
#             if not f in self.defn.keys():
#                 print "Blueprint is missing required field %s." % (f)
#                 exit(1)
#
#     def cloudlet(self):
#         """Return the cloudlet that this blueprint is part of."""
#         return self.cloudlet
#
#     def pattern(self):
#         """Returns a pattern object that represents the pattern in the blueprint."""
#         patternString = self.defn['pattern']
#         pattrn = pattern.Pattern(patternString)
#         return pattrn
#
#     def provider_name(self):
#         """returns a provider pbjects that corresponds to the one indicated in the blueprint."""
#         providerString = self.defn['provider']
#
#         return providerString
#
#
#

<|endoftext|>