<commit_before>from typing import Any, Optional, Tuple, Text, Dict, Set, List, Union

import numpy as np
import scipy.sparse
import typing

from rasa.exceptions import RasaException
from rasa.nlu.constants import (
    ENTITIES,
    INTENT,
    RESPONSE,
    RESPONSE_KEY_ATTRIBUTE,
    TEXT,
    RESPONSE_IDENTIFIER_DELIMITER,
    FEATURE_TYPE_SEQUENCE,
    FEATURE_TYPE_SENTENCE,
)
from rasa.nlu.utils import ordered

if typing.TYPE_CHECKING:
    from rasa.nlu.featurizers.featurizer import Features

class Message:
    def __init__(
        self,
        text: Optional[Text] = None,
        data: Optional[Dict[Text, Any]] = None,
        output_properties: Optional[Set] = None,
        time: Optional[Text] = None,
        features: Optional[List["Features"]] = None,
        **kwargs,
    ) -> None:
        import copy

        self.time = time
        self.data = copy.deepcopy(data) if data else {}
        if text is not None:
            self.data.update({TEXT: text})
        self.features = features if features else []

        self.data.update(**kwargs)

        if output_properties:
            self.output_properties = output_properties
        else:
            self.output_properties = set()

    def add_features(self, features: Optional["Features"]) -> None:
        if features is not None:
            self.features.append(features)

    def set(self, prop, info, add_to_output=False) -> None:
        self.data[prop] = info
        if add_to_output:
            self.output_properties.add(prop)

    def get(self, prop, default=None) -> Any:
        return self.data.get(prop, default)

    def as_dict_nlu(self) -> dict:
        """Get dict representation of message as it would appear in training data"""

        d = self.as_dict()
        if d.get(INTENT, None):
            d[INTENT] = self.get_combined_intent_response_key()
        d.pop(RESPONSE_KEY_ATTRIBUTE, None)
        d.pop(RESPONSE, None)
        return d

    def as_dict(self, only_output_properties=False) -> dict:
        if only_output_properties:
            d = {
                key: value
                for key, value in self.data.items()
                if key in self.output_properties
            }
        else:
            d = self.data

        # Filter all keys with None value. These could have come while building the
        # Message object in markdown format
        d = {key: value for key, value in d.items() if value is not None}

        return d

    def __eq__(self, other) -> bool:
        if not isinstance(other, Message):
            return False
        else:
            return ordered(other.data) == ordered(self.data)

    def __hash__(self) -> int:
        return hash(str(ordered(self.data)))

    @classmethod
    def build(
        cls,
        text: Text,
        intent: Optional[Text] = None,
        entities: List[Dict[Text, Any]] = None,
        **kwargs,
    ) -> "Message":
        data = {}
        if intent:
            split_intent, response_key = cls.separate_intent_response_key(intent)
            if split_intent:
                data[INTENT] = split_intent
            if response_key:
                data[RESPONSE_KEY_ATTRIBUTE] = response_key
        if entities:
            data[ENTITIES] = entities
        return cls(text, data, **kwargs)

    def get_combined_intent_response_key(self) -> Text:
        """Get intent as it appears in training data"""

        intent = self.get(INTENT)
        response_key = self.get(RESPONSE_KEY_ATTRIBUTE)
        response_key_suffix = (
            f"{RESPONSE_IDENTIFIER_DELIMITER}{response_key}" if response_key else ""
        )
        return f"{intent}{response_key_suffix}"

    @staticmethod
    def separate_intent_response_key(
        original_intent: Text,
    ) -> Tuple[Text, Optional[Text]]:

        split_title = original_intent.split(RESPONSE_IDENTIFIER_DELIMITER)
        if len(split_title) == 2:
            return split_title[0], split_title[1]
        elif len(split_title) == 1:
            return split_title[0], None

        raise RasaException(
            f"Intent name '{original_intent}' is invalid, "
            f"it cannot contain more than one '{RESPONSE_IDENTIFIER_DELIMITER}'."
        )

    def get_sparse_features(
        self, attribute: Text, featurizers: Optional[List[Text]] = None
    ) -> Tuple[Optional[scipy.sparse.spmatrix], Optional[scipy.sparse.spmatrix]]:
        """Get all sparse features for the given attribute that are coming from the
        given list of featurizers.

        If no featurizers are provided, all available features will be considered.

        Args:
            attribute: message attribute
            featurizers: names of featurizers to consider

        Returns:
            Sparse features.
        """
        if featurizers is None:
            featurizers = []

        sequence_features, sentence_features = self._filter_sparse_features(
            attribute, featurizers
        )

        sequence_features = self._combine_features(sequence_features)
        sentence_features = self._combine_features(sentence_features)

        return sequence_features, sentence_features

    def get_dense_features(
        self, attribute: Text, featurizers: Optional[List[Text]] = None
    ) -> Tuple[Optional[np.ndarray], Optional[np.ndarray]]:
        """Get all dense features for the given attribute that are coming from the given
        list of featurizers.

        If no featurizers are provided, all available features will be considered.

        Args:
            attribute: message attribute
            featurizers: names of featurizers to consider

        Returns:
            Dense features.
        """
        if featurizers is None:
            featurizers = []

        sequence_features, sentence_features = self._filter_dense_features(
            attribute, featurizers
        )

        sequence_features = self._combine_features(sequence_features)
        sentence_features = self._combine_features(sentence_features)

        return sequence_features, sentence_features

    def features_present(
        self, attribute: Text, featurizers: Optional[List[Text]] = None
    ) -> bool:
        """Check if there are any features present for the given attribute and
        featurizers.

        If no featurizers are provided, all available features will be considered.

        Args:
            attribute: message attribute
            featurizers: names of featurizers to consider

        Returns:
            ``True``, if features are present, ``False`` otherwise
        """
        if featurizers is None:
            featurizers = []

        (
            sequence_sparse_features,
            sentence_sparse_features,
        ) = self._filter_sparse_features(attribute, featurizers)
        sequence_dense_features, sentence_dense_features = self._filter_dense_features(
            attribute, featurizers
        )

        return (
            len(sequence_sparse_features) > 0
            or len(sentence_sparse_features) > 0
            or len(sequence_dense_features) > 0
            or len(sentence_dense_features) > 0
        )

    def _filter_dense_features(
        self, attribute: Text, featurizers: List[Text]
    ) -> Tuple[List["Features"], List["Features"]]:
        sentence_features = [
            f
            for f in self.features
            if f.message_attribute == attribute
            and f.is_dense()
            and f.type == FEATURE_TYPE_SENTENCE
            and (f.origin in featurizers or not featurizers)
        ]
        sequence_features = [
            f
            for f in self.features
            if f.message_attribute == attribute
            and f.is_dense()
            and f.type == FEATURE_TYPE_SEQUENCE
            and (f.origin in featurizers or not featurizers)
        ]
        return sequence_features, sentence_features

    def _filter_sparse_features(
        self, attribute: Text, featurizers: List[Text]
    ) -> Tuple[List["Features"], List["Features"]]:
        sentence_features = [
            f
            for f in self.features
            if f.message_attribute == attribute
            and f.is_sparse()
            and f.type == FEATURE_TYPE_SENTENCE
            and (f.origin in featurizers or not featurizers)
        ]
        sequence_features = [
            f
            for f in self.features
            if f.message_attribute == attribute
            and f.is_sparse()
            and f.type == FEATURE_TYPE_SEQUENCE
            and (f.origin in featurizers or not featurizers)
        ]

        return sequence_features, sentence_features

    @staticmethod
    def _combine_features(
        features: List["Features"],
    ) -> Optional[Union[np.ndarray, scipy.sparse.spmatrix]]:
        combined_features = None

        for f in features:
            combined_features = f.combine_with_features(combined_features)

        return combined_features
<commit_msg>text optional<commit_after>from typing import Any, Optional, Tuple, Text, Dict, Set, List, Union

import numpy as np
import scipy.sparse
import typing

from rasa.exceptions import RasaException
from rasa.nlu.constants import (
    ENTITIES,
    INTENT,
    RESPONSE,
    RESPONSE_KEY_ATTRIBUTE,
    TEXT,
    RESPONSE_IDENTIFIER_DELIMITER,
    FEATURE_TYPE_SEQUENCE,
    FEATURE_TYPE_SENTENCE,
)
from rasa.nlu.utils import ordered

if typing.TYPE_CHECKING:
    from rasa.nlu.featurizers.featurizer import Features

class Message:
    def __init__(
        self,
        text: Optional[Text] = None,
        data: Optional[Dict[Text, Any]] = None,
        output_properties: Optional[Set] = None,
        time: Optional[Text] = None,
        features: Optional[List["Features"]] = None,
        **kwargs,
    ) -> None:
        import copy

        self.time = time
        self.data = copy.deepcopy(data) if data else {}
        if text is not None:
            self.data.update({TEXT: text})
        self.features = features if features else []

        self.data.update(**kwargs)

        if output_properties:
            self.output_properties = output_properties
        else:
            self.output_properties = set()

    def add_features(self, features: Optional["Features"]) -> None:
        if features is not None:
            self.features.append(features)

    def set(self, prop, info, add_to_output=False) -> None:
        self.data[prop] = info
        if add_to_output:
            self.output_properties.add(prop)

    def get(self, prop, default=None) -> Any:
        return self.data.get(prop, default)

    def as_dict_nlu(self) -> dict:
        """Get dict representation of message as it would appear in training data"""

        d = self.as_dict()
        if d.get(INTENT, None):
            d[INTENT] = self.get_combined_intent_response_key()
        d.pop(RESPONSE_KEY_ATTRIBUTE, None)
        d.pop(RESPONSE, None)
        return d

    def as_dict(self, only_output_properties=False) -> dict:
        if only_output_properties:
            d = {
                key: value
                for key, value in self.data.items()
                if key in self.output_properties
            }
        else:
            d = self.data

        # Filter all keys with None value. These could have come while building the
        # Message object in markdown format
        d = {key: value for key, value in d.items() if value is not None}

        return d

    def __eq__(self, other) -> bool:
        if not isinstance(other, Message):
            return False
        else:
            return ordered(other.data) == ordered(self.data)

    def __hash__(self) -> int:
        return hash(str(ordered(self.data)))

    @classmethod
    def build(
        cls,
        text: Optional[Text] = None,
        intent: Optional[Text] = None,
        entities: List[Dict[Text, Any]] = None,
        **kwargs,
    ) -> "Message":
        data = {}
        if intent:
            split_intent, response_key = cls.separate_intent_response_key(intent)
            if split_intent:
                data[INTENT] = split_intent
            if response_key:
                data[RESPONSE_KEY_ATTRIBUTE] = response_key
        if entities:
            data[ENTITIES] = entities
        return cls(text, data, **kwargs)

    def get_combined_intent_response_key(self) -> Text:
        """Get intent as it appears in training data"""

        intent = self.get(INTENT)
        response_key = self.get(RESPONSE_KEY_ATTRIBUTE)
        response_key_suffix = (
            f"{RESPONSE_IDENTIFIER_DELIMITER}{response_key}" if response_key else ""
        )
        return f"{intent}{response_key_suffix}"

    @staticmethod
    def separate_intent_response_key(
        original_intent: Text,
    ) -> Tuple[Text, Optional[Text]]:

        split_title = original_intent.split(RESPONSE_IDENTIFIER_DELIMITER)
        if len(split_title) == 2:
            return split_title[0], split_title[1]
        elif len(split_title) == 1:
            return split_title[0], None

        raise RasaException(
            f"Intent name '{original_intent}' is invalid, "
            f"it cannot contain more than one '{RESPONSE_IDENTIFIER_DELIMITER}'."
        )

    def get_sparse_features(
        self, attribute: Text, featurizers: Optional[List[Text]] = None
    ) -> Tuple[Optional[scipy.sparse.spmatrix], Optional[scipy.sparse.spmatrix]]:
        """Get all sparse features for the given attribute that are coming from the
        given list of featurizers.

        If no featurizers are provided, all available features will be considered.

        Args:
            attribute: message attribute
            featurizers: names of featurizers to consider

        Returns:
            Sparse features.
        """
        if featurizers is None:
            featurizers = []

        sequence_features, sentence_features = self._filter_sparse_features(
            attribute, featurizers
        )

        sequence_features = self._combine_features(sequence_features)
        sentence_features = self._combine_features(sentence_features)

        return sequence_features, sentence_features

    def get_dense_features(
        self, attribute: Text, featurizers: Optional[List[Text]] = None
    ) -> Tuple[Optional[np.ndarray], Optional[np.ndarray]]:
        """Get all dense features for the given attribute that are coming from the given
        list of featurizers.

        If no featurizers are provided, all available features will be considered.

        Args:
            attribute: message attribute
            featurizers: names of featurizers to consider

        Returns:
            Dense features.
        """
        if featurizers is None:
            featurizers = []

        sequence_features, sentence_features = self._filter_dense_features(
            attribute, featurizers
        )

        sequence_features = self._combine_features(sequence_features)
        sentence_features = self._combine_features(sentence_features)

        return sequence_features, sentence_features

    def features_present(
        self, attribute: Text, featurizers: Optional[List[Text]] = None
    ) -> bool:
        """Check if there are any features present for the given attribute and
        featurizers.

        If no featurizers are provided, all available features will be considered.

        Args:
            attribute: message attribute
            featurizers: names of featurizers to consider

        Returns:
            ``True``, if features are present, ``False`` otherwise
        """
        if featurizers is None:
            featurizers = []

        (
            sequence_sparse_features,
            sentence_sparse_features,
        ) = self._filter_sparse_features(attribute, featurizers)
        sequence_dense_features, sentence_dense_features = self._filter_dense_features(
            attribute, featurizers
        )

        return (
            len(sequence_sparse_features) > 0
            or len(sentence_sparse_features) > 0
            or len(sequence_dense_features) > 0
            or len(sentence_dense_features) > 0
        )

    def _filter_dense_features(
        self, attribute: Text, featurizers: List[Text]
    ) -> Tuple[List["Features"], List["Features"]]:
        sentence_features = [
            f
            for f in self.features
            if f.message_attribute == attribute
            and f.is_dense()
            and f.type == FEATURE_TYPE_SENTENCE
            and (f.origin in featurizers or not featurizers)
        ]
        sequence_features = [
            f
            for f in self.features
            if f.message_attribute == attribute
            and f.is_dense()
            and f.type == FEATURE_TYPE_SEQUENCE
            and (f.origin in featurizers or not featurizers)
        ]
        return sequence_features, sentence_features

    def _filter_sparse_features(
        self, attribute: Text, featurizers: List[Text]
    ) -> Tuple[List["Features"], List["Features"]]:
        sentence_features = [
            f
            for f in self.features
            if f.message_attribute == attribute
            and f.is_sparse()
            and f.type == FEATURE_TYPE_SENTENCE
            and (f.origin in featurizers or not featurizers)
        ]
        sequence_features = [
            f
            for f in self.features
            if f.message_attribute == attribute
            and f.is_sparse()
            and f.type == FEATURE_TYPE_SEQUENCE
            and (f.origin in featurizers or not featurizers)
        ]

        return sequence_features, sentence_features

    @staticmethod
    def _combine_features(
        features: List["Features"],
    ) -> Optional[Union[np.ndarray, scipy.sparse.spmatrix]]:
        combined_features = None

        for f in features:
            combined_features = f.combine_with_features(combined_features)

        return combined_features
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import print_function
import types
from .use_gpu_numpy import use_gpu_numpy
import six

if use_gpu_numpy():
    print("Using GPU-supporting numpy wrapper")
    import gpu_numpy as np
else:
    import numpy as np

import warnings
from autograd.core import primitive

def wrap_namespace(old, new):
    unchanged_types =  set([float, int, type(None), type])
    function_types = set([np.ufunc, types.FunctionType, types.BuiltinFunctionType])
    for name, obj in six.iteritems(old):
        if type(obj) in function_types:
            new[name] = primitive(obj)
        elif type(obj) in unchanged_types:
            new[name] = obj

wrap_namespace(np.__dict__, globals())

# ----- Special treatment of list-input functions -----

@primitive
def concatenate_args(axis, *args):
    return np.concatenate(args, axis).view(ndarray)
concatenate = lambda arr_list, axis=0 : concatenate_args(axis, *arr_list)

def array(A, *args, **kwargs):
    if isinstance(A, np.ndarray):
        return np.array(A, *args, **kwargs)
    else:
        raw_array = np.array(A, *args, **kwargs)
        return wrap_if_nodes_inside(raw_array)

def wrap_if_nodes_inside(raw_array, slow_op_name=None):
    if raw_array.dtype is np.dtype('O'):
        if slow_op_name:
            warnings.warn("{0} is slow for array inputs. "
                          "np.concatenate() is faster.".format(slow_op_name))
        return array_from_args(*raw_array.ravel()).reshape(raw_array.shape)
    else:
        return raw_array

@primitive
def array_from_args(*args):
    return np.array(args)

def array_from_args_gradmaker(argnum, ans, args, kwargs):
    return lambda g : g[argnum]
array_from_args.gradmaker = array_from_args_gradmaker

# ----- Enable functions called using [] ----

class r_class():
    def __getitem__(self, args):
        raw_array = np.r_[args]
        return wrap_if_nodes_inside(raw_array, slow_op_name = "r_")
r_ = r_class()

class c_class():
    def __getitem__(self, args):
        raw_array = np.c_[args]
        return wrap_if_nodes_inside(raw_array, slow_op_name = "c_")
c_ = c_class()
<commit_msg>fix 0dim array problem with output of np.select, fixes #36<commit_after>from __future__ import absolute_import
from __future__ import print_function
import types
from .use_gpu_numpy import use_gpu_numpy
import six

if use_gpu_numpy():
    print("Using GPU-supporting numpy wrapper")
    import gpu_numpy as np
else:
    import numpy as np

import warnings
from autograd.core import primitive

def wrap_namespace(old, new):
    unchanged_types =  set([float, int, type(None), type])
    function_types = set([np.ufunc, types.FunctionType, types.BuiltinFunctionType])
    for name, obj in six.iteritems(old):
        if type(obj) in function_types:
            new[name] = primitive(obj)
        elif type(obj) in unchanged_types:
            new[name] = obj

wrap_namespace(np.__dict__, globals())

# ----- Special treatment of list-input functions -----

@primitive
def concatenate_args(axis, *args):
    return np.concatenate(args, axis).view(ndarray)
concatenate = lambda arr_list, axis=0 : concatenate_args(axis, *arr_list)

def array(A, *args, **kwargs):
    if isinstance(A, np.ndarray):
        return np.array(A, *args, **kwargs)
    else:
        raw_array = np.array(A, *args, **kwargs)
        return wrap_if_nodes_inside(raw_array)

def wrap_if_nodes_inside(raw_array, slow_op_name=None):
    if raw_array.dtype is np.dtype('O'):
        if slow_op_name:
            warnings.warn("{0} is slow for array inputs. "
                          "np.concatenate() is faster.".format(slow_op_name))
        return array_from_args(*raw_array.ravel()).reshape(raw_array.shape)
    else:
        return raw_array

@primitive
def array_from_args(*args):
    return np.array(args)

def array_from_args_gradmaker(argnum, ans, args, kwargs):
    return lambda g : g[argnum]
array_from_args.gradmaker = array_from_args_gradmaker

@primitive
def select(condlist, choicelist, **kwargs):
    return np.select(condlist, choicelist, **kwargs)[()]

# ----- Enable functions called using [] ----

class r_class():
    def __getitem__(self, args):
        raw_array = np.r_[args]
        return wrap_if_nodes_inside(raw_array, slow_op_name = "r_")
r_ = r_class()

class c_class():
    def __getitem__(self, args):
        raw_array = np.c_[args]
        return wrap_if_nodes_inside(raw_array, slow_op_name = "c_")
c_ = c_class()
<|endoftext|>


<commit_before><commit_msg>Fix issue if url is a complex one.<commit_after><|endoftext|>


<commit_before>import zlib

import base64
import json
import logging
import os
from tqdm import tqdm
from typing import Optional, Any, Dict, List, Text

from rasa_core import utils
from rasa_core.domain import Domain
from rasa_core.events import ActionExecuted
from rasa_core.featurizers import (
    TrackerFeaturizer, MaxHistoryTrackerFeaturizer)
from rasa_core.policies.policy import Policy
from rasa_core.trackers import DialogueStateTracker

logger = logging.getLogger(__name__)

class MemoizationPolicy(Policy):
    """The policy that remembers exact examples of
        `max_history` turns from training stories.

        Since `slots` that are set some time in the past are
        preserved in all future feature vectors until they are set
        to None, this policy implicitly remembers and most importantly
        recalls examples in the context of the current dialogue
        longer than `max_history`.

        This policy is not supposed to be the only policy in an ensemble,
        it is optimized for precision and not recall.
        It should get a 100% precision because it emits probabilities of 1.1
        along it's predictions, which makes every mistake fatal as
        no other policy can overrule it.

        If it is needed to recall turns from training dialogues where
        some slots might not be set during prediction time, and there are
        training stories for this, use AugmentedMemoizationPolicy.
    """

    ENABLE_FEATURE_STRING_COMPRESSION = True

    SUPPORTS_ONLINE_TRAINING = True

    USE_NLU_CONFIDENCE_AS_SCORE = False

    @staticmethod
    def _standard_featurizer(max_history=None):
        # Memoization policy always uses MaxHistoryTrackerFeaturizer
        # without state_featurizer
        return MaxHistoryTrackerFeaturizer(state_featurizer=None,
                                           max_history=max_history,
                                           use_intent_probabilities=False)

    def __init__(self,
                 featurizer: Optional[TrackerFeaturizer] = None,
                 priority: int = 2,
                 max_history: Optional[int] = None,
                 lookup: Optional[Dict] = None
                 ) -> None:

        if not featurizer:
            featurizer = self._standard_featurizer(max_history)

        super(MemoizationPolicy, self).__init__(featurizer, priority)

        self.max_history = self.featurizer.max_history
        self.lookup = lookup if lookup is not None else {}
        self.is_enabled = True

    def toggle(self, activate: bool) -> None:
        self.is_enabled = activate

    def _add_states_to_lookup(self, trackers_as_states, trackers_as_actions,
                              domain, online=False):
        """Add states to lookup dict"""
        if not trackers_as_states:
            return

        assert len(trackers_as_states[0]) == self.max_history, \
            ("Trying to mem featurized data with {} historic turns. Expected: "
             "{}".format(len(trackers_as_states[0]), self.max_history))

        assert len(trackers_as_actions[0]) == 1, \
            ("The second dimension of trackers_as_action should be 1, "
             "instead of {}".format(len(trackers_as_actions[0])))

        ambiguous_feature_keys = set()

        pbar = tqdm(zip(trackers_as_states, trackers_as_actions),
                    desc="Processed actions", disable=online)
        for states, actions in pbar:
            action = actions[0]

            feature_key = self._create_feature_key(states)
            feature_item = domain.index_for_action(action)

            if feature_key not in ambiguous_feature_keys:
                if feature_key in self.lookup.keys():
                    if self.lookup[feature_key] != feature_item:
                        if online:
                            logger.info("Original stories are "
                                        "different for {} -- {}\n"
                                        "Memorized the new ones for "
                                        "now. Delete contradicting "
                                        "examples after exporting "
                                        "the new stories."
                                        "".format(states, action))
                            self.lookup[feature_key] = feature_item
                        else:
                            # delete contradicting example created by
                            # partial history augmentation from memory
                            ambiguous_feature_keys.add(feature_key)
                            del self.lookup[feature_key]
                else:
                    self.lookup[feature_key] = feature_item
            pbar.set_postfix({"# examples": "{:d}".format(
                len(self.lookup))})

    def _create_feature_key(self, states):
        feature_str = json.dumps(states, sort_keys=True).replace("\"", "")
        if self.ENABLE_FEATURE_STRING_COMPRESSION:
            compressed = zlib.compress(bytes(feature_str, "utf-8"))
            return base64.b64encode(compressed).decode("utf-8")
        else:
            return feature_str

    def train(self,
              training_trackers: List[DialogueStateTracker],
              domain: Domain,
              **kwargs: Any
              ) -> None:
        """Trains the policy on given training trackers."""
        self.lookup = {}
        # only considers original trackers (no augmented ones)
        training_trackers = \
            [t for t in training_trackers if not
             hasattr(t, 'is_augmented') or not t.is_augmented
             ]
        (trackers_as_states,
         trackers_as_actions) = self.featurizer.training_states_and_actions(
            training_trackers, domain)
        self._add_states_to_lookup(trackers_as_states, trackers_as_actions,
                                   domain)
        logger.debug("Memorized {} unique examples."
                     "".format(len(self.lookup)))

    def continue_training(self,
                          training_trackers: List[DialogueStateTracker],
                          domain: Domain,
                          **kwargs: Any) -> None:

        # add only the last tracker, because it is the only new one
        (trackers_as_states,
         trackers_as_actions) = self.featurizer.training_states_and_actions(
            training_trackers[-1:], domain)
        self._add_states_to_lookup(trackers_as_states, trackers_as_actions,
                                   domain, online=True)

    def _recall_states(self, states: List[Dict[Text, float]]) -> Optional[int]:

        return self.lookup.get(self._create_feature_key(states))

    def recall(self,
               states: List[Dict[Text, float]],
               tracker: DialogueStateTracker,
               domain: Domain
               ) -> Optional[int]:

        return self._recall_states(states)

    def predict_action_probabilities(self,
                                     tracker: DialogueStateTracker,
                                     domain: Domain) -> List[float]:
        """Predicts the next action the bot should take
            after seeing the tracker.

            Returns the list of probabilities for the next actions.
            If memorized action was found returns 1.1 for its index,
            else returns 0.0 for all actions."""
        result = [0.0] * domain.num_actions

        if not self.is_enabled:
            return result

        tracker_as_states = self.featurizer.prediction_states(
            [tracker], domain)
        states = tracker_as_states[0]
        logger.debug("Current tracker state {}".format(states))
        recalled = self.recall(states, tracker, domain)
        if recalled is not None:
            logger.debug("There is a memorised next action '{}'"
                         "".format(recalled))

            if self.USE_NLU_CONFIDENCE_AS_SCORE:
                # the memoization will use the confidence of NLU on the latest
                # user message to set the confidence of the action
                score = tracker.latest_message.intent.get("confidence", 1.0)
            else:
                score = 1.0

            result[recalled] = score
        else:
            logger.debug("There is no memorised next action")

        return result

    def persist(self, path: Text) -> None:

        self.featurizer.persist(path)

        memorized_file = os.path.join(path, 'memorized_turns.json')
        data = {
            "priority": self.priority,
            "max_history": self.max_history,
            "lookup": self.lookup
        }
        utils.create_dir_for_file(memorized_file)
        utils.dump_obj_as_json_to_file(memorized_file, data)

    @classmethod
    def load(cls, path: Text) -> 'MemoizationPolicy':

        featurizer = TrackerFeaturizer.load(path)
        memorized_file = os.path.join(path, 'memorized_turns.json')
        if os.path.isfile(memorized_file):
            data = json.loads(utils.read_file(memorized_file))
            return cls(featurizer=featurizer, priority=data["priority"],
                       lookup=data["lookup"])
        else:
            logger.info("Couldn't load memoization for policy. "
                        "File '{}' doesn't exist. Falling back to empty "
                        "turn memory.".format(memorized_file))
            return cls()

class AugmentedMemoizationPolicy(MemoizationPolicy):
    """The policy that remembers examples from training stories
        for `max_history` turns.

        If it is needed to recall turns from training dialogues
        where some slots might not be set during prediction time,
        add relevant stories without such slots to training data.
        E.g. reminder stories.

        Since `slots` that are set some time in the past are
        preserved in all future feature vectors until they are set
        to None, this policy has a capability to recall the turns
        up to `max_history` from training stories during prediction
        even if additional slots were filled in the past
        for current dialogue.
    """

    @staticmethod
    def _back_to_the_future_again(tracker):
        """Send Marty to the past to get
            the new featurization for the future"""

        idx_of_first_action = None
        idx_of_second_action = None

        # we need to find second executed action
        for e_i, event in enumerate(tracker.applied_events()):
            # find second ActionExecuted
            if isinstance(event, ActionExecuted):
                if idx_of_first_action is None:
                    idx_of_first_action = e_i
                else:
                    idx_of_second_action = e_i
                    break

        if idx_of_second_action is None:
            return None
        # make second ActionExecuted the first one
        events = tracker.applied_events()[idx_of_second_action:]
        if not events:
            return None

        mcfly_tracker = tracker.init_copy()
        for e in events:
            mcfly_tracker.update(e)

        return mcfly_tracker

    def _recall_using_delorean(self, old_states, tracker, domain):
        """Recursively go to the past to correctly forget slots,
            and then back to the future to recall."""

        logger.debug("Launch DeLorean...")
        mcfly_tracker = self._back_to_the_future_again(tracker)
        while mcfly_tracker is not None:
            tracker_as_states = self.featurizer.prediction_states(
                [mcfly_tracker], domain)
            states = tracker_as_states[0]

            if old_states != states:
                # check if we like new futures
                memorised = self._recall_states(states)
                if memorised is not None:
                    logger.debug("Current tracker state {}".format(states))
                    return memorised
                old_states = states

            # go back again
            mcfly_tracker = self._back_to_the_future_again(mcfly_tracker)

        # No match found
        logger.debug("Current tracker state {}".format(old_states))
        return None

    def recall(self,
               states: List[Dict[Text, float]],
               tracker: DialogueStateTracker,
               domain: Domain
               ) -> Optional[int]:

        recalled = self._recall_states(states)
        if recalled is None:
            # let's try a different method to recall that tracker
            return self._recall_using_delorean(states, tracker, domain)
        else:
            return recalled
<commit_msg>Update rasa_core/policies/memoization.py<commit_after>import zlib

import base64
import json
import logging
import os
from tqdm import tqdm
from typing import Optional, Any, Dict, List, Text

from rasa_core import utils
from rasa_core.domain import Domain
from rasa_core.events import ActionExecuted
from rasa_core.featurizers import (
    TrackerFeaturizer, MaxHistoryTrackerFeaturizer)
from rasa_core.policies.policy import Policy
from rasa_core.trackers import DialogueStateTracker

logger = logging.getLogger(__name__)

class MemoizationPolicy(Policy):
    """The policy that remembers exact examples of
        `max_history` turns from training stories.

        Since `slots` that are set some time in the past are
        preserved in all future feature vectors until they are set
        to None, this policy implicitly remembers and most importantly
        recalls examples in the context of the current dialogue
        longer than `max_history`.

        This policy is not supposed to be the only policy in an ensemble,
        it is optimized for precision and not recall.
        It should get a 100% precision because it emits probabilities of 1.1
        along it's predictions, which makes every mistake fatal as
        no other policy can overrule it.

        If it is needed to recall turns from training dialogues where
        some slots might not be set during prediction time, and there are
        training stories for this, use AugmentedMemoizationPolicy.
    """

    ENABLE_FEATURE_STRING_COMPRESSION = True

    SUPPORTS_ONLINE_TRAINING = True

    USE_NLU_CONFIDENCE_AS_SCORE = False

    @staticmethod
    def _standard_featurizer(max_history=None):
        # Memoization policy always uses MaxHistoryTrackerFeaturizer
        # without state_featurizer
        return MaxHistoryTrackerFeaturizer(state_featurizer=None,
                                           max_history=max_history,
                                           use_intent_probabilities=False)

    def __init__(self,
                 featurizer: Optional[TrackerFeaturizer] = None,
                 priority: int = 2,
                 max_history: Optional[int] = None,
                 lookup: Optional[Dict] = None
                 ) -> None:

        if not featurizer:
            featurizer = self._standard_featurizer(max_history)

        super(MemoizationPolicy, self).__init__(featurizer, priority)

        self.max_history = self.featurizer.max_history
        self.lookup = lookup if lookup is not None else {}
        self.is_enabled = True

    def toggle(self, activate: bool) -> None:
        self.is_enabled = activate

    def _add_states_to_lookup(self, trackers_as_states, trackers_as_actions,
                              domain, online=False):
        """Add states to lookup dict"""
        if not trackers_as_states:
            return

        assert len(trackers_as_states[0]) == self.max_history, \
            ("Trying to mem featurized data with {} historic turns. Expected: "
             "{}".format(len(trackers_as_states[0]), self.max_history))

        assert len(trackers_as_actions[0]) == 1, \
            ("The second dimension of trackers_as_action should be 1, "
             "instead of {}".format(len(trackers_as_actions[0])))

        ambiguous_feature_keys = set()

        pbar = tqdm(zip(trackers_as_states, trackers_as_actions),
                    desc="Processed actions", disable=online)
        for states, actions in pbar:
            action = actions[0]

            feature_key = self._create_feature_key(states)
            feature_item = domain.index_for_action(action)

            if feature_key not in ambiguous_feature_keys:
                if feature_key in self.lookup.keys():
                    if self.lookup[feature_key] != feature_item:
                        if online:
                            logger.info("Original stories are "
                                        "different for {} -- {}\n"
                                        "Memorized the new ones for "
                                        "now. Delete contradicting "
                                        "examples after exporting "
                                        "the new stories."
                                        "".format(states, action))
                            self.lookup[feature_key] = feature_item
                        else:
                            # delete contradicting example created by
                            # partial history augmentation from memory
                            ambiguous_feature_keys.add(feature_key)
                            del self.lookup[feature_key]
                else:
                    self.lookup[feature_key] = feature_item
            pbar.set_postfix({"# examples": "{:d}".format(
                len(self.lookup))})

    def _create_feature_key(self, states):
        feature_str = json.dumps(states, sort_keys=True).replace("\"", "")
        if self.ENABLE_FEATURE_STRING_COMPRESSION:
            compressed = zlib.compress(bytes(feature_str, "utf-8"))
            return base64.b64encode(compressed).decode("utf-8")
        else:
            return feature_str

    def train(self,
              training_trackers: List[DialogueStateTracker],
              domain: Domain,
              **kwargs: Any
              ) -> None:
        """Trains the policy on given training trackers."""
        self.lookup = {}
        # only considers original trackers (no augmented ones)
        training_trackers = [
            [t for t in training_trackers if not
             hasattr(t, 'is_augmented') or not t.is_augmented
             ]
        (trackers_as_states,
         trackers_as_actions) = self.featurizer.training_states_and_actions(
            training_trackers, domain)
        self._add_states_to_lookup(trackers_as_states, trackers_as_actions,
                                   domain)
        logger.debug("Memorized {} unique examples."
                     "".format(len(self.lookup)))

    def continue_training(self,
                          training_trackers: List[DialogueStateTracker],
                          domain: Domain,
                          **kwargs: Any) -> None:

        # add only the last tracker, because it is the only new one
        (trackers_as_states,
         trackers_as_actions) = self.featurizer.training_states_and_actions(
            training_trackers[-1:], domain)
        self._add_states_to_lookup(trackers_as_states, trackers_as_actions,
                                   domain, online=True)

    def _recall_states(self, states: List[Dict[Text, float]]) -> Optional[int]:

        return self.lookup.get(self._create_feature_key(states))

    def recall(self,
               states: List[Dict[Text, float]],
               tracker: DialogueStateTracker,
               domain: Domain
               ) -> Optional[int]:

        return self._recall_states(states)

    def predict_action_probabilities(self,
                                     tracker: DialogueStateTracker,
                                     domain: Domain) -> List[float]:
        """Predicts the next action the bot should take
            after seeing the tracker.

            Returns the list of probabilities for the next actions.
            If memorized action was found returns 1.1 for its index,
            else returns 0.0 for all actions."""
        result = [0.0] * domain.num_actions

        if not self.is_enabled:
            return result

        tracker_as_states = self.featurizer.prediction_states(
            [tracker], domain)
        states = tracker_as_states[0]
        logger.debug("Current tracker state {}".format(states))
        recalled = self.recall(states, tracker, domain)
        if recalled is not None:
            logger.debug("There is a memorised next action '{}'"
                         "".format(recalled))

            if self.USE_NLU_CONFIDENCE_AS_SCORE:
                # the memoization will use the confidence of NLU on the latest
                # user message to set the confidence of the action
                score = tracker.latest_message.intent.get("confidence", 1.0)
            else:
                score = 1.0

            result[recalled] = score
        else:
            logger.debug("There is no memorised next action")

        return result

    def persist(self, path: Text) -> None:

        self.featurizer.persist(path)

        memorized_file = os.path.join(path, 'memorized_turns.json')
        data = {
            "priority": self.priority,
            "max_history": self.max_history,
            "lookup": self.lookup
        }
        utils.create_dir_for_file(memorized_file)
        utils.dump_obj_as_json_to_file(memorized_file, data)

    @classmethod
    def load(cls, path: Text) -> 'MemoizationPolicy':

        featurizer = TrackerFeaturizer.load(path)
        memorized_file = os.path.join(path, 'memorized_turns.json')
        if os.path.isfile(memorized_file):
            data = json.loads(utils.read_file(memorized_file))
            return cls(featurizer=featurizer, priority=data["priority"],
                       lookup=data["lookup"])
        else:
            logger.info("Couldn't load memoization for policy. "
                        "File '{}' doesn't exist. Falling back to empty "
                        "turn memory.".format(memorized_file))
            return cls()

class AugmentedMemoizationPolicy(MemoizationPolicy):
    """The policy that remembers examples from training stories
        for `max_history` turns.

        If it is needed to recall turns from training dialogues
        where some slots might not be set during prediction time,
        add relevant stories without such slots to training data.
        E.g. reminder stories.

        Since `slots` that are set some time in the past are
        preserved in all future feature vectors until they are set
        to None, this policy has a capability to recall the turns
        up to `max_history` from training stories during prediction
        even if additional slots were filled in the past
        for current dialogue.
    """

    @staticmethod
    def _back_to_the_future_again(tracker):
        """Send Marty to the past to get
            the new featurization for the future"""

        idx_of_first_action = None
        idx_of_second_action = None

        # we need to find second executed action
        for e_i, event in enumerate(tracker.applied_events()):
            # find second ActionExecuted
            if isinstance(event, ActionExecuted):
                if idx_of_first_action is None:
                    idx_of_first_action = e_i
                else:
                    idx_of_second_action = e_i
                    break

        if idx_of_second_action is None:
            return None
        # make second ActionExecuted the first one
        events = tracker.applied_events()[idx_of_second_action:]
        if not events:
            return None

        mcfly_tracker = tracker.init_copy()
        for e in events:
            mcfly_tracker.update(e)

        return mcfly_tracker

    def _recall_using_delorean(self, old_states, tracker, domain):
        """Recursively go to the past to correctly forget slots,
            and then back to the future to recall."""

        logger.debug("Launch DeLorean...")
        mcfly_tracker = self._back_to_the_future_again(tracker)
        while mcfly_tracker is not None:
            tracker_as_states = self.featurizer.prediction_states(
                [mcfly_tracker], domain)
            states = tracker_as_states[0]

            if old_states != states:
                # check if we like new futures
                memorised = self._recall_states(states)
                if memorised is not None:
                    logger.debug("Current tracker state {}".format(states))
                    return memorised
                old_states = states

            # go back again
            mcfly_tracker = self._back_to_the_future_again(mcfly_tracker)

        # No match found
        logger.debug("Current tracker state {}".format(old_states))
        return None

    def recall(self,
               states: List[Dict[Text, float]],
               tracker: DialogueStateTracker,
               domain: Domain
               ) -> Optional[int]:

        recalled = self._recall_states(states)
        if recalled is None:
            # let's try a different method to recall that tracker
            return self._recall_using_delorean(states, tracker, domain)
        else:
            return recalled
<|endoftext|>


<commit_before>"""
Some test utilities to create the spark context.
"""
import sys
if sys.version_info[:2] <= (2, 6):
    try:
        import unittest2 as unittest
    except ImportError:
        sys.stderr.write('Please install unittest2 to test with Python 2.6 or earlier')
        sys.exit(1)
else:
    import unittest
import numpy as np
from scipy.sparse import csr_matrix

from pyspark import SparkContext
from pyspark.sql import SparkSession
from pyspark.ml.linalg import Vectors

# Used as decorator to wrap around a class deriving from unittest.TestCase. Wraps current
# unittest methods setUpClass() and tearDownClass(), invoked by the nosetest command before
# and after unit tests are run. This enables us to create one PySpark SparkSession per
# test fixture. The session can be referred to with self.spark or ClassName.spark.
#
# The SparkSession is set up before invoking the class' own set up and torn down after the
# class' tear down, so you may safer to it in those methods.
def fixtureReuseSparkSession(cls):
    setup = getattr(cls, 'setUpClass', None)
    teardown = getattr(cls, 'tearDownClass', None)
    def setUpClass(cls):
        cls.spark = SparkSession.builder.master("local").appName("Unit Tests").getOrCreate()
        if setup: setup()
    def tearDownClass(cls):
        if teardown: teardown()
        if cls.spark:
            cls.spark.stop()
            # Next session will attempt to reuse the previous stopped
            # SparkContext if it's not cleared.
            SparkSession._instantiatedContext = None
        cls.spark = None

    cls.setUpClass = classmethod(setUpClass)
    cls.tearDownClass = classmethod(tearDownClass)
    return cls

class MLlibTestCase(unittest.TestCase):
    def setUp(self):
        super(MLlibTestCase, self).setUp()
        self.sc = self.spark.sparkContext
        self.sql = self.spark
        self.X = np.array([[1,2,3],
                           [-1,2,3], [1,-2,3], [1,2,-3],
                           [-1,-2,3], [1,-2,-3], [-1,2,-3],
                           [-1,-2,-3]])
        self.y = np.array([1, 0, 1, 1, 0, 1, 0, 0])
        data = [(float(self.y[i]), Vectors.dense(self.X[i])) for i in range(len(self.y))]
        self.df = self.sql.createDataFrame(data, ["label", "features"])

    @staticmethod
    def list2csr(x):
        """
        Convert list to a scipy.sparse.csr_matrix
        :param data: list
        :return:  csr_matrix with 1 row
        """
        return csr_matrix((np.array(x), np.array(range(0, len(x))), np.array([0, len(x)])))
<commit_msg>First round PR comments<commit_after>"""
Some test utilities to create the spark context.
"""
import sys
if sys.version_info[:2] <= (2, 6):
    try:
        import unittest2 as unittest
    except ImportError:
        sys.stderr.write('Please install unittest2 to test with Python 2.6 or earlier')
        sys.exit(1)
else:
    import unittest
import numpy as np
from scipy.sparse import csr_matrix

from pyspark import SparkContext
from pyspark.sql import SparkSession
from pyspark.ml.linalg import Vectors

# Used as decorator to wrap around a class deriving from unittest.TestCase. Wraps current
# unittest methods setUpClass() and tearDownClass(), invoked by the nosetest command before
# and after unit tests are run. This enables us to create one PySpark SparkSession per
# test fixture. The session can be referred to with self.spark or ClassName.spark.
#
# The SparkSession is set up before invoking the class' own set up and torn down after the
# class' tear down, so you may safer to it in those methods.
def fixtureReuseSparkSession(cls):
    setup = getattr(cls, 'setUpClass', None)
    teardown = getattr(cls, 'tearDownClass', None)
    def setUpClass(cls):
        cls.spark = SparkSession.builder.master("local").appName("Unit Tests").getOrCreate()
        if setup:
            setup()
    def tearDownClass(cls):
        if teardown:
            teardown()
        if cls.spark:
            cls.spark.stop()
            # Next session will attempt to reuse the previous stopped
            # SparkContext if it's not cleared.
            SparkSession._instantiatedContext = None
        cls.spark = None

    cls.setUpClass = classmethod(setUpClass)
    cls.tearDownClass = classmethod(tearDownClass)
    return cls

class MLlibTestCase(unittest.TestCase):
    def setUp(self):
        super(MLlibTestCase, self).setUp()
        self.sc = self.spark.sparkContext
        self.sql = self.spark
        self.X = np.array([[1,2,3],
                           [-1,2,3], [1,-2,3], [1,2,-3],
                           [-1,-2,3], [1,-2,-3], [-1,2,-3],
                           [-1,-2,-3]])
        self.y = np.array([1, 0, 1, 1, 0, 1, 0, 0])
        data = [(float(self.y[i]), Vectors.dense(self.X[i])) for i in range(len(self.y))]
        self.df = self.sql.createDataFrame(data, ["label", "features"])

    @staticmethod
    def list2csr(x):
        """
        Convert list to a scipy.sparse.csr_matrix
        :param data: list
        :return:  csr_matrix with 1 row
        """
        return csr_matrix((np.array(x), np.array(range(0, len(x))), np.array([0, len(x)])))
<|endoftext|>


<commit_before># Copyright (C) 2016 SignalFx, Inc. All rights reserved.

import logging
import pprint
import requests

from . import constants, version

# TODO: this file needs rework. We want the REST API client to expose the same
# object abstractions as the API itself, with functions to manipulate those,
# instead of a disorderly bag of utility functions.

_logger = logging.getLogger(__name__)

class SignalFxRestClient(object):
    """SignalFx REST API client."""

    _METRIC_ENDPOINT_SUFFIX = 'v2/metric'
    _DIMENSION_ENDPOINT_SUFFIX = 'v2/dimension'
    _DETECTOR_ENDPOINT_SUFFIX = 'v2/detector'
    _MTS_ENDPOINT_SUFFIX = 'v2/metrictimeseries'
    _TAG_ENDPOINT_SUFFIX = 'v2/tag'
    _ORGANIZATION_ENDPOINT_SUFFIX = 'v2/organization'

    def __init__(self, token, endpoint=constants.DEFAULT_API_ENDPOINT,
                 timeout=constants.DEFAULT_TIMEOUT):
        self._token = token
        self._endpoint = endpoint
        self._timeout = timeout

        self._session = requests.Session()
        self._session.headers.update({
            'Content-Type': 'application/json',
            'X-SF-Token': self._token,
            'User-Agent': '{0}/{1}'.format(version.name, version.version),
        })

    def _u(self, *args):
        return '{0}/{1}'.format(self._endpoint, '/'.join(args))

    def _get(self, url, params=None, session=None, timeout=None):
        session = session or self._session
        timeout = timeout or self._timeout
        _logger.debug('URL being retrieved: %s (params: %s)',
                      pprint.pformat(url), params)
        response = session.get(url, timeout=timeout, params=params)
        _logger.debug('Getting from SignalFx %s (%d %s)',
                      'succeeded' if response.ok else 'failed',
                      response.status_code, response.text)
        return response

    def _put(self, url, data, session=None, timeout=None):
        session = session or self._session
        timeout = timeout or self._timeout
        _logger.debug('Raw datastream being sent: %s', pprint.pformat(data))
        response = session.put(url, json=data, timeout=timeout)
        _logger.debug('Putting to SignalFx %s (%d %s)',
                      'succeeded' if response.ok else 'failed',
                      response.status_code, response.text)
        return response

    def _post(self, url, data, session=None, timeout=None):
        session = session or self._session
        timeout = timeout or self._timeout
        _logger.debug('Raw datastream being sent: %s', pprint.pformat(data))
        response = session.post(url, json=data, timeout=timeout)
        _logger.debug('Posting to SignalFx %s (%d %s)',
                      'succeeded' if response.ok else 'failed',
                      response.status_code, response.text)
        return response

    def _delete(self, url, session=None, timeout=None):
        session = session or self._session
        timeout = timeout or self._timeout
        _logger.debug('url associated with delete request: %s',
                      pprint.pformat(url))
        response = session.delete(url, timeout=timeout)
        _logger.debug('Deleting from SignalFx %s (%d %s)',
                      'succeeded' if response.ok else 'failed',
                      response.status_code, response.text)
        return response

    def _search_metrics_and_metadata(self, metadata_endpoint, query,
                                     order_by=None, offset=None,
                                     limit=None, timeout=None):
        """
        generic function for elasticsearch queries; can search metrics,
            dimensions, metrictimeseries by changing metadata_endpoint
        Args:
            metadata_endpoint (string): API endpoint suffix (e.g. 'v2/metric')
            query (string): elasticsearch string query
            order_by (optional[string]): property by which to order results
            offset (optional[int]): number of results to skip for pagination
                (default=0)
            limit (optional[int]): how many results to return (default=50)
            timeout (optional[int]): how long to wait for response (in seconds)

        Returns:
            dictionary of query result
        """
        _logger.debug('Performing an elasticsearch for %(qry)s at %(pt)s',
                      {'qry': query, 'pt': metadata_endpoint})
        url_to_get = '{0}?query={1}'.format(self._u(metadata_endpoint), query)
        if order_by is not None:
            url_to_get += '&orderBy=' + order_by
        # for offset and limit, use API defaults (by leaving them out of url)
        if offset is not None:
            url_to_get += '&offset=' + str(offset)
        if limit is not None:
            url_to_get += '&limit=' + str(limit)
        timeout = timeout or self._timeout
        resp = self._get(url_to_get, session=self._session, timeout=timeout)
        resp.raise_for_status()
        return resp.json()

    def _get_object_by_name(self, object_endpoint, object_name, timeout=None):
        """
        generic function to get object (metadata, tag, ) by name from SignalFx.

        Args:
            object_endpoint (string): API endpoint suffix (e.g. 'v2/tag')
            object_name (string): name of the object (e.g. 'jvm.cpu.load')

        Returns:
            dictionary of response
        """
        timeout = timeout or self._timeout
        resp = self._get(self._u(object_endpoint, object_name),
                         session=self._session, timeout=timeout)
        resp.raise_for_status()
        return resp.json()

    # functionality related to metrics
    def search_metrics(self, *args, **kwargs):
        """
        Args:
            query (string): elasticsearch string query
            order_by (optional[string]): property by which to order results
            offset (optional[int]): number of results to skip for pagination
                (default=0)
            limit (optional[int]): how many results to return (default=50)
            timeout (optional[int]): how long to wait for response (in seconds)

        Returns:
            result of query search on metrics
        """
        return self._search_metrics_and_metadata(
            self._METRIC_ENDPOINT_SUFFIX, *args, **kwargs)

    def get_metric_by_name(self, metric_name, **kwargs):
        """
        get a metric by name

        Args:
            metric_name (string): name of metric

        Returns:
            dictionary of response
        """
        return self._get_object_by_name(self._METRIC_ENDPOINT_SUFFIX,
                                        metric_name,
                                        **kwargs)

    def update_metric_by_name(self, metric_name, metric_type, description=None,
                              custom_properties=None, tags=None, **kwargs):
        """
        Create or update a metric object

        Args:
            metric_name (string): name of metric
            type (string): metric type, must be one of 'gauge', 'counter',
                            'cumulative_counter'
            description (optional[string]): a description
            custom_properties (optional[dict]): dictionary of custom properties
            tags (optional[list of strings]): list of tags associated with
                metric
        """
        data = {'type': metric_type.upper(),
                'description': description or '',
                'customProperties': custom_properties or {},
                'tags': tags or []}
        resp = self._put(self._u(self._METRIC_ENDPOINT_SUFFIX,
                                 str(metric_name)),
                         data=data, **kwargs)
        resp.raise_for_status()
        return resp.json()

    # functionality related to dimensions
    def search_dimensions(self, *args, **kwargs):
        """
        Args:
            query (string): elasticsearch string query
            order_by (optional[string]): property by which to order results
            offset (optional[int]): number of results to skip for pagination
                (default=0)
            limit (optional[int]): how many results to return (default=50)
            timeout (optional[int]): how long to wait for response (in seconds)

        Returns:
            result of query search on dimensions
        """
        return self._search_metrics_and_metadata(
            self._DIMENSION_ENDPOINT_SUFFIX, *args, **kwargs)

    def get_dimension(self, key, value, **kwargs):
        """
        get a dimension by key and value

        Args:
            key (string): key of the dimension
            value (string): value of the dimension

        Returns:
            dictionary of response
        """
        return self._get_object_by_name(self._DIMENSION_ENDPOINT_SUFFIX,
                                        '{0}/{1}'.format(key, value),
                                        **kwargs)

    def update_dimension(self, key, value, description=None,
                         custom_properties=None, tags=None, **kwargs):
        """
        update a dimension
        Args:
            key (string): key of the dimension
            value (string): value of the dimension
            description (optional[string]): a description
            custom_properties (optional[dict]): dictionary of custom properties
            tags (optional[list of strings]): list of tags associated with
                metric
        """
        data = {'description': description or '',
                'customProperties': custom_properties or {},
                'tags': tags or [],
                'key': key,
                'value': value}
        resp = self._put(self._u(self._DIMENSION_ENDPOINT_SUFFIX, key, value),
                         data=data, **kwargs)
        resp.raise_for_status()
        return resp.json()

    # functionality related to metrictimeseries
    def search_metric_time_series(self, *args, **kwargs):
        """
        Args:
            query (string): elasticsearch string query
            order_by (optional[string]): property by which to order results
            offset (optional[int]): number of results to skip for pagination
                (default=0)
            limit (optional[int]): how many results to return (default=50)
            timeout (optional[int]): how long to wait for response (in seconds)

        Returns:
            result of query search on metric time series

        """
        return self._search_metrics_and_metadata(self._MTS_ENDPOINT_SUFFIX,
                                                 *args, **kwargs)

    def get_metric_time_series(self, mts_id, **kwargs):
        """get a metric time series by id"""
        return self._get_object_by_name(self._MTS_ENDPOINT_SUFFIX,
                                        mts_id,
                                        **kwargs)

    # functionality related to tags
    def search_tags(self, *args, **kwargs):
        """
        Args:
            query (string): elasticsearch string query
            order_by (optional[string]): property by which to order results
            offset (optional[int]): number of results to skip for pagination
                (default=0)
            limit (optional[int]): how many results to return (default=50)
            timeout (optional[int]): how long to wait for response (in seconds)

        Returns:
            result of query search on tags

        """
        return self._search_metrics_and_metadata(self._TAG_ENDPOINT_SUFFIX,
                                                 *args, **kwargs)

    def get_tag(self, tag_name, **kwargs):
        """get a tag by name

        Args:
            tag_name (string): name of tag to get

        Returns:
            dictionary of the response

        """
        return self._get_object_by_name(self._TAG_ENDPOINT_SUFFIX,
                                        tag_name,
                                        **kwargs)

    def update_tag(self, tag_name, description=None,
                   custom_properties=None, **kwargs):
        """update a tag by name

        Args:
            tag_name (string): name of tag to update
            description (optional[string]): a description
            custom_properties (optional[dict]): dictionary of custom properties
        """
        data = {'description': description or '',
                'customProperties': custom_properties or {}}
        resp = self._put(self._u(self._TAG_ENDPOINT_SUFFIX, tag_name),
                         data=data, **kwargs)
        resp.raise_for_status()
        return resp.json()

    def delete_tag(self, tag_name, **kwargs):
        """delete a tag by name

        Args:
            tag_name (string): name of tag to delete
        """
        resp = self._delete(self._u(self._TAG_ENDPOINT_SUFFIX, tag_name),
                            **kwargs)
        resp.raise_for_status()
        # successful delete returns 204, which has no associated json
        return resp

    # functionality related to organizations (more to come)
    def get_organization(self, **kwargs):
        """Get the organization to which the user belongs

        Returns:
            dictionary of the response
        """
        resp = self._get(self._u(self._ORGANIZATION_ENDPOINT_SUFFIX),
                         **kwargs)
        resp.raise_for_status()
        return resp.json()

    # functionality related to detectors
    def get_detectors(self, name=None, batch_size=100, **kwargs):
        """Retrieve all (v2) detectors matching the given name; all (v2)
        detectors otherwise.

        Note that this method will loop through the paging of the results and
        accumulate all detectors that match the query. This may be expensive.
        """
        detectors = []
        offset = 0
        while True:
            resp = self._get(
                self._u(self._DETECTOR_ENDPOINT_SUFFIX),
                params={'offset': offset, 'limit': batch_size, 'name': name},
                **kwargs)
            resp.raise_for_status()
            data = resp.json()
            detectors += data['results']
            if len(detectors) == data['count']:
                break
            offset = len(detectors)
        return detectors

    def create_detector(self, detector):
        """Creates a new detector.

        Args:
            detector (object): the detector model object. Will be serialized as
                JSON.
        Returns:
            dictionary of the response (created detector model).
        """
        resp = self._post(self._u(self._DETECTOR_ENDPOINT_SUFFIX),
                          data=detector)
        resp.raise_for_status()
        return resp.json()

    def update_detector(self, detector_id, detector):
        """Update an existing detector.

        Args:
            detector_id (string): the ID of the detector.
            detector (object): the detector model object. Will be serialized as
                JSON.
        Returns:
            dictionary of the response (updated detector model).
        """
        resp = self._put(self._u(self._DETECTOR_ENDPOINT_SUFFIX, detector_id),
                         data=detector)
        resp.raise_for_status()
        return resp.json()

    def delete_detector(self, detector_id):
        """Remove a detector.

        Args:
            detector_id (string): the ID of the detector.
        """
        resp = self._delete(self._u(self._DETECTOR_ENDPOINT_SUFFIX,
                                    detector_id))
        resp.raise_for_status()
        # successful delete returns 204, which has no response json
        return resp
<commit_msg>Logging tweaks and ignore_not_found on delete operations<commit_after># Copyright (C) 2016 SignalFx, Inc. All rights reserved.

import logging
import pprint
import requests

from . import constants, version

# TODO: this file needs rework. We want the REST API client to expose the same
# object abstractions as the API itself, with functions to manipulate those,
# instead of a disorderly bag of utility functions.

_logger = logging.getLogger(__name__)

class SignalFxRestClient(object):
    """SignalFx REST API client."""

    _METRIC_ENDPOINT_SUFFIX = 'v2/metric'
    _DIMENSION_ENDPOINT_SUFFIX = 'v2/dimension'
    _DETECTOR_ENDPOINT_SUFFIX = 'v2/detector'
    _MTS_ENDPOINT_SUFFIX = 'v2/metrictimeseries'
    _TAG_ENDPOINT_SUFFIX = 'v2/tag'
    _ORGANIZATION_ENDPOINT_SUFFIX = 'v2/organization'

    def __init__(self, token, endpoint=constants.DEFAULT_API_ENDPOINT,
                 timeout=constants.DEFAULT_TIMEOUT):
        self._token = token
        self._endpoint = endpoint
        self._timeout = timeout

        self._session = requests.Session()
        self._session.headers.update({
            'Content-Type': 'application/json',
            'X-SF-Token': self._token,
            'User-Agent': '{0}/{1}'.format(version.name, version.version),
        })

    def _u(self, *args):
        return '{0}/{1}'.format(self._endpoint, '/'.join(args))

    def _get(self, url, params=None, session=None, timeout=None):
        session = session or self._session
        timeout = timeout or self._timeout
        _logger.debug('GET %s (params: %s)', url, params)
        response = session.get(url, timeout=timeout, params=params)
        _logger.debug('Getting from SignalFx %s (%d): %s',
                      'succeeded' if response.ok else 'failed',
                      response.status_code, response.text)
        return response

    def _put(self, url, data, session=None, timeout=None):
        session = session or self._session
        timeout = timeout or self._timeout
        _logger.debug('PUT %s: %s', url, pprint.pformat(data))
        response = session.put(url, json=data, timeout=timeout)
        _logger.debug('Putting to SignalFx %s (%d): %s',
                      'succeeded' if response.ok else 'failed',
                      response.status_code, response.text)
        return response

    def _post(self, url, data, session=None, timeout=None):
        session = session or self._session
        timeout = timeout or self._timeout
        _logger.debug('POST %s: %s', url, pprint.pformat(data))
        response = session.post(url, json=data, timeout=timeout)
        _logger.debug('Posting to SignalFx %s (%d): %s',
                      'succeeded' if response.ok else 'failed',
                      response.status_code, response.text)
        return response

    def _delete(self, url, session=None, timeout=None,
                ignore_not_found=False):
        session = session or self._session
        timeout = timeout or self._timeout
        _logger.debug('DELETE %s', url)
        response = session.delete(url, timeout=timeout)
        _logger.debug('Deleting from SignalFx %s (%d)',
                      'succeeded' if response.ok else 'failed',
                      response.status_code)
        if response.status_code is requests.codes.not_found and \
                ignore_not_found:
            response.status_code = requests.codes.no_content
        return response

    def _search_metrics_and_metadata(self, metadata_endpoint, query,
                                     order_by=None, offset=None,
                                     limit=None, timeout=None):
        """
        generic function for elasticsearch queries; can search metrics,
            dimensions, metrictimeseries by changing metadata_endpoint
        Args:
            metadata_endpoint (string): API endpoint suffix (e.g. 'v2/metric')
            query (string): elasticsearch string query
            order_by (optional[string]): property by which to order results
            offset (optional[int]): number of results to skip for pagination
                (default=0)
            limit (optional[int]): how many results to return (default=50)
            timeout (optional[int]): how long to wait for response (in seconds)

        Returns:
            dictionary of query result
        """
        _logger.debug('Performing an elasticsearch for %(qry)s at %(pt)s',
                      {'qry': query, 'pt': metadata_endpoint})
        url_to_get = '{0}?query={1}'.format(self._u(metadata_endpoint), query)
        if order_by is not None:
            url_to_get += '&orderBy=' + order_by
        # for offset and limit, use API defaults (by leaving them out of url)
        if offset is not None:
            url_to_get += '&offset=' + str(offset)
        if limit is not None:
            url_to_get += '&limit=' + str(limit)
        timeout = timeout or self._timeout
        resp = self._get(url_to_get, session=self._session, timeout=timeout)
        resp.raise_for_status()
        return resp.json()

    def _get_object_by_name(self, object_endpoint, object_name, timeout=None):
        """
        generic function to get object (metadata, tag, ) by name from SignalFx.

        Args:
            object_endpoint (string): API endpoint suffix (e.g. 'v2/tag')
            object_name (string): name of the object (e.g. 'jvm.cpu.load')

        Returns:
            dictionary of response
        """
        timeout = timeout or self._timeout
        resp = self._get(self._u(object_endpoint, object_name),
                         session=self._session, timeout=timeout)
        resp.raise_for_status()
        return resp.json()

    # functionality related to metrics
    def search_metrics(self, *args, **kwargs):
        """
        Args:
            query (string): elasticsearch string query
            order_by (optional[string]): property by which to order results
            offset (optional[int]): number of results to skip for pagination
                (default=0)
            limit (optional[int]): how many results to return (default=50)
            timeout (optional[int]): how long to wait for response (in seconds)

        Returns:
            result of query search on metrics
        """
        return self._search_metrics_and_metadata(
            self._METRIC_ENDPOINT_SUFFIX, *args, **kwargs)

    def get_metric_by_name(self, metric_name, **kwargs):
        """
        get a metric by name

        Args:
            metric_name (string): name of metric

        Returns:
            dictionary of response
        """
        return self._get_object_by_name(self._METRIC_ENDPOINT_SUFFIX,
                                        metric_name,
                                        **kwargs)

    def update_metric_by_name(self, metric_name, metric_type, description=None,
                              custom_properties=None, tags=None, **kwargs):
        """
        Create or update a metric object

        Args:
            metric_name (string): name of metric
            type (string): metric type, must be one of 'gauge', 'counter',
                            'cumulative_counter'
            description (optional[string]): a description
            custom_properties (optional[dict]): dictionary of custom properties
            tags (optional[list of strings]): list of tags associated with
                metric
        """
        data = {'type': metric_type.upper(),
                'description': description or '',
                'customProperties': custom_properties or {},
                'tags': tags or []}
        resp = self._put(self._u(self._METRIC_ENDPOINT_SUFFIX,
                                 str(metric_name)),
                         data=data, **kwargs)
        resp.raise_for_status()
        return resp.json()

    # functionality related to dimensions
    def search_dimensions(self, *args, **kwargs):
        """
        Args:
            query (string): elasticsearch string query
            order_by (optional[string]): property by which to order results
            offset (optional[int]): number of results to skip for pagination
                (default=0)
            limit (optional[int]): how many results to return (default=50)
            timeout (optional[int]): how long to wait for response (in seconds)

        Returns:
            result of query search on dimensions
        """
        return self._search_metrics_and_metadata(
            self._DIMENSION_ENDPOINT_SUFFIX, *args, **kwargs)

    def get_dimension(self, key, value, **kwargs):
        """
        get a dimension by key and value

        Args:
            key (string): key of the dimension
            value (string): value of the dimension

        Returns:
            dictionary of response
        """
        return self._get_object_by_name(self._DIMENSION_ENDPOINT_SUFFIX,
                                        '{0}/{1}'.format(key, value),
                                        **kwargs)

    def update_dimension(self, key, value, description=None,
                         custom_properties=None, tags=None, **kwargs):
        """
        update a dimension
        Args:
            key (string): key of the dimension
            value (string): value of the dimension
            description (optional[string]): a description
            custom_properties (optional[dict]): dictionary of custom properties
            tags (optional[list of strings]): list of tags associated with
                metric
        """
        data = {'description': description or '',
                'customProperties': custom_properties or {},
                'tags': tags or [],
                'key': key,
                'value': value}
        resp = self._put(self._u(self._DIMENSION_ENDPOINT_SUFFIX, key, value),
                         data=data, **kwargs)
        resp.raise_for_status()
        return resp.json()

    # functionality related to metrictimeseries
    def search_metric_time_series(self, *args, **kwargs):
        """
        Args:
            query (string): elasticsearch string query
            order_by (optional[string]): property by which to order results
            offset (optional[int]): number of results to skip for pagination
                (default=0)
            limit (optional[int]): how many results to return (default=50)
            timeout (optional[int]): how long to wait for response (in seconds)

        Returns:
            result of query search on metric time series

        """
        return self._search_metrics_and_metadata(self._MTS_ENDPOINT_SUFFIX,
                                                 *args, **kwargs)

    def get_metric_time_series(self, mts_id, **kwargs):
        """get a metric time series by id"""
        return self._get_object_by_name(self._MTS_ENDPOINT_SUFFIX,
                                        mts_id,
                                        **kwargs)

    # functionality related to tags
    def search_tags(self, *args, **kwargs):
        """
        Args:
            query (string): elasticsearch string query
            order_by (optional[string]): property by which to order results
            offset (optional[int]): number of results to skip for pagination
                (default=0)
            limit (optional[int]): how many results to return (default=50)
            timeout (optional[int]): how long to wait for response (in seconds)

        Returns:
            result of query search on tags

        """
        return self._search_metrics_and_metadata(self._TAG_ENDPOINT_SUFFIX,
                                                 *args, **kwargs)

    def get_tag(self, tag_name, **kwargs):
        """get a tag by name

        Args:
            tag_name (string): name of tag to get

        Returns:
            dictionary of the response

        """
        return self._get_object_by_name(self._TAG_ENDPOINT_SUFFIX,
                                        tag_name,
                                        **kwargs)

    def update_tag(self, tag_name, description=None,
                   custom_properties=None, **kwargs):
        """update a tag by name

        Args:
            tag_name (string): name of tag to update
            description (optional[string]): a description
            custom_properties (optional[dict]): dictionary of custom properties
        """
        data = {'description': description or '',
                'customProperties': custom_properties or {}}
        resp = self._put(self._u(self._TAG_ENDPOINT_SUFFIX, tag_name),
                         data=data, **kwargs)
        resp.raise_for_status()
        return resp.json()

    def delete_tag(self, tag_name, **kwargs):
        """delete a tag by name

        Args:
            tag_name (string): name of tag to delete
        """
        resp = self._delete(self._u(self._TAG_ENDPOINT_SUFFIX, tag_name),
                            **kwargs)
        resp.raise_for_status()
        # successful delete returns 204, which has no associated json
        return resp

    # functionality related to organizations (more to come)
    def get_organization(self, **kwargs):
        """Get the organization to which the user belongs

        Returns:
            dictionary of the response
        """
        resp = self._get(self._u(self._ORGANIZATION_ENDPOINT_SUFFIX),
                         **kwargs)
        resp.raise_for_status()
        return resp.json()

    # functionality related to detectors
    def get_detectors(self, name=None, batch_size=100, **kwargs):
        """Retrieve all (v2) detectors matching the given name; all (v2)
        detectors otherwise.

        Note that this method will loop through the paging of the results and
        accumulate all detectors that match the query. This may be expensive.
        """
        detectors = []
        offset = 0
        while True:
            resp = self._get(
                self._u(self._DETECTOR_ENDPOINT_SUFFIX),
                params={'offset': offset, 'limit': batch_size, 'name': name},
                **kwargs)
            resp.raise_for_status()
            data = resp.json()
            detectors += data['results']
            if len(detectors) == data['count']:
                break
            offset = len(detectors)
        return detectors

    def create_detector(self, detector):
        """Creates a new detector.

        Args:
            detector (object): the detector model object. Will be serialized as
                JSON.
        Returns:
            dictionary of the response (created detector model).
        """
        resp = self._post(self._u(self._DETECTOR_ENDPOINT_SUFFIX),
                          data=detector)
        resp.raise_for_status()
        return resp.json()

    def update_detector(self, detector_id, detector):
        """Update an existing detector.

        Args:
            detector_id (string): the ID of the detector.
            detector (object): the detector model object. Will be serialized as
                JSON.
        Returns:
            dictionary of the response (updated detector model).
        """
        resp = self._put(self._u(self._DETECTOR_ENDPOINT_SUFFIX, detector_id),
                         data=detector)
        resp.raise_for_status()
        return resp.json()

    def delete_detector(self, detector_id, **kwargs):
        """Remove a detector.

        Args:
            detector_id (string): the ID of the detector.
        """
        resp = self._delete(self._u(self._DETECTOR_ENDPOINT_SUFFIX,
                                    detector_id),
                            **kwargs)
        resp.raise_for_status()
        # successful delete returns 204, which has no response json
        return resp
<|endoftext|>


<commit_before>
from xstruct import pack

from Description import Description
from Constants import *

# Import prebuild orders
from ObjectDesc import *
_descriptions = None
def descriptions(added=None):
	global _descriptions

	if _descriptions == None:
		_descriptions = import_subtype(edir(__file__))

	if added != None:
		_descriptions[ added.subtype ] = added
        return _descriptions

struct_map = {
	ARG_ABS_COORD:	("qqq",			3),
	ARG_TIME: 		("I",			1),
	ARG_OBJECT:		("I",			1),
	ARG_PLAYER:		("2I",			2),
	ARG_REL_COORD:	("Iqqq",		3),
	ARG_RANGE:		("iiii",		4),
	ARG_LIST:		("[ISI][II]", 	2),
}

class ClassNicePrint(type):
	def __str__(self):
		return "<dynamic-class '%s' at %s>" % (self.name, hex(id(self)))
	__repr__ = __str__

from Header import Header
from Order import Order
class DynamicBaseOrder(Order):
	"""\
	An Order Type built by a OrderDesc.
	"""
	substruct = ""
	subtype = -1
	name = "Base"

	__metaclass__ = ClassNicePrint

	def __init__(self, sequence, \
			id, type, slot, turns, resources, \
			*args):
		Order.__init__(self, sequence, \
			id, type, slot, turns, resources)

		# Figure out if we are in single or multiple mode
		short = len(args) == len(self.names)

		for name, type in self.names:
			struct, size = struct_map[type]

			if short:
				size = 1

			if size == 1:
				setattr(self, name, args[0])
			else:
				if len(args) < size:
					raise ValueError("Incorrect number of arguments")
				setattr(self, name, args[0:size])

			args = args[size:]

		# FIXME: Need to figure out the length a better way
		self.length = len(self.__repr__()) - Header.size

	def __repr__(self):
		args = []
		for name, type in self.names:
			struct, size = struct_map[type]

			attr = getattr(self, name)
			if size == 1:
				args.append(attr)
			else:
				args = args + list(attr)

		output = Order.__repr__(self)
		output += apply(pack, [self.substruct,] + args)
		return output

	def __str__(self):
		return "<netlib.objects.OrderExtra.DynamicOrder - %s @ %s>" % (self.name, id(self))

class OrderDesc(Description):
	"""\
	The OrderDesc packet consists of:
	    * a UInt32, order type
	    * a String, name
	    * a String, description
	    * a list of
		  * a String, argument name
		  * a UInt32, argument type
		  * a String, description

	IE
	ID: 1001
	Name: Drink With Friends
	Description: Go to the pub and drink with friends.
	Arguments:
		Name: How Long
		Type: ARG_TIME
		Description: How many turns to drink for.

		Name: Who With
		Type: ARG_PLAYER
		Description: Which player to drink with.

		Name: Where
		Type: ARG_COORD
		Description: Where to go drinking.

		Name: Cargo
		Type: ARG_INT
		Description: How much beer to drink.
	"""
	no = 9
	struct="I SS [SIS]"

	def __init__(self, sequence, \
			id, name, description, \
			arguments):
		Description.__init__(self, sequence, id)

		self.name = name
		self.description = description
		self.arguments = arguments

		self.length = 4 + \
			4 + len(name) + 1 + \
			4 + len(description) + 1 + \
			4

		for argument in arguments:
			self.length += \
				4 + len(argument[0]) + 1 + \
				4 + \
				4 + len(argument[2]) + 1

	def __repr__(self):
		output = Description.__repr__(self)
		output += pack(self.struct, \
				self.id, \
				self.name, \
				self.description, \
				self.arguments)

		return output

	def build(self):
		"""\
		*Internal*

		Builds a class from this description.
		"""
		class DynamicOrder(DynamicBaseOrder):
			pass

		DynamicOrder.name = self.name
#		DynamicOrder.__doc__ = self.description

		# Arguments
		DynamicOrder.names = []
		DynamicOrder.subtype = self.id

		for name, type, desc in self.arguments:
			struct, size = struct_map[type]

 			DynamicOrder.names.append((name, type))
			DynamicOrder.substruct += struct
			setattr(DynamicOrder, name + "__doc__", desc)

		return DynamicOrder

	def register(self):
		descriptions(self.build())

__all__ = ["descriptions", "OrderDesc"]
<commit_msg>Small updates...<commit_after>
from xstruct import pack

from Description import Description
from Constants import *

# Import prebuild orders
from ObjectDesc import *
_descriptions = None
def descriptions(added=None):
	global _descriptions

	if _descriptions == None:
		_descriptions = import_subtype(edir(__file__))

	if added != None:
		_descriptions[ added.subtype ] = added
        return _descriptions

struct_map = {
	ARG_ABS_COORD:	("qqq",			3),
	ARG_TIME: 		("I",			1),
	ARG_OBJECT:		("I",			1),
	ARG_PLAYER:		("II",			2),
	ARG_REL_COORD:	("Iqqq",		3),
	ARG_RANGE:		("iiii",		4),
	ARG_LIST:		("[ISI][II]", 	2),
}

class ClassNicePrint(type):
	def __str__(self):
		return "<dynamic-class '%s' at %s>" % (self.name, hex(id(self)))
	__repr__ = __str__

from Header import Header
from Order import Order
class DynamicBaseOrder(Order):
	"""\
	An Order Type built by a OrderDesc.
	"""
	substruct = ""
	subtype = -1
	name = "Base"

	__metaclass__ = ClassNicePrint

	def __init__(self, sequence, \
			id, type, slot, turns, resources, \
			*args):
		Order.__init__(self, sequence, \
			id, type, slot, turns, resources)

		# Figure out if we are in single or multiple mode
		short = len(args) == len(self.names)

		for name, type in self.names:
			struct, size = struct_map[type]

			if short:
				size = 1

			if size == 1:
				setattr(self, name, args[0])
			else:
				if len(args) < size:
					raise ValueError("Incorrect number of arguments")
				setattr(self, name, args[0:size])

			args = args[size:]

		# FIXME: Need to figure out the length a better way
		self.length = len(self.__repr__()) - Header.size

	def __repr__(self):
		args = []
		for name, type in self.names:
			struct, size = struct_map[type]

			attr = getattr(self, name)
			if size == 1:
				args.append(attr)
			else:
				args = args + list(attr)

		output = Order.__repr__(self)
		output += apply(pack, [self.substruct,] + args)
		return output

	def __str__(self):
		return "<netlib.objects.OrderExtra.DynamicOrder - %s @ %s>" % (self.name, hex(id(self)))

class OrderDesc(Description):
	"""\
	The OrderDesc packet consists of:
	    * a UInt32, order type
	    * a String, name
	    * a String, description
	    * a list of
		  * a String, argument name
		  * a UInt32, argument type
		  * a String, description

	IE
	ID: 1001
	Name: Drink With Friends
	Description: Go to the pub and drink with friends.
	Arguments:
		Name: How Long
		Type: ARG_TIME
		Description: How many turns to drink for.

		Name: Who With
		Type: ARG_PLAYER
		Description: Which player to drink with.

		Name: Where
		Type: ARG_COORD
		Description: Where to go drinking.

		Name: Cargo
		Type: ARG_INT
		Description: How much beer to drink.
	"""
	no = 9
	struct="I SS [SIS]"

	def __init__(self, sequence, \
			id, name, description, \
			arguments):
		Description.__init__(self, sequence, id)

		self.name = name
		self.description = description
		self.arguments = arguments

		self.length = 4 + \
			4 + len(name) + 1 + \
			4 + len(description) + 1 + \
			4

		for argument in arguments:
			self.length += \
				4 + len(argument[0]) + 1 + \
				4 + \
				4 + len(argument[2]) + 1

	def __repr__(self):
		output = Description.__repr__(self)
		output += pack(self.struct, \
				self.id, \
				self.name, \
				self.description, \
				self.arguments)

		return output

	def build(self):
		"""\
		*Internal*

		Builds a class from this description.
		"""
		class DynamicOrder(DynamicBaseOrder):
			pass

		DynamicOrder.name = self.name
#		DynamicOrder.__doc__ = self.description

		# Arguments
		DynamicOrder.names = []
		DynamicOrder.subtype = self.id

		for name, type, desc in self.arguments:
			struct, size = struct_map[type]

 			DynamicOrder.names.append((name, type))
			DynamicOrder.substruct += struct
			setattr(DynamicOrder, name + "__doc__", desc)

		return DynamicOrder

	def register(self):
		descriptions(self.build())

__all__ = ["descriptions", "OrderDesc"]
<|endoftext|>