<commit_before>from typing import Any, Dict, Optional, Text
from rasa.nlu.config import RasaNLUModelConfig
from rasa.nlu.featurizers.featurzier import Featurizer
from rasa.nlu.training_data import Message, TrainingData
from rasa.nlu.constants import (
    MESSAGE_TEXT_ATTRIBUTE,
    MESSAGE_VECTOR_DENSE_FEATURE_NAMES,
    SPACY_FEATURIZABLE_ATTRIBUTES,
)
import torch
from transformers import *
import numpy as np

logger = logging.getLogger(__name__)

model_dictionary = {
    "bert-base-uncased": BertModel,
    "openai-gpt": OpenAIGPTModel,
    # "gpt2": GPT2Model,
    # "transfo-xl-wt103": TransfoXLModel,
    # "xlnet-base-cased": XLNetModel,
    # "xlm-mlm-enfr-1024": XLMModel,
    # "distilbert-base-uncased": DistilBertModel,
    # "roberta-base": RobertaModel,
}

tokenizer_dictionary = {
    "bert-base-uncased": BertTokenizer,
    "openai-gpt": OpenAIGPTTokenizer,
    # "gpt2": GPT2Tokenizer,
    # "transfo-xl-wt103": TransfoXLTokenizer,
    # "xlnet-base-cased": XLNetTokenizer,
    # "xlm-mlm-enfr-1024": XLMTokenizer,
    # "distilbert-base-uncased": DistilBertTokenizer,
    # "roberta-base": RobertaTokenizer,
}

special_tokens_present = {
    "bert-base-uncased": True,
    "openai-gpt": False,
    # "gpt2": False,
    # "transfo-xl-wt103": False,
    # "xlnet-base-cased": True,
    # "xlm-mlm-enfr-1024": True,
    # "distilbert-base-uncased": True,
    # "roberta-base": True,
}

class PreTrainedLMFeaturizer(Featurizer):

    provides = [
        MESSAGE_VECTOR_DENSE_FEATURE_NAMES[attribute]
        for attribute in SPACY_FEATURIZABLE_ATTRIBUTES
    ]

    defaults = {
        # model key identified by HF Transformers
        "model_key": "bert-base-uncased"
    }

    def _load_transformers_params(self):

        self.lm_key = self.component_config["lm_key"]

        if self.lm_key not in tokenizer_dictionary:
            logger.error("{} not a valid model key name".format(self.lm_key))
            raise

        logger.info("Loading Tokenizer and Model for {}".format(self.lm_key))
        self.tokenizer = tokenizer_dictionary[self.lm_key].from_pretrained(self.lm_key)
        self.model = model_dictionary[self.lm_key].from_pretrained(self.lm_key)
        self.contains_special_token = special_tokens_present[self.lm_key]
        if self.contains_special_token:
            self.pad_token_id = self.tokenizer.pad_token_id
        else:
            special_tokens_dict = {"pad_token": "[PAD]"}
            self.tokenizer.add_special_tokens(special_tokens_dict)
            self.model.resize_token_embeddings(len(self.tokenizer))
            self.pad_token_id = self.tokenizer.pad_token_id

    def __init__(self, component_config: Dict[Text, Any] = None) -> None:

        super(PreTrainedLMFeaturizer, self).__init__(component_config)

        self._load_transformers_params()

    def train(
        self,
        training_data: TrainingData,
        config: Optional[RasaNLUModelConfig],
        **kwargs: Any,
    ) -> None:

        bs = 128

        for attribute in [MESSAGE_TEXT_ATTRIBUTE]:

            start_index = 0

            while start_index < len(training_data.intent_examples):

                end_index = min(start_index + bs, len(training_data.intent_examples))
                batch_examples = training_data.intent_examples[start_index:end_index]
                batch_text = [
                    self._clean_text(ex.get(attribute)) for ex in batch_examples
                ]

                batch_feats = self._compute_features(batch_text)

                for index, ex in enumerate(batch_examples):

                    ex.set(
                        MESSAGE_VECTOR_DENSE_FEATURE_NAMES[attribute],
                        self._combine_with_existing_dense_features(
                            ex,
                            batch_feats[index],
                            MESSAGE_VECTOR_DENSE_FEATURE_NAMES[attribute],
                        ),
                    )

                    # print(ex.get(attribute), batch_feats[index].shape[0])

                start_index += bs

        # for example in training_data.intent_examples:
        #     for attribute in SPACY_FEATURIZABLE_ATTRIBUTES:
        #         self._set_lm_features(example, attribute)

    @staticmethod
    def _clean_text(text):

        cleaned_text = re.sub(
            # there is a space or an end of a string after it
            r"[^\w#@&]+(?=\s|$)|"
            # there is a space or beginning of a string before it
            # not followed by a number
            r"(\s|^)[^\w#@&]+(?=[^0-9\s])|"
            # not in between numbers and not . or @ or & or - or #
            # e.g. 10'000.00 or blabla@gmail.com
            # and not url characters
            r"(?<=[^0-9\s])[^\w._~:/?#\[\]()@!$&*+,;=-]+(?=[^0-9\s])",
            " ",
            text,
        )

        if not cleaned_text.strip():
            cleaned_text = text

        return cleaned_text

    def _compute_input_ids(self, batch_examples):

        batch_input_ids = []
        max_seq_len = 0
        actual_seq_lengths = []
        for example in batch_examples:

            example_input_ids = self.tokenizer.encode(
                example, add_special_tokens=self.contains_special_token
            )
            max_seq_len = max(max_seq_len, len(example_input_ids))
            actual_seq_lengths.append(len(example_input_ids))
            batch_input_ids.append(example_input_ids)

        # add padding
        padded_input_ids = []

        # Some models don't contain pad token, we use unknown token as padding token.This doesn't affect the computation
        # since we compute an attention mask anyways.

        # pad_token_id = self.tokenizer.pad_token_id if self.contains_special_token else self.tokenizer.unk_token_id
        for example_input_ids in batch_input_ids:
            padded_input_ids.append(
                example_input_ids
                + [self.pad_token_id] * (max_seq_len - len(example_input_ids))
            )

        return torch.tensor(padded_input_ids), actual_seq_lengths

    def _compute_attention_mask(self, actual_seq_lengths):

        attention_mask = []
        max_seq_length = max(actual_seq_lengths)
        for index in range(len(actual_seq_lengths)):
            example_seq_length = actual_seq_lengths[index]
            attention_mask.append(
                [1] * example_seq_length + [0] * (max_seq_length - example_seq_length)
            )

        attention_mask = np.array(attention_mask).astype(np.float32)

        return torch.tensor(attention_mask)

    def _compute_features(self, batch_inputs):

        batch_model_inputs, actual_seq_lengths = self._compute_input_ids(batch_inputs)
        batch_attention_mask = self._compute_attention_mask(actual_seq_lengths)

        with torch.no_grad():
            last_hidden_states = self.model(
                batch_model_inputs, attention_mask=batch_attention_mask
            )[
                0
            ].numpy()  # Models outputs are now numpy array
            sequence_embedding = last_hidden_states  # First element of batch

            truncated_embeds = self._extract_nonpadded_embeddings(
                sequence_embedding, actual_seq_lengths
            )

            return truncated_embeds

    def _extract_nonpadded_embeddings(self, embeddings, actual_seq_lengths):

        truncated_embeds = []
        for index, embedding in enumerate(embeddings):
            unmasked_embedding = embedding[: actual_seq_lengths[index]]

            if self.contains_special_token:
                # dim - (seq + 2, hdim)
                # Discard SEP token and move CLS token to last index
                unmasked_embedding = unmasked_embedding[:-1, :]  # Discard SEP
                unmasked_embedding = np.roll(
                    unmasked_embedding, -1, axis=0
                )  # Move CLS to back
            else:
                unmasked_embedding = np.concatenate(
                    [unmasked_embedding, np.zeros((1, unmasked_embedding.shape[-1]))],
                    axis=0,
                )
            truncated_embeds.append(unmasked_embedding)

        return np.array(truncated_embeds)

    def _set_lm_features(self, example, attribute=MESSAGE_TEXT_ATTRIBUTE):

        message_attribute_text = example.get(attribute)
        if message_attribute_text:
            # Encode text
            input_ids = torch.tensor(
                [
                    self.tokenizer.encode(
                        message_attribute_text,
                        add_special_tokens=self.contains_special_token,
                    )
                ]
            )  # Add special tokens takes care of adding [CLS], [SEP], <s>... tokens in the right way for each model.
            with torch.no_grad():
                last_hidden_states = self.model(input_ids)[
                    0
                ].numpy()  # Models outputs are now numpy array
                sequence_embedding = last_hidden_states[0]  # First element of batch

                if self.contains_special_token:
                    # dim - (seq + 2, hdim)
                    # Discard SEP token and move CLS token to last index
                    sequence_embedding = sequence_embedding[:-1]  # Discard SEP
                    sequence_embedding = np.roll(
                        sequence_embedding, -1
                    )  # Move CLS to back
                else:
                    sequence_embedding = np.concatenate(
                        [
                            sequence_embedding,
                            np.zeros((1, sequence_embedding.shape[-1])),
                        ],
                        axis=0,
                    )

                features = self._combine_with_existing_dense_features(
                    example,
                    sequence_embedding,
                    MESSAGE_VECTOR_DENSE_FEATURE_NAMES[attribute],
                )
                example.set(MESSAGE_VECTOR_DENSE_FEATURE_NAMES[attribute], features)

    def process(self, message: Message, **kwargs: Any) -> None:

        feats = self._compute_features([message.get(MESSAGE_TEXT_ATTRIBUTE)])
        message.set(
            MESSAGE_VECTOR_DENSE_FEATURE_NAMES[MESSAGE_TEXT_ATTRIBUTE],
            self._combine_with_existing_dense_features(
                message,
                feats[0],
                MESSAGE_VECTOR_DENSE_FEATURE_NAMES[MESSAGE_TEXT_ATTRIBUTE],
            ),
        )
<commit_msg>fix import<commit_after>from typing import Any, Dict, Optional, Text
from rasa.nlu.config import RasaNLUModelConfig
from rasa.nlu.featurizers.featurzier import Featurizer
from rasa.nlu.training_data import Message, TrainingData
from rasa.nlu.constants import (
    MESSAGE_TEXT_ATTRIBUTE,
    MESSAGE_VECTOR_DENSE_FEATURE_NAMES,
    SPACY_FEATURIZABLE_ATTRIBUTES,
)
import torch
import re
from transformers import *
import numpy as np

logger = logging.getLogger(__name__)

model_dictionary = {
    "bert-base-uncased": BertModel,
    "openai-gpt": OpenAIGPTModel,
    # "gpt2": GPT2Model,
    # "transfo-xl-wt103": TransfoXLModel,
    # "xlnet-base-cased": XLNetModel,
    # "xlm-mlm-enfr-1024": XLMModel,
    # "distilbert-base-uncased": DistilBertModel,
    # "roberta-base": RobertaModel,
}

tokenizer_dictionary = {
    "bert-base-uncased": BertTokenizer,
    "openai-gpt": OpenAIGPTTokenizer,
    # "gpt2": GPT2Tokenizer,
    # "transfo-xl-wt103": TransfoXLTokenizer,
    # "xlnet-base-cased": XLNetTokenizer,
    # "xlm-mlm-enfr-1024": XLMTokenizer,
    # "distilbert-base-uncased": DistilBertTokenizer,
    # "roberta-base": RobertaTokenizer,
}

special_tokens_present = {
    "bert-base-uncased": True,
    "openai-gpt": False,
    # "gpt2": False,
    # "transfo-xl-wt103": False,
    # "xlnet-base-cased": True,
    # "xlm-mlm-enfr-1024": True,
    # "distilbert-base-uncased": True,
    # "roberta-base": True,
}

class PreTrainedLMFeaturizer(Featurizer):

    provides = [
        MESSAGE_VECTOR_DENSE_FEATURE_NAMES[attribute]
        for attribute in SPACY_FEATURIZABLE_ATTRIBUTES
    ]

    defaults = {
        # model key identified by HF Transformers
        "model_key": "bert-base-uncased"
    }

    def _load_transformers_params(self):

        self.lm_key = self.component_config["lm_key"]

        if self.lm_key not in tokenizer_dictionary:
            logger.error("{} not a valid model key name".format(self.lm_key))
            raise

        logger.info("Loading Tokenizer and Model for {}".format(self.lm_key))
        self.tokenizer = tokenizer_dictionary[self.lm_key].from_pretrained(self.lm_key)
        self.model = model_dictionary[self.lm_key].from_pretrained(self.lm_key)
        self.contains_special_token = special_tokens_present[self.lm_key]
        if self.contains_special_token:
            self.pad_token_id = self.tokenizer.pad_token_id
        else:
            special_tokens_dict = {"pad_token": "[PAD]"}
            self.tokenizer.add_special_tokens(special_tokens_dict)
            self.model.resize_token_embeddings(len(self.tokenizer))
            self.pad_token_id = self.tokenizer.pad_token_id

    def __init__(self, component_config: Dict[Text, Any] = None) -> None:

        super(PreTrainedLMFeaturizer, self).__init__(component_config)

        self._load_transformers_params()

    def train(
        self,
        training_data: TrainingData,
        config: Optional[RasaNLUModelConfig],
        **kwargs: Any,
    ) -> None:

        bs = 128

        for attribute in [MESSAGE_TEXT_ATTRIBUTE]:

            start_index = 0

            while start_index < len(training_data.intent_examples):

                end_index = min(start_index + bs, len(training_data.intent_examples))
                batch_examples = training_data.intent_examples[start_index:end_index]
                batch_text = [
                    self._clean_text(ex.get(attribute)) for ex in batch_examples
                ]

                batch_feats = self._compute_features(batch_text)

                for index, ex in enumerate(batch_examples):

                    ex.set(
                        MESSAGE_VECTOR_DENSE_FEATURE_NAMES[attribute],
                        self._combine_with_existing_dense_features(
                            ex,
                            batch_feats[index],
                            MESSAGE_VECTOR_DENSE_FEATURE_NAMES[attribute],
                        ),
                    )

                    # print(ex.get(attribute), batch_feats[index].shape[0])

                start_index += bs

        # for example in training_data.intent_examples:
        #     for attribute in SPACY_FEATURIZABLE_ATTRIBUTES:
        #         self._set_lm_features(example, attribute)

    @staticmethod
    def _clean_text(text):

        cleaned_text = re.sub(
            # there is a space or an end of a string after it
            r"[^\w#@&]+(?=\s|$)|"
            # there is a space or beginning of a string before it
            # not followed by a number
            r"(\s|^)[^\w#@&]+(?=[^0-9\s])|"
            # not in between numbers and not . or @ or & or - or #
            # e.g. 10'000.00 or blabla@gmail.com
            # and not url characters
            r"(?<=[^0-9\s])[^\w._~:/?#\[\]()@!$&*+,;=-]+(?=[^0-9\s])",
            " ",
            text,
        )

        if not cleaned_text.strip():
            cleaned_text = text

        return cleaned_text

    def _compute_input_ids(self, batch_examples):

        batch_input_ids = []
        max_seq_len = 0
        actual_seq_lengths = []
        for example in batch_examples:

            example_input_ids = self.tokenizer.encode(
                example, add_special_tokens=self.contains_special_token
            )
            max_seq_len = max(max_seq_len, len(example_input_ids))
            actual_seq_lengths.append(len(example_input_ids))
            batch_input_ids.append(example_input_ids)

        # add padding
        padded_input_ids = []

        # Some models don't contain pad token, we use unknown token as padding token.This doesn't affect the computation
        # since we compute an attention mask anyways.

        # pad_token_id = self.tokenizer.pad_token_id if self.contains_special_token else self.tokenizer.unk_token_id
        for example_input_ids in batch_input_ids:
            padded_input_ids.append(
                example_input_ids
                + [self.pad_token_id] * (max_seq_len - len(example_input_ids))
            )

        return torch.tensor(padded_input_ids), actual_seq_lengths

    def _compute_attention_mask(self, actual_seq_lengths):

        attention_mask = []
        max_seq_length = max(actual_seq_lengths)
        for index in range(len(actual_seq_lengths)):
            example_seq_length = actual_seq_lengths[index]
            attention_mask.append(
                [1] * example_seq_length + [0] * (max_seq_length - example_seq_length)
            )

        attention_mask = np.array(attention_mask).astype(np.float32)

        return torch.tensor(attention_mask)

    def _compute_features(self, batch_inputs):

        batch_model_inputs, actual_seq_lengths = self._compute_input_ids(batch_inputs)
        batch_attention_mask = self._compute_attention_mask(actual_seq_lengths)

        with torch.no_grad():
            last_hidden_states = self.model(
                batch_model_inputs, attention_mask=batch_attention_mask
            )[
                0
            ].numpy()  # Models outputs are now numpy array
            sequence_embedding = last_hidden_states  # First element of batch

            truncated_embeds = self._extract_nonpadded_embeddings(
                sequence_embedding, actual_seq_lengths
            )

            return truncated_embeds

    def _extract_nonpadded_embeddings(self, embeddings, actual_seq_lengths):

        truncated_embeds = []
        for index, embedding in enumerate(embeddings):
            unmasked_embedding = embedding[: actual_seq_lengths[index]]

            if self.contains_special_token:
                # dim - (seq + 2, hdim)
                # Discard SEP token and move CLS token to last index
                unmasked_embedding = unmasked_embedding[:-1, :]  # Discard SEP
                unmasked_embedding = np.roll(
                    unmasked_embedding, -1, axis=0
                )  # Move CLS to back
            else:
                unmasked_embedding = np.concatenate(
                    [unmasked_embedding, np.zeros((1, unmasked_embedding.shape[-1]))],
                    axis=0,
                )
            truncated_embeds.append(unmasked_embedding)

        return np.array(truncated_embeds)

    def _set_lm_features(self, example, attribute=MESSAGE_TEXT_ATTRIBUTE):

        message_attribute_text = example.get(attribute)
        if message_attribute_text:
            # Encode text
            input_ids = torch.tensor(
                [
                    self.tokenizer.encode(
                        message_attribute_text,
                        add_special_tokens=self.contains_special_token,
                    )
                ]
            )  # Add special tokens takes care of adding [CLS], [SEP], <s>... tokens in the right way for each model.
            with torch.no_grad():
                last_hidden_states = self.model(input_ids)[
                    0
                ].numpy()  # Models outputs are now numpy array
                sequence_embedding = last_hidden_states[0]  # First element of batch

                if self.contains_special_token:
                    # dim - (seq + 2, hdim)
                    # Discard SEP token and move CLS token to last index
                    sequence_embedding = sequence_embedding[:-1]  # Discard SEP
                    sequence_embedding = np.roll(
                        sequence_embedding, -1
                    )  # Move CLS to back
                else:
                    sequence_embedding = np.concatenate(
                        [
                            sequence_embedding,
                            np.zeros((1, sequence_embedding.shape[-1])),
                        ],
                        axis=0,
                    )

                features = self._combine_with_existing_dense_features(
                    example,
                    sequence_embedding,
                    MESSAGE_VECTOR_DENSE_FEATURE_NAMES[attribute],
                )
                example.set(MESSAGE_VECTOR_DENSE_FEATURE_NAMES[attribute], features)

    def process(self, message: Message, **kwargs: Any) -> None:

        feats = self._compute_features([message.get(MESSAGE_TEXT_ATTRIBUTE)])
        message.set(
            MESSAGE_VECTOR_DENSE_FEATURE_NAMES[MESSAGE_TEXT_ATTRIBUTE],
            self._combine_with_existing_dense_features(
                message,
                feats[0],
                MESSAGE_VECTOR_DENSE_FEATURE_NAMES[MESSAGE_TEXT_ATTRIBUTE],
            ),
        )
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8
"""
Tools for working with dust attenuation for M31.

2015-05-12 - Created by Jonathan Sick
"""

import numpy as np

from starfisher.dust import SF11ExtinctionCurve

def mw_Av():
    """Build the A_V attenuation by the MW towards M31."""
    curve = SF11ExtinctionCurve()
    ratio = curve['Landolt V']  # A_V / E(B-V) from T6 of SF2011
    return ratio * 0.07  # 0.07 is E(B-V) to M31 from Schlegel 1998

def phat_rel_extinction():
    """Build array of relative extinction in PHAT bands."""
    curve = SF11ExtinctionCurve()
    rel_extinction = np.ones(6, dtype=float)
    rel_extinction[0] = curve.extinction_ratio('WFC3 F275W')
    rel_extinction[1] = curve.extinction_ratio('WFC3 F336W')
    rel_extinction[2] = curve.extinction_ratio('ACS F475W')
    rel_extinction[3] = curve.extinction_ratio('ACS F814W')
    rel_extinction[4] = curve.extinction_ratio('WFC3 F110W')
    rel_extinction[5] = curve.extinction_ratio('WFC3 F160W')
    return rel_extinction
<commit_msg>Add LewisDustLaw<commit_after>#!/usr/bin/env python
# encoding: utf-8
"""
Tools for working with dust attenuation for M31.

2015-05-12 - Created by Jonathan Sick
"""

import numpy as np
import astropy

from m31hst.draine import spire350_dust_mass_map
from starfisher.dust import SF11ExtinctionCurve

def mw_Av():
    """Build the A_V attenuation by the MW towards M31."""
    curve = SF11ExtinctionCurve()
    ratio = curve['Landolt V']  # A_V / E(B-V) from T6 of SF2011
    return ratio * 0.07  # 0.07 is E(B-V) to M31 from Schlegel 1998

def phat_rel_extinction():
    """Build array of relative extinction in PHAT bands."""
    curve = SF11ExtinctionCurve()
    rel_extinction = np.ones(6, dtype=float)
    rel_extinction[0] = curve.extinction_ratio('WFC3 F275W')
    rel_extinction[1] = curve.extinction_ratio('WFC3 F336W')
    rel_extinction[2] = curve.extinction_ratio('ACS F475W')
    rel_extinction[3] = curve.extinction_ratio('ACS F814W')
    rel_extinction[4] = curve.extinction_ratio('WFC3 F110W')
    rel_extinction[5] = curve.extinction_ratio('WFC3 F160W')
    return rel_extinction

class LewisDustLaw(object):
    """Assign extinction Av+dAv from the Drain et al dust map."""
    def __init__(self):
        super(LewisDustLaw, self).__init__()
        self._f = astropy.io.fits.open(spire350_dust_mass_map())
        self._wcs = astropy.wcs.WCS(self._f[0].header)
        self._naxis1 = self._f[0].header['NAXIS1']
        self._naxis2 = self._f[0].header['NAXIS2']

    def estimate_extinction(self, ra, dec):
        """Estimate the maximum Av estinction to this coordinate."""
        x, y = self.wcs.all_world2pix([ra], [dec], 0)
        assert x > 0 and x < self._naxis1
        assert y > 0 and y < self._naxis2
        x = int(x)
        y = int(y)
        Sigma_dust = self._f[0].data[y, x]
        return 10. ** (-5.4) * Sigma_dust
<|endoftext|>


<commit_before>import sys
import contextlib

RESULT_ERROR = "An error occured"
RESULT_NONE = "No result"

class AnalysisLogEntry(object):
    def __init__(self, message, exc_info=False):
        if exc_info:
            (e_type, value, traceback) = sys.exc_info()
            self.exc_type = e_type
            self.exc_value = value
            self.exc_traceback = traceback
        self.message = message

    def __getstate__(self):
        return str(self.__dict__.get("exc_type")), \
               str(self.__dict__.get("exc_value")), \
               str(self.__dict__.get("exc_traceback"))

    def __setstate__(self, s):
        self.exc_type, self.exc_value, self.exc_traceback = s

class AnalysisMeta(type):
    """
    Whenever Analaysis is subclassed, make sure we know about it.
    """

    def __new__(mcs, name, bases, d):
        t = type.__new__(mcs, name, bases, d)
        if name != 'Analysis':
            chosen_name = d.get('__analysis_name__', name)
            registered_analyses[chosen_name] = (t.__module__, name)
        return t

class Analyses(object):
    """
    This class contains functions for all the registered and runnable analyses,
    """
    def __init__(self, p):
        """
        Creates an Analyses object

        @param p: the angr.Project object
        @param analysis_results: the result cache
        """
        self._p = p
        self._registered_analyses = {}
        self.reload_analyses()

    def reload_analyses(self):
        for analysis_name, (module_name, key) in registered_analyses.iteritems():
            module = reload(sys.modules[module_name])
            analysis = getattr(module, key)
            self._registered_analyses[analysis_name] = self._specialize_analysis(analysis)

    def _specialize_analysis(self, analysis):
        @staticmethod
        def __new__(cls, *args, **kwargs): # pylint: disable=unused-argument
            fail_fast = kwargs.pop('fail_fast', False)

            oself = object.__new__(cls)
            oself.named_errors = {}
            oself.errors = []
            oself.log = []

            oself._fail_fast = fail_fast
            oself._p = self._p

            oself.result = RESULT_NONE
            return oself

        analysis.__new__ = __new__
        return analysis

    def __getstate__(self):
        return self._p

    def __setstate__(self, s):
        self.__init__(s)

    def __getattr__(self, k):
        r = super(Analyses, self).__getattribute__('_registered_analyses')
        if k == '_registered_analyses':
            return r
        if k in r:
            return r[k]
        return super(Analyses, self).__getattribute__(k)

    def __dir__(self):
        return dir(Analyses) + self._registered_analyses.keys()

registered_analyses = {}

class Analysis(object):
    # pylint: disable=no-member
    __metaclass__ = AnalysisMeta
    def post_load(self):
        pass

    @contextlib.contextmanager
    def _resilience(self, name=None, exception=Exception):
        try:
            yield
        except exception:  # pylint:disable=broad-except
            if self._fail_fast:
                raise
            else:
                error = AnalysisLogEntry("exception occurred", exc_info=True)
                if name is None:
                    self.errors.append(error)
                else:
                    self.named_errors[name] = error

    def _log(self, event):
        '''

        :return:
        '''
        # TODO: This function is not properly designed nor implemented!
        le = AnalysisLogEntry(event)
        self.log.append(le)

    def _checkpoint(self):
        pass

    def __repr__(self):
        return '<%s Analysis Result at %#x>' % id(self)
<commit_msg>an idea<commit_after>import sys
import contextlib

RESULT_ERROR = "An error occured"
RESULT_NONE = "No result"

class AnalysisLogEntry(object):
    def __init__(self, message, exc_info=False):
        if exc_info:
            (e_type, value, traceback) = sys.exc_info()
            self.exc_type = e_type
            self.exc_value = value
            self.exc_traceback = traceback
        self.message = message

    def __getstate__(self):
        return str(self.__dict__.get("exc_type")), \
               str(self.__dict__.get("exc_value")), \
               str(self.__dict__.get("exc_traceback"))

    def __setstate__(self, s):
        self.exc_type, self.exc_value, self.exc_traceback = s

class AnalysisMeta(type):
    """
    Whenever Analaysis is subclassed, make sure we know about it.
    """

    def __new__(mcs, name, bases, d):
        t = type.__new__(mcs, name, bases, d)
        if name != 'Analysis':
            chosen_name = d.get('__analysis_name__', name)
            registered_analyses[chosen_name] = (t.__module__, name)
        return t

class Analyses(object):
    """
    This class contains functions for all the registered and runnable analyses,
    """
    def __init__(self, p):
        """
        Creates an Analyses object

        @param p: the angr.Project object
        @param analysis_results: the result cache
        """
        self._p = p
        self._registered_analyses = {}
        self.reload_analyses()

    def reload_analyses(self):
        for analysis_name, (module_name, key) in registered_analyses.iteritems():
            module = reload(sys.modules[module_name])
            analysis = getattr(module, key)
            self._registered_analyses[analysis_name] = self._specialize_analysis(analysis)

    def _specialize_analysis(self, analysis):
        def make_analysis(*args, **kwargs): # pylint: disable=unused-argument
            fail_fast = kwargs.pop('fail_fast', False)

            oself = analisys.__new__(cls)
            oself.named_errors = {}
            oself.errors = []
            oself.log = []

            oself._fail_fast = fail_fast
            oself._p = self._p

            oself.__init__(*args, **kwargs)
            return oself

        make_analysis.__doc__ = analysis.__init__.__doc__
        return make_analysis

    def __getstate__(self):
        return self._p

    def __setstate__(self, s):
        self.__init__(s)

    def __getattr__(self, k):
        r = super(Analyses, self).__getattribute__('_registered_analyses')
        if k == '_registered_analyses':
            return r
        if k in r:
            return r[k]
        return super(Analyses, self).__getattribute__(k)

    def __dir__(self):
        return dir(Analyses) + self._registered_analyses.keys()

registered_analyses = {}

class Analysis(object):
    # pylint: disable=no-member
    __metaclass__ = AnalysisMeta
    def post_load(self):
        pass

    @contextlib.contextmanager
    def _resilience(self, name=None, exception=Exception):
        try:
            yield
        except exception:  # pylint:disable=broad-except
            if self._fail_fast:
                raise
            else:
                error = AnalysisLogEntry("exception occurred", exc_info=True)
                if name is None:
                    self.errors.append(error)
                else:
                    self.named_errors[name] = error

    def _log(self, event):
        '''

        :return:
        '''
        # TODO: This function is not properly designed nor implemented!
        le = AnalysisLogEntry(event)
        self.log.append(le)

    def _checkpoint(self):
        pass

    def __repr__(self):
        return '<%s Analysis Result at %#x>' % id(self)
<|endoftext|>


<commit_before># coding: utf-8
"""
This module provides an implementation of the Amazon MWS Sellers API.
"""

__created__ = "2012-12-03"
__modified__ = "2013-03-28"

import datetime

import amazonmws.mws
from amazonmws.util import datetime_to_iso8601

#: Actions.
ACTIONS = {
	'get_status': 'GetServiceStatus',
	'list_marketplaces': 'ListMarketplaceParticipations',
	'list_marketplaces_next': 'ListMarketplaceParticipationsByNextToken',
}

class MWSSellers(amazonmws.mws.MWS):
	"""
	The ``MWSSellers`` class is used to send requests to the Amazon MWS
	Sellers API. The primary purpose of this class is to allow sellers
	request information about their seller account.
	"""

	client_api_version = __modified__
	"""
	*client_api_version* (``str``) is the version of this client API.
	"""

	sellers_api_version = '2011-07-01'
	"""
	*sellers_api_version* (``str``) is the version of the MWS Sellers API
	implemented.
	"""

	path = "/Sellers/2011-07-01"
	"""
	*path* (``str``) is path all Sellers API requests are sent to.
	"""

	def get_status(self, debug=None):
		"""
		Requests the operational status of the Sellers API.

		Returns the response XML (``str``).
		"""
		args = self.new_args()
		args['Action'] = ACTIONS['get_status']
		return self.send_request(args, path=self.path, debug=debug)

	def list_marketplaces(self, debug=None):
		"""
		Requests the marketplaces the seller can sell in.

		Returns the response XML (``str``).
		"""
		args = self.new_args()
		args['Action'] = ACTIONS['list_marketplaces']
		return self.send_request(args, path=self.path, debug=debug)

	def list_marketplaces_next(self, next_token, debug=None):
		"""
		Requests the next batch of marketplaces the seller can sell in.

		*next_token* (``str``) is the token used to fetch the next batch of
		results.

		Returns the response XML (``str``).
		"""
		if not isinstance(next_token, str):
			raise TypeError("next_token:{!r} is not a str.".format(next_token))
		elif not next_token:
			raise ValueError("next_token:{!r} cannot be empty.".format(next_token))

		args = self.new_args()
		args['Action'] = ACTIONS['list_marketplaces_next']
		args['NextToken'] = next_token
		return self.send_request(args, path=self.path, debug=debug)

	def new_args(self):
		"""
		Returns a new ``dict`` of default arguments.
		"""
		return {
			'AWSAccessKeyId': self.access_key,
			'SellerId': self.merchant_id,
			'Timestamp': datetime_to_iso8601(datetime.datetime.utcnow()),
			'Version': self.sellers_api_version
		}
<commit_msg>Python3 compatibility<commit_after># coding: utf-8
"""
This module provides an implementation of the Amazon MWS Sellers API.
"""

__author__ = "Caleb P. Burns"
__created__ = "2012-12-03"
__modified__ = "2016-03-29"
__modified_by___ = "Joshua D. Burns"

import six # Python2/Python3 compatibility library.
import datetime
import amazonmws.mws
from amazonmws.util import datetime_to_iso8601

#: Actions.
ACTIONS = {
	'get_status': 'GetServiceStatus',
	'list_marketplaces': 'ListMarketplaceParticipations',
	'list_marketplaces_next': 'ListMarketplaceParticipationsByNextToken',
}

class MWSSellers(amazonmws.mws.MWS):
	"""
	The ``MWSSellers`` class is used to send requests to the Amazon MWS
	Sellers API. The primary purpose of this class is to allow sellers
	request information about their seller account.
	"""

	client_api_version = __modified__
	"""
	*client_api_version* (``str``) is the version of this client API.
	"""

	sellers_api_version = '2011-07-01'
	"""
	*sellers_api_version* (``str``) is the version of the MWS Sellers API
	implemented.
	"""

	path = "/Sellers/2011-07-01"
	"""
	*path* (``str``) is path all Sellers API requests are sent to.
	"""

	def get_status(self, debug=None):
		"""
		Requests the operational status of the Sellers API.

		Returns the response XML (``str``).
		"""
		args = self.new_args()
		args['Action'] = ACTIONS['get_status']
		return self.send_request(args, path=self.path, debug=debug)

	def list_marketplaces(self, debug=None):
		"""
		Requests the marketplaces the seller can sell in.

		Returns the response XML (``str``).
		"""
		args = self.new_args()
		args['Action'] = ACTIONS['list_marketplaces']
		return self.send_request(args, path=self.path, debug=debug)

	def list_marketplaces_next(self, next_token, debug=None):
		"""
		Requests the next batch of marketplaces the seller can sell in.

		*next_token* (``str``) is the token used to fetch the next batch of
		results.

		Returns the response XML (``str``).
		"""
		if not isinstance(next_token, six.string_types):
			raise TypeError("next_token:{!r} is not a str.".format(next_token))
		elif not next_token:
			raise ValueError("next_token:{!r} cannot be empty.".format(next_token))

		args = self.new_args()
		args['Action'] = ACTIONS['list_marketplaces_next']
		args['NextToken'] = next_token
		return self.send_request(args, path=self.path, debug=debug)

	def new_args(self):
		"""
		Returns a new ``dict`` of default arguments.
		"""
		return {
			'AWSAccessKeyId': self.access_key,
			'SellerId': self.merchant_id,
			'Timestamp': datetime_to_iso8601(datetime.datetime.utcnow()),
			'Version': self.sellers_api_version
		}
<|endoftext|>


<commit_before>import calendar
import time
from colorama import *
from ratelimit import rate_limited

import requests

def get_epoch():
    '''
    Utility function used to get the current
    epoch time. Required for some calls to
    the remote.

    :return: String representing the current epoch time.
    '''
    return calendar.timegm(time.gmtime())

def enum(**enums):
    '''
    Utility function to create a simple
    enum-like data type. Behind the scenes it
    is just an array.

    :param list enums: A list of key value pairs.
    :return: A simple list.
    '''
    return type('Enum', (), enums)

VARIANTS = enum(PERSONAL=0, SMALL=1, MEDIUM=2, LARGE=3)
PAYMENT_METHODS = enum(CASH_ON_DELIVERY=0, CARD=1, PAYPAL=2)

class Store(object):
    """
    Parses raw data returned from the API and stores it in an easy-to-use object
    :param raw_data: The raw data returned from a Local Store search
    """
    def __init__(self, raw_data):
        local_store = raw_data['localStore']
        self.id = local_store['id']
        self.name = local_store['name']
        self.collection = local_store['isCollectionAvailable']
        self.delivery = raw_data['isDeliveryAvailableFromStore']
        self.menu_version = local_store['menuVersion']

class Item(object):
    """
    Stores necessary information on menu items
    :param raw_data: The raw data from a menu search
    """
    def __init__(self, raw_data, subcategory):
        self.name = raw_data['name'].replace("®", "").replace("™", "")
        self.price = raw_data['price']
        self.id = raw_data['productId']
        self.skus = raw_data['productSkus']
        self.type = subcategory

class Menu(object):
    """
    Stores a list of Item objects retrieved from a Get Menu call
    :param raw_data: The raw data from a Get Menu call
    """
    def __init__(self, raw_data):
        self.categories = []
        self.items = []
        for category in raw_data:
            self.categories.append(category['name'])
            for subcategory in category['subcategories']:
                for item in subcategory['products']:
                    self.items.append(Item(item, subcategory['type']))

    def get_product_by_name(self, name):
        """
        Gets an Item from the Menu by name. Note that the name is not case-sensitive but
        must be spelt correctly. None will be returned if no matching name is found.
        Note that the Trademark Symbols have been removed.
        :param name: The name of the item
        :return: An item object matching the search, or None if none were found
        """

        for item in self.items:
            if item.name.lower() == name.lower():
                return item

        return None

class Client(object):
    '''
    API class for the UK version of Dominos
    pizza website.
    '''

    class ApiError(Exception):
        '''
        API exception class. It is exactly the
        same as a regular exception.
        '''
        pass

    BASE_URL = 'https://www.dominos.co.uk'

    def __init__(self):
        init()

        self.session = requests.session()

        self.reset_session()
        self.reset_store()

    @rate_limited(1)
    def reset_session(self):
        '''
        Clear out the current session on the remote
        and setup a new one.

        :return: A response object.
        '''
        response = self.session.get(self.__url('/Home/SessionExpire'))

        if response.status_code != 200:
            raise self.ApiError('Failed to clear session: {}'.format(response.status_code))

        self.session = requests.session()
        self.session.headers.update({'content-type': 'application/json; charset=utf-8'})

        return response

    @rate_limited(1)
    def reset_store(self):
        '''
        Clears out the current store and gets a cookie.
        Set the cross site request forgery token for
        each subsequent request.

        :return: A response object.
        '''
        response = self.session.get(self.__url('/Store/Reset'))

        if response.status_code != 200:
            raise self.ApiError('Cannot get cookie: {}'.format(response.status_code))

        self.session.headers.update({'X-XSRF-TOKEN': self.session.cookies['XSRF-TOKEN']})

        return response

    @rate_limited(1)
    def get_stores(self, search):
        '''
        Search for dominos pizza stores using a search
        term.

        :param search: Search term.
        :return: A response object.
        '''
        params = {'search': search}
        response = self.session.get(self.__url('/storefindermap/storenamesearch'), params=params)

        if response.status_code != 200:
            raise self.ApiError('Cannot fetch stores: {}'.format(response.status_code))

        return response

    @rate_limited(1)
    def get_nearest_store(self, postcode):
        '''
        Search for domino pizza stores using a postcode.
        This will only search for local stores indicating
        delivery status and payment details.

        :param postcode: A postcode.
        :return: A Store object.
        '''
        params = {'SearchText': postcode}
        response = self.session.get(self.__url('/storefindermap/storesearch'), params=params)

        if response.status_code != 200:
            raise self.ApiError('Cannot fetch nearest store for {}: {}'.format(postcode, response.status_code))

        return Store(response.json())

    @rate_limited(1)
    def set_delivery_system(self, idx, postcode):
        '''
        Set local cookies by initialising the delivery
        system on the remote. Requires a store ID and
        a delivery postcode.

        :param idx: Store id.
        :param postcode: A postcode.
        :return: A response object.
        '''
        params = {'fulfilmentmethod': 'delivery', 'postcode': postcode, 'storeid': idx}
        response = self.session.post(self.__url('/Journey/Initialize'), json=params)

        if response.status_code != 200:
            raise self.ApiError('Cannot create cookie: {}'.format(response.status_code))

        return response

    @rate_limited(1)
    def get_menu(self, store):
        '''
        Retrieve the menu from the selected store.

        :param store: The store to retrieve menu from.
        :return: A Menu object.
        '''

        params = {
            "collectionOnly": store.delivery,
            "menuVersion": store.menu_version,
            "storeId": store.id,
        }
        response = self.session.get(self.__url('/ProductCatalog/GetStoreCatalog'), params=params)

        if response.status_code != 200:
            raise self.ApiError('Cannot get menu: {}'.format(response.status_code))

        return Menu(response.json())

    @rate_limited(1)
    def get_basket(self):
        '''
        Retrieve the basket for the current session.

        :return: A response object.
        '''
        response = self.session.get(self.__url('/CheckoutBasket/GetBasket'))

        if response.status_code != 200:
            raise self.ApiError('Cannot get basket: {}'.format(response.status_code))

        return response

    def add_item_to_basket(self, item, variant, quantity=1):
        '''
        Add an item to the current basket.

        :param item: Item from menu.
        :param variant: Item SKU id.
        :param quantity: The quantity of item to be added
        :return: A response object, or None if an item type is not recognised.
        '''

        item_type = item.type

        if item_type == 'Pizza':
            return self.add_pizza_to_basket(item, variant, quantity)
        elif item_type == 'Side':
            return self.add_side_to_basket(item, variant, quantity)
        return None

    @rate_limited(1, 5)
    def add_pizza_to_basket(self, item, variant=VARIANTS.MEDIUM, quantity=1):
        '''
        Add a pizza to the current basket.

        :param item: Item from menu.
        :param variant: Item SKU id. Some defaults are defined in the VARIANTS enum.
        :param quantity: The quantity of pizzas to be added
        :return: A response object.
        '''

        item_variant = item.skus[variant]
        ingredients = item_variant['ingredients'].update([36, 42])

        params = {'stepId': 0, 'quantity': quantity, 'sizeId': variant, 'productId': item.id, 'ingredients': ingredients, 'productIdHalfTwo': 0, 'ingredientsHalfTwo': [], 'recipeReferrer': 0}
        response = self.session.post(self.__url('/Basket/AddPizza/'), json=params)

        if response.status_code != 200:
            raise self.ApiError('Cannot add {} {} to basket: {}'.format(item_variant['name'], item['name'], response.status_code))

        return response

    @rate_limited(1, 5)
    def add_side_to_basket(self, item, variant=VARIANTS.PERSONAL, quantity=1):
        '''
        Add a side to the current basket.

        :param item: Item from menu.
        :param variant: Item SKU id. Some defaults are defined in the VARIANTS enum.
        :param quantity: The quantity of sides to be added
        :return: A response object.
        '''

        item_variant = item.skus[variant]

        params = {'productSkuId': item_variant['productSkuId'], 'quantity': quantity, 'ComplimentaryItems': []}
        response = self.session.post(self.__url('/Basket/AddProduct'), json=params)

        if response.status_code != 200:
            raise self.ApiError('Cannot add {} {} to basket: {}'.format(item_variant['name'], item['name'], response.status_code))

        return response

    @rate_limited(1, 5)
    def remove_item_from_basket(self, item):
        '''
        Remove an item from the current basket.

        :param item: Basket item object.
        :return: A response object.
        '''
        params = {'basketItemId': item.id, 'wizardItemDelete': False}
        response = self.session.post(self.__url('/Basket/RemoveBasketItem'), params=params)

        if response.status_code != 200:
            raise self.ApiError('Cannot remove {} from basket: {}'.format(item.name, response.status_code))

        return response

    @rate_limited(1)
    def get_payment_options(self):
        '''
        Retrieve a series of payment options
        alongside the card url used to
        authorise payments through mastercard
        datacash service.

        There is no guarantee the card url will
        be accepted as mastercard will likely
        reject the origin of the request.

        :return: A response object.
        '''
        response = self.session.post(self.__url('/PaymentOptions/GetPaymentDetailsData'))

        if response.status_code != 200:
            raise self.ApiError('Cannot get payment details: {}'.format(response.status_code))

        return response

    @rate_limited(1)
    def set_payment_method(self, method=PAYMENT_METHODS.PAYPAL):
        '''
        Select the payment method going to be
        used to make a purchase.

        :param method: Payment method id.
        :return: A response object.
        '''
        params = {'paymentMethod': method}
        response = self.session.post(self.__url('/PaymentOptions/SetPaymentMethod'), json=params)

        if response.status_code != 200:
            raise self.ApiError('Cannot set payment method: {}'.format(response.status_code))

        return response

    @rate_limited(1)
    def process_payment(self):
        '''
        Proceed with payment using the payment
        method selected earlier.

        :return: A response object.
        '''
        params = {'__RequestVerificationToken': self.session.cookies, 'method': 'submit'}
        response = self.session.post(self.__url('/PaymentOptions/Proceed'), json=params)

        if response.status_code != 200:
            raise self.ApiError('Cannot process payment: {}'.format(response.status_code))

        return response

    def __url(self, path):
        '''
        Helper method to generate fully qualified URIs
        pertaining to specific API actions.

        :param path: Relative API path to resource.
        :return: Fully qualified URI to API resource.
        '''
        return self.BASE_URL + path
<commit_msg>Removed Colorama Import<commit_after>import calendar
import time
from ratelimit import rate_limited

import requests

def get_epoch():
    '''
    Utility function used to get the current
    epoch time. Required for some calls to
    the remote.

    :return: String representing the current epoch time.
    '''
    return calendar.timegm(time.gmtime())

def enum(**enums):
    '''
    Utility function to create a simple
    enum-like data type. Behind the scenes it
    is just an array.

    :param list enums: A list of key value pairs.
    :return: A simple list.
    '''
    return type('Enum', (), enums)

VARIANTS = enum(PERSONAL=0, SMALL=1, MEDIUM=2, LARGE=3)
PAYMENT_METHODS = enum(CASH_ON_DELIVERY=0, CARD=1, PAYPAL=2)

class Store(object):
    """
    Parses raw data returned from the API and stores it in an easy-to-use object
    :param raw_data: The raw data returned from a Local Store search
    """
    def __init__(self, raw_data):
        local_store = raw_data['localStore']
        self.id = local_store['id']
        self.name = local_store['name']
        self.collection = local_store['isCollectionAvailable']
        self.delivery = raw_data['isDeliveryAvailableFromStore']
        self.menu_version = local_store['menuVersion']

class Item(object):
    """
    Stores necessary information on menu items
    :param raw_data: The raw data from a menu search
    """
    def __init__(self, raw_data, subcategory):
        self.name = raw_data['name'].replace("®", "").replace("™", "")
        self.price = raw_data['price']
        self.id = raw_data['productId']
        self.skus = raw_data['productSkus']
        self.type = subcategory

class Menu(object):
    """
    Stores a list of Item objects retrieved from a Get Menu call
    :param raw_data: The raw data from a Get Menu call
    """
    def __init__(self, raw_data):
        self.categories = []
        self.items = []
        for category in raw_data:
            self.categories.append(category['name'])
            for subcategory in category['subcategories']:
                for item in subcategory['products']:
                    self.items.append(Item(item, subcategory['type']))

    def get_product_by_name(self, name):
        """
        Gets an Item from the Menu by name. Note that the name is not case-sensitive but
        must be spelt correctly. None will be returned if no matching name is found.
        Note that the Trademark Symbols have been removed.
        :param name: The name of the item
        :return: An item object matching the search, or None if none were found
        """

        for item in self.items:
            if item.name.lower() == name.lower():
                return item

        return None

class Client(object):
    '''
    API class for the UK version of Dominos
    pizza website.
    '''

    class ApiError(Exception):
        '''
        API exception class. It is exactly the
        same as a regular exception.
        '''
        pass

    BASE_URL = 'https://www.dominos.co.uk'

    def __init__(self):
        init()

        self.session = requests.session()

        self.reset_session()
        self.reset_store()

    @rate_limited(1)
    def reset_session(self):
        '''
        Clear out the current session on the remote
        and setup a new one.

        :return: A response object.
        '''
        response = self.session.get(self.__url('/Home/SessionExpire'))

        if response.status_code != 200:
            raise self.ApiError('Failed to clear session: {}'.format(response.status_code))

        self.session = requests.session()
        self.session.headers.update({'content-type': 'application/json; charset=utf-8'})

        return response

    @rate_limited(1)
    def reset_store(self):
        '''
        Clears out the current store and gets a cookie.
        Set the cross site request forgery token for
        each subsequent request.

        :return: A response object.
        '''
        response = self.session.get(self.__url('/Store/Reset'))

        if response.status_code != 200:
            raise self.ApiError('Cannot get cookie: {}'.format(response.status_code))

        self.session.headers.update({'X-XSRF-TOKEN': self.session.cookies['XSRF-TOKEN']})

        return response

    @rate_limited(1)
    def get_stores(self, search):
        '''
        Search for dominos pizza stores using a search
        term.

        :param search: Search term.
        :return: A response object.
        '''
        params = {'search': search}
        response = self.session.get(self.__url('/storefindermap/storenamesearch'), params=params)

        if response.status_code != 200:
            raise self.ApiError('Cannot fetch stores: {}'.format(response.status_code))

        return response

    @rate_limited(1)
    def get_nearest_store(self, postcode):
        '''
        Search for domino pizza stores using a postcode.
        This will only search for local stores indicating
        delivery status and payment details.

        :param postcode: A postcode.
        :return: A Store object.
        '''
        params = {'SearchText': postcode}
        response = self.session.get(self.__url('/storefindermap/storesearch'), params=params)

        if response.status_code != 200:
            raise self.ApiError('Cannot fetch nearest store for {}: {}'.format(postcode, response.status_code))

        return Store(response.json())

    @rate_limited(1)
    def set_delivery_system(self, idx, postcode):
        '''
        Set local cookies by initialising the delivery
        system on the remote. Requires a store ID and
        a delivery postcode.

        :param idx: Store id.
        :param postcode: A postcode.
        :return: A response object.
        '''
        params = {'fulfilmentmethod': 'delivery', 'postcode': postcode, 'storeid': idx}
        response = self.session.post(self.__url('/Journey/Initialize'), json=params)

        if response.status_code != 200:
            raise self.ApiError('Cannot create cookie: {}'.format(response.status_code))

        return response

    @rate_limited(1)
    def get_menu(self, store):
        '''
        Retrieve the menu from the selected store.

        :param store: The store to retrieve menu from.
        :return: A Menu object.
        '''

        params = {
            "collectionOnly": store.delivery,
            "menuVersion": store.menu_version,
            "storeId": store.id,
        }
        response = self.session.get(self.__url('/ProductCatalog/GetStoreCatalog'), params=params)

        if response.status_code != 200:
            raise self.ApiError('Cannot get menu: {}'.format(response.status_code))

        return Menu(response.json())

    @rate_limited(1)
    def get_basket(self):
        '''
        Retrieve the basket for the current session.

        :return: A response object.
        '''
        response = self.session.get(self.__url('/CheckoutBasket/GetBasket'))

        if response.status_code != 200:
            raise self.ApiError('Cannot get basket: {}'.format(response.status_code))

        return response

    def add_item_to_basket(self, item, variant, quantity=1):
        '''
        Add an item to the current basket.

        :param item: Item from menu.
        :param variant: Item SKU id.
        :param quantity: The quantity of item to be added
        :return: A response object, or None if an item type is not recognised.
        '''

        item_type = item.type

        if item_type == 'Pizza':
            return self.add_pizza_to_basket(item, variant, quantity)
        elif item_type == 'Side':
            return self.add_side_to_basket(item, variant, quantity)
        return None

    @rate_limited(1, 5)
    def add_pizza_to_basket(self, item, variant=VARIANTS.MEDIUM, quantity=1):
        '''
        Add a pizza to the current basket.

        :param item: Item from menu.
        :param variant: Item SKU id. Some defaults are defined in the VARIANTS enum.
        :param quantity: The quantity of pizzas to be added
        :return: A response object.
        '''

        item_variant = item.skus[variant]
        ingredients = item_variant['ingredients'].update([36, 42])

        params = {'stepId': 0, 'quantity': quantity, 'sizeId': variant, 'productId': item.id, 'ingredients': ingredients, 'productIdHalfTwo': 0, 'ingredientsHalfTwo': [], 'recipeReferrer': 0}
        response = self.session.post(self.__url('/Basket/AddPizza/'), json=params)

        if response.status_code != 200:
            raise self.ApiError('Cannot add {} {} to basket: {}'.format(item_variant['name'], item['name'], response.status_code))

        return response

    @rate_limited(1, 5)
    def add_side_to_basket(self, item, variant=VARIANTS.PERSONAL, quantity=1):
        '''
        Add a side to the current basket.

        :param item: Item from menu.
        :param variant: Item SKU id. Some defaults are defined in the VARIANTS enum.
        :param quantity: The quantity of sides to be added
        :return: A response object.
        '''

        item_variant = item.skus[variant]

        params = {'productSkuId': item_variant['productSkuId'], 'quantity': quantity, 'ComplimentaryItems': []}
        response = self.session.post(self.__url('/Basket/AddProduct'), json=params)

        if response.status_code != 200:
            raise self.ApiError('Cannot add {} {} to basket: {}'.format(item_variant['name'], item['name'], response.status_code))

        return response

    @rate_limited(1, 5)
    def remove_item_from_basket(self, item):
        '''
        Remove an item from the current basket.

        :param item: Basket item object.
        :return: A response object.
        '''
        params = {'basketItemId': item.id, 'wizardItemDelete': False}
        response = self.session.post(self.__url('/Basket/RemoveBasketItem'), params=params)

        if response.status_code != 200:
            raise self.ApiError('Cannot remove {} from basket: {}'.format(item.name, response.status_code))

        return response

    @rate_limited(1)
    def get_payment_options(self):
        '''
        Retrieve a series of payment options
        alongside the card url used to
        authorise payments through mastercard
        datacash service.

        There is no guarantee the card url will
        be accepted as mastercard will likely
        reject the origin of the request.

        :return: A response object.
        '''
        response = self.session.post(self.__url('/PaymentOptions/GetPaymentDetailsData'))

        if response.status_code != 200:
            raise self.ApiError('Cannot get payment details: {}'.format(response.status_code))

        return response

    @rate_limited(1)
    def set_payment_method(self, method=PAYMENT_METHODS.PAYPAL):
        '''
        Select the payment method going to be
        used to make a purchase.

        :param method: Payment method id.
        :return: A response object.
        '''
        params = {'paymentMethod': method}
        response = self.session.post(self.__url('/PaymentOptions/SetPaymentMethod'), json=params)

        if response.status_code != 200:
            raise self.ApiError('Cannot set payment method: {}'.format(response.status_code))

        return response

    @rate_limited(1)
    def process_payment(self):
        '''
        Proceed with payment using the payment
        method selected earlier.

        :return: A response object.
        '''
        params = {'__RequestVerificationToken': self.session.cookies, 'method': 'submit'}
        response = self.session.post(self.__url('/PaymentOptions/Proceed'), json=params)

        if response.status_code != 200:
            raise self.ApiError('Cannot process payment: {}'.format(response.status_code))

        return response

    def __url(self, path):
        '''
        Helper method to generate fully qualified URIs
        pertaining to specific API actions.

        :param path: Relative API path to resource.
        :return: Fully qualified URI to API resource.
        '''
        return self.BASE_URL + path
<|endoftext|>


<commit_before>import tensorflow as tf
import numpy as np
import hyperparameters as hp
from action import Action

def weight_variable(shape, name):
  initial = tf.truncated_normal(shape, stddev=0.1)
  return tf.Variable(initial, name=name)

def bias_variable(shape, name):
  initial = tf.constant(0.1, shape=shape)
  return tf.Variable(initial, name=name)

def conv2d(x, W, s):
  return tf.nn.conv2d(x, W, strides=[1, s, s, 1], padding="VALID")

def output_size(in_size, filter_size, stride):
	return (in_size - filter_size)/stride + 1

conv1_out_size = output_size(hp.INPUT_SIZE, hp.CONV_1_SIZE, hp.CONV_1_STRIDE)
conv2_out_size = output_size(conv1_out_size, hp.CONV_2_SIZE, hp.CONV_2_STRIDE)
conv3_out_size = output_size(conv2_out_size, hp.CONV_3_SIZE, hp.CONV_3_STRIDE)
conv3_out_len = hp.CONV_3_DEPTH * (conv3_out_size ** 2)

with tf.name_scope("input"):
	s = tf.placeholder(tf.float32, shape=[
		None,
		hp.INPUT_SIZE,
		hp.INPUT_SIZE,
		hp.AGENT_HISTORY_LENGTH
	], name="s")

	sp = tf.placeholder(tf.float32, shape=[
		None,
		hp.INPUT_SIZE,
		hp.INPUT_SIZE,
		hp.AGENT_HISTORY_LENGTH
	], name="sp")

# this corresponds to Q
with tf.name_scope("regular_net"):
	r_W_conv1 = weight_variable([
		hp.CONV_1_SIZE,
		hp.CONV_1_SIZE,
		hp.AGENT_HISTORY_LENGTH,
		hp.CONV_1_DEPTH
	], name="r_W_conv1")
	r_b_conv1 = bias_variable([hp.CONV_1_DEPTH], name="r_b_conv1")

	r_W_conv2 = weight_variable([
		hp.CONV_2_SIZE,
		hp.CONV_2_SIZE,
		hp.CONV_1_DEPTH,
		hp.CONV_2_DEPTH
	], name="r_W_conv2")
	r_b_conv2 = bias_variable([hp.CONV_2_DEPTH], name="r_b_conv2")

	r_W_conv3 = weight_variable([
		hp.CONV_3_SIZE,
		hp.CONV_3_SIZE,
		hp.CONV_2_DEPTH,
		hp.CONV_3_DEPTH
	], name="r_W_conv3")
	r_b_conv3 = bias_variable([hp.CONV_3_DEPTH], name="r_b_conv3")

	r_W_fc1 = weight_variable([conv3_out_len, hp.FC_SIZE], name="r_W_fc1")
	r_b_fc1 = bias_variable([hp.FC_SIZE], name="r_b_fc1")

	r_W_out = weight_variable([hp.FC_SIZE, Action.num_actions], name="r_W_out")
	r_b_out = bias_variable([Action.num_actions], name="b_out")

	r_h_conv1 = tf.nn.relu(conv2d(s, r_W_conv1, hp.CONV_1_STRIDE) + r_b_conv1)
	r_h_conv2 = tf.nn.relu(conv2d(r_h_conv1, r_W_conv2, hp.CONV_2_STRIDE) + r_b_conv2)
	r_h_conv3 = tf.nn.relu(conv2d(r_h_conv2, r_W_conv3, hp.CONV_3_STRIDE) + r_b_conv3)
	r_resized = tf.reshape(r_h_conv3, [-1, conv3_out_len], name="r_resized")
	r_h_fc1 = tf.nn.relu(tf.matmul(r_resized, r_W_fc1) + r_b_fc1)

	# minibatch_size x # of actions
	# position (i,j) in r_actions is score for action j in minibatch item i
	r_actions = tf.matmul(r_h_fc1, r_W_out) + r_b_out
	best_action = tf.argmax(r_actions, 1)

# this corresponds to Q_hat
with tf.name_scope("target_params"):
	t_W_conv1 = tf.Variable(r_W_conv1.initialized_value(), trainable=False)
	t_W_conv2 = tf.Variable(r_W_conv2.initialized_value(), trainable=False)
	t_W_conv3 = tf.Variable(r_W_conv3.initialized_value(), trainable=False)
	t_W_fc1 = tf.Variable(r_W_fc1.initialized_value(), trainable=False)
	t_W_out = tf.Variable(r_W_out.initialized_value(), trainable=False)
	t_b_conv1 = tf.Variable(r_b_conv1.initialized_value(), trainable=False)
	t_b_conv2 = tf.Variable(r_b_conv2.initialized_value(), trainable=False)
	t_b_conv3 = tf.Variable(r_b_conv3.initialized_value(), trainable=False)
	t_b_fc1 = tf.Variable(r_b_fc1.initialized_value(), trainable=False)
	t_b_out = tf.Variable(r_b_out.initialized_value(), trainable=False)

	t_h_conv1 = tf.nn.relu(conv2d(sp, t_W_conv1, hp.CONV_1_STRIDE) + t_b_conv1)
	t_h_conv2 = tf.nn.relu(conv2d(t_h_conv1, t_W_conv2, hp.CONV_2_STRIDE) + t_b_conv2)
	t_h_conv3 = tf.nn.relu(conv2d(t_h_conv2, t_W_conv3, hp.CONV_3_STRIDE) + t_b_conv3)
	t_resized = tf.reshape(t_h_conv3, [-1, conv3_out_len], name="t_resized")
	t_h_fc1 = tf.nn.relu(tf.matmul(t_resized, t_W_fc1) + t_b_fc1)
	t_actions = tf.matmul(t_h_fc1, t_W_out) + t_b_out

# if you call update_target, it copies Q to Q_hat
with tf.name_scope("updaters"):
	regular = [
		r_W_conv1,
		r_W_conv2,
		r_W_conv3,
		r_W_fc1,
		r_W_out,
		r_b_conv1,
		r_b_conv2,
		r_b_conv3,
		r_b_fc1,
		r_b_out,
	]

	target = [
		t_W_conv1,
		t_W_conv2,
		t_W_conv3,
		t_W_fc1,
		t_W_out,
		t_b_conv1,
		t_b_conv2,
		t_b_conv3,
		t_b_fc1,
		t_b_out,
	]

	all_vars = regular + target

	target_update = []

	for i in range(len(regular)):
		t, r = target[i], regular[i]
		# generate the graph and assign nodes from one to the other
		target_update.append(tf.assign(t, r))

	# tf.group executes all the nodes which are passed to it
	update_target = tf.group(*target_update)

with tf.name_scope("loss"):
	actions = tf.placeholder(tf.int32, hp.MINIBATCH_SIZE)
	rewards = tf.placeholder(tf.float32, hp.MINIBATCH_SIZE)

	# array of zeros/ones
	terminals = tf.placeholder(tf.int32, hp.MINIBATCH_SIZE)
	gamma = tf.constant(hp.DISCOUNT_FACTOR, dtype=tf.float32)

	# ys is a vector of size minibatch_size which corresponds to y_j in algorithm 1
	ys = rewards + tf.to_float(1 - terminals) * gamma * tf.reduce_max(t_actions, reduction_indices=1)

	r_actions_flat = tf.reshape(r_actions, [-1])
	# Q(s)[a]
	# vector of size minibatch, each element is the Q value of taking the action that we took
	gathered = tf.gather(r_actions_flat, hp.MINIBATCH_SIZE * actions)
	loss = tf.reduce_mean(tf.square(ys - gathered))

	# actually perform one gradient descent step
	optimizer = tf.train.RMSPropOptimizer(hp.LEARNING_RATE, hp.GRADIENT_MOMENTUM, momentum=hp.SQUARED_GRADIENT_MOMENTUM, epsilon=hp.MIN_SQUARED_GRADIENT)
	minimize_loss = optimizer.minimize(loss, var_list=regular)

<commit_msg>fixed rmsprop hyperparams<commit_after>import tensorflow as tf
import numpy as np
import hyperparameters as hp
from action import Action

def weight_variable(shape, name):
  initial = tf.truncated_normal(shape, stddev=0.1)
  return tf.Variable(initial, name=name)

def bias_variable(shape, name):
  initial = tf.constant(0.1, shape=shape)
  return tf.Variable(initial, name=name)

def conv2d(x, W, s):
  return tf.nn.conv2d(x, W, strides=[1, s, s, 1], padding="VALID")

def output_size(in_size, filter_size, stride):
	return (in_size - filter_size)/stride + 1

conv1_out_size = output_size(hp.INPUT_SIZE, hp.CONV_1_SIZE, hp.CONV_1_STRIDE)
conv2_out_size = output_size(conv1_out_size, hp.CONV_2_SIZE, hp.CONV_2_STRIDE)
conv3_out_size = output_size(conv2_out_size, hp.CONV_3_SIZE, hp.CONV_3_STRIDE)
conv3_out_len = hp.CONV_3_DEPTH * (conv3_out_size ** 2)

with tf.name_scope("input"):
	s = tf.placeholder(tf.float32, shape=[
		None,
		hp.INPUT_SIZE,
		hp.INPUT_SIZE,
		hp.AGENT_HISTORY_LENGTH
	], name="s")

	sp = tf.placeholder(tf.float32, shape=[
		None,
		hp.INPUT_SIZE,
		hp.INPUT_SIZE,
		hp.AGENT_HISTORY_LENGTH
	], name="sp")

# this corresponds to Q
with tf.name_scope("regular_net"):
	r_W_conv1 = weight_variable([
		hp.CONV_1_SIZE,
		hp.CONV_1_SIZE,
		hp.AGENT_HISTORY_LENGTH,
		hp.CONV_1_DEPTH
	], name="r_W_conv1")
	r_b_conv1 = bias_variable([hp.CONV_1_DEPTH], name="r_b_conv1")

	r_W_conv2 = weight_variable([
		hp.CONV_2_SIZE,
		hp.CONV_2_SIZE,
		hp.CONV_1_DEPTH,
		hp.CONV_2_DEPTH
	], name="r_W_conv2")
	r_b_conv2 = bias_variable([hp.CONV_2_DEPTH], name="r_b_conv2")

	r_W_conv3 = weight_variable([
		hp.CONV_3_SIZE,
		hp.CONV_3_SIZE,
		hp.CONV_2_DEPTH,
		hp.CONV_3_DEPTH
	], name="r_W_conv3")
	r_b_conv3 = bias_variable([hp.CONV_3_DEPTH], name="r_b_conv3")

	r_W_fc1 = weight_variable([conv3_out_len, hp.FC_SIZE], name="r_W_fc1")
	r_b_fc1 = bias_variable([hp.FC_SIZE], name="r_b_fc1")

	r_W_out = weight_variable([hp.FC_SIZE, Action.num_actions], name="r_W_out")
	r_b_out = bias_variable([Action.num_actions], name="b_out")

	r_h_conv1 = tf.nn.relu(conv2d(s, r_W_conv1, hp.CONV_1_STRIDE) + r_b_conv1)
	r_h_conv2 = tf.nn.relu(conv2d(r_h_conv1, r_W_conv2, hp.CONV_2_STRIDE) + r_b_conv2)
	r_h_conv3 = tf.nn.relu(conv2d(r_h_conv2, r_W_conv3, hp.CONV_3_STRIDE) + r_b_conv3)
	r_resized = tf.reshape(r_h_conv3, [-1, conv3_out_len], name="r_resized")
	r_h_fc1 = tf.nn.relu(tf.matmul(r_resized, r_W_fc1) + r_b_fc1)

	# minibatch_size x # of actions
	# position (i,j) in r_actions is score for action j in minibatch item i
	r_actions = tf.matmul(r_h_fc1, r_W_out) + r_b_out
	best_action = tf.argmax(r_actions, 1)

# this corresponds to Q_hat
with tf.name_scope("target_params"):
	t_W_conv1 = tf.Variable(r_W_conv1.initialized_value(), trainable=False)
	t_W_conv2 = tf.Variable(r_W_conv2.initialized_value(), trainable=False)
	t_W_conv3 = tf.Variable(r_W_conv3.initialized_value(), trainable=False)
	t_W_fc1 = tf.Variable(r_W_fc1.initialized_value(), trainable=False)
	t_W_out = tf.Variable(r_W_out.initialized_value(), trainable=False)
	t_b_conv1 = tf.Variable(r_b_conv1.initialized_value(), trainable=False)
	t_b_conv2 = tf.Variable(r_b_conv2.initialized_value(), trainable=False)
	t_b_conv3 = tf.Variable(r_b_conv3.initialized_value(), trainable=False)
	t_b_fc1 = tf.Variable(r_b_fc1.initialized_value(), trainable=False)
	t_b_out = tf.Variable(r_b_out.initialized_value(), trainable=False)

	t_h_conv1 = tf.nn.relu(conv2d(sp, t_W_conv1, hp.CONV_1_STRIDE) + t_b_conv1)
	t_h_conv2 = tf.nn.relu(conv2d(t_h_conv1, t_W_conv2, hp.CONV_2_STRIDE) + t_b_conv2)
	t_h_conv3 = tf.nn.relu(conv2d(t_h_conv2, t_W_conv3, hp.CONV_3_STRIDE) + t_b_conv3)
	t_resized = tf.reshape(t_h_conv3, [-1, conv3_out_len], name="t_resized")
	t_h_fc1 = tf.nn.relu(tf.matmul(t_resized, t_W_fc1) + t_b_fc1)
	t_actions = tf.matmul(t_h_fc1, t_W_out) + t_b_out

# if you call update_target, it copies Q to Q_hat
with tf.name_scope("updaters"):
	regular = [
		r_W_conv1,
		r_W_conv2,
		r_W_conv3,
		r_W_fc1,
		r_W_out,
		r_b_conv1,
		r_b_conv2,
		r_b_conv3,
		r_b_fc1,
		r_b_out,
	]

	target = [
		t_W_conv1,
		t_W_conv2,
		t_W_conv3,
		t_W_fc1,
		t_W_out,
		t_b_conv1,
		t_b_conv2,
		t_b_conv3,
		t_b_fc1,
		t_b_out,
	]

	all_vars = regular + target

	target_update = []

	for i in range(len(regular)):
		t, r = target[i], regular[i]
		# generate the graph and assign nodes from one to the other
		target_update.append(tf.assign(t, r))

	# tf.group executes all the nodes which are passed to it
	update_target = tf.group(*target_update)

with tf.name_scope("loss"):
	actions = tf.placeholder(tf.int32, hp.MINIBATCH_SIZE)
	rewards = tf.placeholder(tf.float32, hp.MINIBATCH_SIZE)

	# array of zeros/ones
	terminals = tf.placeholder(tf.int32, hp.MINIBATCH_SIZE)
	gamma = tf.constant(hp.DISCOUNT_FACTOR, dtype=tf.float32)

	# ys is a vector of size minibatch_size which corresponds to y_j in algorithm 1
	ys = rewards + tf.to_float(1 - terminals) * gamma * tf.reduce_max(t_actions, reduction_indices=1)

	r_actions_flat = tf.reshape(r_actions, [-1])
	# Q(s)[a]
	# vector of size minibatch, each element is the Q value of taking the action that we took
	gathered = tf.gather(r_actions_flat, hp.MINIBATCH_SIZE * actions)
	loss = tf.reduce_mean(tf.square(ys - gathered))

	# actually perform one gradient descent step
	optimizer = tf.train.RMSPropOptimizer(hp.LEARNING_RATE, momentum=hp.GRADIENT_MOMENTUM)
	minimize_loss = optimizer.minimize(loss, var_list=regular)

<|endoftext|>