<commit_before># coding=utf-8
# Copyright 2018 Microsoft Authors and the HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest

from transformers import DebertaV2Config, is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor

if is_torch_available():
    import torch

    from transformers import (
        DebertaV2ForMaskedLM,
        DebertaV2ForMultipleChoice,
        DebertaV2ForQuestionAnswering,
        DebertaV2ForSequenceClassification,
        DebertaV2ForTokenClassification,
        DebertaV2Model,
    )
    from transformers.models.deberta_v2.modeling_deberta_v2 import DEBERTA_V2_PRETRAINED_MODEL_ARCHIVE_LIST

class DebertaV2ModelTester(object):
    def __init__(
        self,
        parent,
        batch_size=13,
        seq_length=7,
        is_training=True,
        use_input_mask=True,
        use_token_type_ids=True,
        use_labels=True,
        vocab_size=99,
        hidden_size=32,
        num_hidden_layers=5,
        num_attention_heads=4,
        intermediate_size=37,
        hidden_act="gelu",
        hidden_dropout_prob=0.1,
        attention_probs_dropout_prob=0.1,
        max_position_embeddings=512,
        type_vocab_size=16,
        type_sequence_label_size=2,
        initializer_range=0.02,
        relative_attention=False,
        position_biased_input=True,
        pos_att_type="None",
        num_labels=3,
        num_choices=4,
        scope=None,
    ):
        self.parent = parent
        self.batch_size = batch_size
        self.seq_length = seq_length
        self.is_training = is_training
        self.use_input_mask = use_input_mask
        self.use_token_type_ids = use_token_type_ids
        self.use_labels = use_labels
        self.vocab_size = vocab_size
        self.hidden_size = hidden_size
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.intermediate_size = intermediate_size
        self.hidden_act = hidden_act
        self.hidden_dropout_prob = hidden_dropout_prob
        self.attention_probs_dropout_prob = attention_probs_dropout_prob
        self.max_position_embeddings = max_position_embeddings
        self.type_vocab_size = type_vocab_size
        self.type_sequence_label_size = type_sequence_label_size
        self.initializer_range = initializer_range
        self.num_labels = num_labels
        self.num_choices = num_choices
        self.relative_attention = relative_attention
        self.position_biased_input = position_biased_input
        self.pos_att_type = pos_att_type
        self.scope = scope

    def prepare_config_and_inputs(self):
        input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size)

        input_mask = None
        if self.use_input_mask:
            input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2)

        token_type_ids = None
        if self.use_token_type_ids:
            token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size)

        sequence_labels = None
        token_labels = None
        choice_labels = None
        if self.use_labels:
            sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size)
            token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels)
            choice_labels = ids_tensor([self.batch_size], self.num_choices)

        config = self.get_config()

        return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels

    def get_config(self):
        return DebertaV2Config(
            vocab_size=self.vocab_size,
            hidden_size=self.hidden_size,
            num_hidden_layers=self.num_hidden_layers,
            num_attention_heads=self.num_attention_heads,
            intermediate_size=self.intermediate_size,
            hidden_act=self.hidden_act,
            hidden_dropout_prob=self.hidden_dropout_prob,
            attention_probs_dropout_prob=self.attention_probs_dropout_prob,
            max_position_embeddings=self.max_position_embeddings,
            type_vocab_size=self.type_vocab_size,
            initializer_range=self.initializer_range,
            relative_attention=self.relative_attention,
            position_biased_input=self.position_biased_input,
            pos_att_type=self.pos_att_type,
        )

    def check_loss_output(self, result):
        self.parent.assertListEqual(list(result.loss.size()), [])

    def create_and_check_deberta_model(
        self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
    ):
        model = DebertaV2Model(config=config)
        model.to(torch_device)
        model.eval()
        sequence_output = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids)[0]
        sequence_output = model(input_ids, token_type_ids=token_type_ids)[0]
        sequence_output = model(input_ids)[0]

        self.parent.assertListEqual(list(sequence_output.size()), [self.batch_size, self.seq_length, self.hidden_size])

    def create_and_check_deberta_for_masked_lm(
        self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
    ):
        model = DebertaV2ForMaskedLM(config=config)
        model.to(torch_device)
        model.eval()
        result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=token_labels)

        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.vocab_size))

    def create_and_check_deberta_for_sequence_classification(
        self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
    ):
        config.num_labels = self.num_labels
        model = DebertaV2ForSequenceClassification(config)
        model.to(torch_device)
        model.eval()
        result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=sequence_labels)
        self.parent.assertListEqual(list(result.logits.size()), [self.batch_size, self.num_labels])
        self.check_loss_output(result)

    def create_and_check_deberta_for_token_classification(
        self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
    ):
        config.num_labels = self.num_labels
        model = DebertaV2ForTokenClassification(config=config)
        model.to(torch_device)
        model.eval()
        result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=token_labels)
        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.num_labels))

    def create_and_check_deberta_for_question_answering(
        self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
    ):
        model = DebertaV2ForQuestionAnswering(config=config)
        model.to(torch_device)
        model.eval()
        result = model(
            input_ids,
            attention_mask=input_mask,
            token_type_ids=token_type_ids,
            start_positions=sequence_labels,
            end_positions=sequence_labels,
        )
        self.parent.assertEqual(result.start_logits.shape, (self.batch_size, self.seq_length))
        self.parent.assertEqual(result.end_logits.shape, (self.batch_size, self.seq_length))

    def create_and_check_deberta_for_multiple_choice(
        self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
    ):
        model = DebertaV2ForMultipleChoice(config=config)
        model.to(torch_device)
        model.eval()
        multiple_choice_inputs_ids = input_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous()
        multiple_choice_token_type_ids = token_type_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous()
        multiple_choice_input_mask = input_mask.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous()
        result = model(
            multiple_choice_inputs_ids,
            attention_mask=multiple_choice_input_mask,
            token_type_ids=multiple_choice_token_type_ids,
            labels=choice_labels,
        )
        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.num_choices))

    def prepare_config_and_inputs_for_common(self):
        config_and_inputs = self.prepare_config_and_inputs()
        (
            config,
            input_ids,
            token_type_ids,
            input_mask,
            sequence_labels,
            token_labels,
            choice_labels,
        ) = config_and_inputs
        inputs_dict = {"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
        return config, inputs_dict

@require_torch
class DebertaV2ModelTest(ModelTesterMixin, unittest.TestCase):

    all_model_classes = (
        (
            DebertaV2Model,
            DebertaV2ForMaskedLM,
            DebertaV2ForSequenceClassification,
            DebertaV2ForTokenClassification,
            DebertaV2ForQuestionAnswering,
            DebertaV2ForMultipleChoice,
        )
        if is_torch_available()
        else ()
    )

    fx_compatible = True
    test_torchscript = False
    test_pruning = False
    test_head_masking = False
    is_encoder_decoder = False

    def setUp(self):
        self.model_tester = DebertaV2ModelTester(self)
        self.config_tester = ConfigTester(self, config_class=DebertaV2Config, hidden_size=37)

    def test_config(self):
        self.config_tester.run_common_tests()

    def test_deberta_model(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_deberta_model(*config_and_inputs)

    def test_for_sequence_classification(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_deberta_for_sequence_classification(*config_and_inputs)

    def test_for_masked_lm(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_deberta_for_masked_lm(*config_and_inputs)

    def test_for_question_answering(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_deberta_for_question_answering(*config_and_inputs)

    def test_for_token_classification(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_deberta_for_token_classification(*config_and_inputs)

    def test_for_multiple_choice(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_deberta_for_multiple_choice(*config_and_inputs)

    @slow
    def test_model_from_pretrained(self):
        for model_name in DEBERTA_V2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            model = DebertaV2Model.from_pretrained(model_name)
            self.assertIsNotNone(model)

@require_torch
@require_sentencepiece
@require_tokenizers
class DebertaV2ModelIntegrationTest(unittest.TestCase):
    @unittest.skip(reason="Model not available yet")
    def test_inference_masked_lm(self):
        pass

    @slow
    def test_inference_no_head(self):
        model = DebertaV2Model.from_pretrained("microsoft/deberta-v2-xlarge")

        input_ids = torch.tensor([[0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]])
        attention_mask = torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])
        output = model(input_ids, attention_mask=attention_mask)[0]
        # compare the actual values for a slice.
        expected_slice = torch.tensor(
            [[[0.2356, 0.1948, 0.0369], [-0.1063, 0.3586, -0.5152], [-0.6399, -0.0259, -0.2525]]]
        )
        self.assertTrue(torch.allclose(output[:, 1:4, 1:4], expected_slice, atol=1e-4), f"{output[:, 1:4, 1:4]}")
<commit_msg>wrap forward passes with torch.no_grad() (#19278)<commit_after># coding=utf-8
# Copyright 2018 Microsoft Authors and the HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest

from transformers import DebertaV2Config, is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor

if is_torch_available():
    import torch

    from transformers import (
        DebertaV2ForMaskedLM,
        DebertaV2ForMultipleChoice,
        DebertaV2ForQuestionAnswering,
        DebertaV2ForSequenceClassification,
        DebertaV2ForTokenClassification,
        DebertaV2Model,
    )
    from transformers.models.deberta_v2.modeling_deberta_v2 import DEBERTA_V2_PRETRAINED_MODEL_ARCHIVE_LIST

class DebertaV2ModelTester(object):
    def __init__(
        self,
        parent,
        batch_size=13,
        seq_length=7,
        is_training=True,
        use_input_mask=True,
        use_token_type_ids=True,
        use_labels=True,
        vocab_size=99,
        hidden_size=32,
        num_hidden_layers=5,
        num_attention_heads=4,
        intermediate_size=37,
        hidden_act="gelu",
        hidden_dropout_prob=0.1,
        attention_probs_dropout_prob=0.1,
        max_position_embeddings=512,
        type_vocab_size=16,
        type_sequence_label_size=2,
        initializer_range=0.02,
        relative_attention=False,
        position_biased_input=True,
        pos_att_type="None",
        num_labels=3,
        num_choices=4,
        scope=None,
    ):
        self.parent = parent
        self.batch_size = batch_size
        self.seq_length = seq_length
        self.is_training = is_training
        self.use_input_mask = use_input_mask
        self.use_token_type_ids = use_token_type_ids
        self.use_labels = use_labels
        self.vocab_size = vocab_size
        self.hidden_size = hidden_size
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.intermediate_size = intermediate_size
        self.hidden_act = hidden_act
        self.hidden_dropout_prob = hidden_dropout_prob
        self.attention_probs_dropout_prob = attention_probs_dropout_prob
        self.max_position_embeddings = max_position_embeddings
        self.type_vocab_size = type_vocab_size
        self.type_sequence_label_size = type_sequence_label_size
        self.initializer_range = initializer_range
        self.num_labels = num_labels
        self.num_choices = num_choices
        self.relative_attention = relative_attention
        self.position_biased_input = position_biased_input
        self.pos_att_type = pos_att_type
        self.scope = scope

    def prepare_config_and_inputs(self):
        input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size)

        input_mask = None
        if self.use_input_mask:
            input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2)

        token_type_ids = None
        if self.use_token_type_ids:
            token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size)

        sequence_labels = None
        token_labels = None
        choice_labels = None
        if self.use_labels:
            sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size)
            token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels)
            choice_labels = ids_tensor([self.batch_size], self.num_choices)

        config = self.get_config()

        return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels

    def get_config(self):
        return DebertaV2Config(
            vocab_size=self.vocab_size,
            hidden_size=self.hidden_size,
            num_hidden_layers=self.num_hidden_layers,
            num_attention_heads=self.num_attention_heads,
            intermediate_size=self.intermediate_size,
            hidden_act=self.hidden_act,
            hidden_dropout_prob=self.hidden_dropout_prob,
            attention_probs_dropout_prob=self.attention_probs_dropout_prob,
            max_position_embeddings=self.max_position_embeddings,
            type_vocab_size=self.type_vocab_size,
            initializer_range=self.initializer_range,
            relative_attention=self.relative_attention,
            position_biased_input=self.position_biased_input,
            pos_att_type=self.pos_att_type,
        )

    def check_loss_output(self, result):
        self.parent.assertListEqual(list(result.loss.size()), [])

    def create_and_check_deberta_model(
        self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
    ):
        model = DebertaV2Model(config=config)
        model.to(torch_device)
        model.eval()
        sequence_output = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids)[0]
        sequence_output = model(input_ids, token_type_ids=token_type_ids)[0]
        sequence_output = model(input_ids)[0]

        self.parent.assertListEqual(list(sequence_output.size()), [self.batch_size, self.seq_length, self.hidden_size])

    def create_and_check_deberta_for_masked_lm(
        self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
    ):
        model = DebertaV2ForMaskedLM(config=config)
        model.to(torch_device)
        model.eval()
        result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=token_labels)

        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.vocab_size))

    def create_and_check_deberta_for_sequence_classification(
        self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
    ):
        config.num_labels = self.num_labels
        model = DebertaV2ForSequenceClassification(config)
        model.to(torch_device)
        model.eval()
        result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=sequence_labels)
        self.parent.assertListEqual(list(result.logits.size()), [self.batch_size, self.num_labels])
        self.check_loss_output(result)

    def create_and_check_deberta_for_token_classification(
        self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
    ):
        config.num_labels = self.num_labels
        model = DebertaV2ForTokenClassification(config=config)
        model.to(torch_device)
        model.eval()
        result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=token_labels)
        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.num_labels))

    def create_and_check_deberta_for_question_answering(
        self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
    ):
        model = DebertaV2ForQuestionAnswering(config=config)
        model.to(torch_device)
        model.eval()
        result = model(
            input_ids,
            attention_mask=input_mask,
            token_type_ids=token_type_ids,
            start_positions=sequence_labels,
            end_positions=sequence_labels,
        )
        self.parent.assertEqual(result.start_logits.shape, (self.batch_size, self.seq_length))
        self.parent.assertEqual(result.end_logits.shape, (self.batch_size, self.seq_length))

    def create_and_check_deberta_for_multiple_choice(
        self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
    ):
        model = DebertaV2ForMultipleChoice(config=config)
        model.to(torch_device)
        model.eval()
        multiple_choice_inputs_ids = input_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous()
        multiple_choice_token_type_ids = token_type_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous()
        multiple_choice_input_mask = input_mask.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous()
        result = model(
            multiple_choice_inputs_ids,
            attention_mask=multiple_choice_input_mask,
            token_type_ids=multiple_choice_token_type_ids,
            labels=choice_labels,
        )
        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.num_choices))

    def prepare_config_and_inputs_for_common(self):
        config_and_inputs = self.prepare_config_and_inputs()
        (
            config,
            input_ids,
            token_type_ids,
            input_mask,
            sequence_labels,
            token_labels,
            choice_labels,
        ) = config_and_inputs
        inputs_dict = {"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
        return config, inputs_dict

@require_torch
class DebertaV2ModelTest(ModelTesterMixin, unittest.TestCase):

    all_model_classes = (
        (
            DebertaV2Model,
            DebertaV2ForMaskedLM,
            DebertaV2ForSequenceClassification,
            DebertaV2ForTokenClassification,
            DebertaV2ForQuestionAnswering,
            DebertaV2ForMultipleChoice,
        )
        if is_torch_available()
        else ()
    )

    fx_compatible = True
    test_torchscript = False
    test_pruning = False
    test_head_masking = False
    is_encoder_decoder = False

    def setUp(self):
        self.model_tester = DebertaV2ModelTester(self)
        self.config_tester = ConfigTester(self, config_class=DebertaV2Config, hidden_size=37)

    def test_config(self):
        self.config_tester.run_common_tests()

    def test_deberta_model(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_deberta_model(*config_and_inputs)

    def test_for_sequence_classification(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_deberta_for_sequence_classification(*config_and_inputs)

    def test_for_masked_lm(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_deberta_for_masked_lm(*config_and_inputs)

    def test_for_question_answering(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_deberta_for_question_answering(*config_and_inputs)

    def test_for_token_classification(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_deberta_for_token_classification(*config_and_inputs)

    def test_for_multiple_choice(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_deberta_for_multiple_choice(*config_and_inputs)

    @slow
    def test_model_from_pretrained(self):
        for model_name in DEBERTA_V2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            model = DebertaV2Model.from_pretrained(model_name)
            self.assertIsNotNone(model)

@require_torch
@require_sentencepiece
@require_tokenizers
class DebertaV2ModelIntegrationTest(unittest.TestCase):
    @unittest.skip(reason="Model not available yet")
    def test_inference_masked_lm(self):
        pass

    @slow
    def test_inference_no_head(self):
        model = DebertaV2Model.from_pretrained("microsoft/deberta-v2-xlarge")

        input_ids = torch.tensor([[0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]])
        attention_mask = torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])
        with torch.no_grad():
            output = model(input_ids, attention_mask=attention_mask)[0]
        # compare the actual values for a slice.
        expected_slice = torch.tensor(
            [[[0.2356, 0.1948, 0.0369], [-0.1063, 0.3586, -0.5152], [-0.6399, -0.0259, -0.2525]]]
        )
        self.assertTrue(torch.allclose(output[:, 1:4, 1:4], expected_slice, atol=1e-4), f"{output[:, 1:4, 1:4]}")
<|endoftext|>


<commit_before>import PIL
from PIL import ImageEnhance, Image, ImageFilter, ImageChops
import numpy as np
randint = np.random.randint

__author__ = "Ronny Restrepo"
__copyright__ = "Copyright 2017, Ronny Restrepo"
__credits__ = ["Ronny Restrepo"]
__license__ = "Apache License"
__version__ = "2.0"

# ==============================================================================
#                                                                      PIL2ARRAY
# ==============================================================================
def pil2array(im):
    """ Given a PIL image it returns a numpy array representation """
    return np.asarray(im, dtype=np.uint8)

# ==============================================================================
#                                                                      ARRAY2PIL
# ==============================================================================
def array2pil(a, mode="RGB"):
    """Given a numpy array containing image information returns a PIL image"""
    if mode.lower() in ["grey", "gray", "l"]:
        mode="L"
    return Image.fromarray(a, mode=mode)

# ==============================================================================
#                                                                    RANDOM_CROP
# ==============================================================================
def random_crop(im, min_scale=0.5, preserve_size=False, resample=PIL.Image.NEAREST):
    """
    Args:
        im:             (PIL image)
        min_scale:      (float) minimum ratio along each dimension to crop from.
        preserve_size:  (bool) Should it resize to original image dimensions?
        resample:       Resampling method during rescale.

    Returns:
        PIL image randomly cropped from `im`.
    """
    if min_scale == 0:
        return im
    else:
        width, height = np.array(np.shape(im)[:2])
        crop_width = np.random.randint(width*min_scale, width)
        crop_height = np.random.randint(height*min_scale, height)
        x_offset = np.random.randint(0, width - crop_width + 1)
        y_offset = np.random.randint(0, height - crop_height + 1)
        im2 = im.crop((x_offset, y_offset,
                       x_offset + crop_width,
                       y_offset + crop_height))
        if preserve_size:
            im2 = im2.resize(im.size, resample=resample)
        return im2

# ==============================================================================
#                                                         CROP_AND_PRESERVE_SIZE
# ==============================================================================
def crop_and_preserve_size(im, crop_dims, offset, resample=PIL.Image.NEAREST):
    """ Given a PIL image, the dimensions of the crop, and the offset of
        the crop, it crops the image, and resizes it back to the original
        dimensions.

    Args:
        im:         (PIL image)
        crop_dims:  Dimensions of the crop region [width, height]
        offset:     Position of the crop box from Top Left corner [x, y]
        resample:   resamplimg method
    """
    crop_width, crop_height = crop_dims
    x_offset, y_offset = offset
    im2 = im.crop((x_offset, y_offset,
                   x_offset + crop_width,
                   y_offset + crop_height))
    im2 = im2.resize(im.size, resample=resample)
    return im2

# ==============================================================================
#                                                                   RANDOM_SHIFT
# ==============================================================================
def random_shift(im, max=(5,5)):
    """ Randomly shifts an image.

    Args:
        im: (pil image)
        max: (tuple of two ints) max amount in each x y direction.
    """
    x_offset = np.random.randint(0, max[0])
    y_offset = np.random.randint(0, max[1])
    return ImageChops.offset(im, xoffset=x_offset, yoffset=y_offset)

# ==============================================================================
#                                                                    SHIFT_IMAGE
# ==============================================================================
def shift_image(im, shift):
    """ Shifts an image.

    Args:
        im
        shift:  [x,y]
    """
    return ImageChops.offset(im, xoffset=shift[0], yoffset=shift[1])

# ==============================================================================
#                                                              RANDOM_BRIGHTNESS
# ==============================================================================
def random_brightness(im, sd=0.5, min=0, max=20):
    """Creates a new image which randomly adjusts the brightness of `im` by
       randomly sampling a brightness value centered at 1, with a standard
       deviation of `sd` from a normal distribution. Clips values to a
       desired min and max range.

    Args:
        im:   PIL image
        sd:   (float) Standard deviation used for sampling brightness value.
        min:  (int or float) Clip contrast value to be no lower than this.
        max:  (int or float) Clip contrast value to be no higher than this.

    Returns:
        PIL image with brightness randomly adjusted.
    """
    brightness = np.clip(np.random.normal(loc=1, scale=sd), min, max)
    enhancer = ImageEnhance.Brightness(im)
    return enhancer.enhance(brightness)

# ==============================================================================
#                                                                RANDOM_CONTRAST
# ==============================================================================
def random_contrast(im, sd=0.5, min=0, max=10):
    """Creates a new image which randomly adjusts the contrast of `im` by
       randomly sampling a contrast value centered at 1, with a standard
       deviation of `sd` from a normal distribution. Clips values to a
       desired min and max range.

    Args:
        im:   PIL image
        sd:   (float) Standard deviation used for sampling contrast value.
        min:  (int or float) Clip contrast value to be no lower than this.
        max:  (int or float) Clip contrast value to be no higher than this.

    Returns:
        PIL image with contrast randomly adjusted.
    """
    contrast = np.clip(np.random.normal(loc=1, scale=sd), min, max)
    enhancer = ImageEnhance.Contrast(im)
    return enhancer.enhance(contrast)

# ==============================================================================
#                                                                    RANDOM_BLUR
# ==============================================================================
def random_blur(im, min=0, max=5):
    """ Creates a new image which applies a random amount of Gaussian Blur, with
        a blur radius that is randomly chosen to be in the range [min, max]
        inclusive.

    Args:
        im:   PIL image
        min:  (int) Min amount of blur desired.
        max:  (int) Max amount of blur desired.

    Returns:
        PIL image with random amount of blur applied.
    """
    blur_radius = randint(min, max+1)
    if blur_radius == 0:
        return im
    else:
        return im.filter(ImageFilter.GaussianBlur(radius=blur_radius))

# ==============================================================================
#                                                                   RANDOM_NOISE
# ==============================================================================
def random_noise(im, sd=5):
    """Creates a new image which has random noise.
       The intensity of the noise is determined by first randomly choosing the
       standard deviation of the noise as a value between 0 to `sd`.
       This value is then used as the standard deviation for randomly sampling
       individual pixel noise from a normal distribution.
       This random noise is added to the original image pixel values, and
       clipped to keep all values between 0-255.

    Args:
        im:   PIL image
        sd:   (int) Max Standard Deviation to select from.

    Returns:
        PIL image with random noise added.
    """
    mode = im.mode
    noise_sd = np.random.randint(0, sd)
    if noise_sd > 0:
        noise = np.random.normal(loc=0, scale=noise_sd, size=np.shape(im))
        im2 = np.clip(im + noise, 0, 255).astype(np.uint8)
        return array2pil(im2, mode=mode)
    else:
        return im

# ==============================================================================
#                                                                RANDOM_ROTATION
# ==============================================================================
def random_rotation(im, max=10, expand=True):
    """ Creates a new image which is rotated by a random amount between
        [-max, +max] inclusive.

    Args:
        im:     PIL image
        max:    (int) Max angle (in either direction).
        expand: (bool) expand image to prevent rotated edges being visible.

    Returns:
        PIL image with random rotation applied.
    """
    angle = randint(-max, max+1)
    if angle == 0:
        return im
    else:
        return im.rotate(angle, resample=Image.BILINEAR, expand=expand)

# ==============================================================================
#                                   RANDOM_TRANSFORMATIONS_FOR_SEGMENTATION_DATA
# ==============================================================================
def random_transformations_for_segmentation_data(X, Y, brightness=True, contrast=True, blur=3, crop=0.5, noise=10):
    """ Takes a batch of input images `X`, segmentation labels `Y` as arrays,
        and does random image transormations on them.

        Ensures that any tansformations that shift or scale the input images
        also have the same transormations applied to the label images.

    NOTE:  Assumes the pixels for input images are in the range of 0-255.

    Args:
        X:          (numpy array) batch of imput images
        Y:          (numpy array) batch of segmentation labels
        brightness: (bool)(default=True)
                    Apply random brightness?
        contrast:   (bool)(default=True)
                    Apply random contrast?
        blur:       (int or None)(default=3)
                    Amount of gaussian blur to apply (in pixels).
        crop:       (float or None)(default=0.5)
                    Smallest crop to take as a ratio of the dimensions of the
                    original image.
                    Eg, `crop=0.5` could crop a region of the image ranging
                    anywhere from 50x50 - 100x100 for an input image of 100x100.
        noise:      (int or None)(default=10)
                    Standard deviation (as a pixel intensity value) to use
                    for random normal noise to apply to each pixel.
                    (gets clipped to keep pixel values between 0-255)
    """
    images = np.zeros_like(X)
    labels = np.zeros_like(Y)
    n_images = len(images)
    for i in range(n_images):
        image = array2pil(X[i], mode="RGB")
        label = array2pil(Y[i], mode="L")

        if brightness:
            image = random_brightness(image, sd=0.5, min=0.2, max=4)
        if contrast:
            image = random_contrast(image, sd=0.5, min=0.2, max=5)
        if crop is not None:
            min_scale = crop
            width, height = np.array(np.shape(image)[:2])
            crop_width = np.random.randint(width*min_scale, width)
            crop_height = np.random.randint(height*min_scale, height)
            x_offset = np.random.randint(0, width - crop_width + 1)
            y_offset = np.random.randint(0, height - crop_height + 1)
            image = crop_and_preserve_size(image, crop_dims=[crop_width, crop_height], offset=[x_offset, y_offset])
            label = crop_and_preserve_size(label, crop_dims=[crop_width, crop_height], offset=[x_offset, y_offset])
        if blur is not None:
            image = random_blur(image, 0, blur)

        if noise is not None:
            image = random_noise(image, sd=noise)

        # Put into array
        images[i] = pil2array(image)
        labels[i] = pil2array(label)
    return images, labels
<commit_msg>FEAT: ML: Add max_scale argument to  random_crop()<commit_after>import PIL
from PIL import ImageEnhance, Image, ImageFilter, ImageChops
import numpy as np
randint = np.random.randint

__author__ = "Ronny Restrepo"
__copyright__ = "Copyright 2017, Ronny Restrepo"
__credits__ = ["Ronny Restrepo"]
__license__ = "Apache License"
__version__ = "2.0"

# ==============================================================================
#                                                                      PIL2ARRAY
# ==============================================================================
def pil2array(im):
    """ Given a PIL image it returns a numpy array representation """
    return np.asarray(im, dtype=np.uint8)

# ==============================================================================
#                                                                      ARRAY2PIL
# ==============================================================================
def array2pil(a, mode="RGB"):
    """Given a numpy array containing image information returns a PIL image"""
    if mode.lower() in ["grey", "gray", "l"]:
        mode="L"
    return Image.fromarray(a, mode=mode)

# ==============================================================================
#                                                                    RANDOM_CROP
# ==============================================================================
def random_crop(im, min_scale=0.5, max_scale=1.0, preserve_size=False, resample=PIL.Image.NEAREST):
    """
    Args:
        im:         PIL image
        min_scale:   (float) minimum ratio along each dimension to crop from.
        max_scale:   (float) maximum ratio along each dimension to crop from.
        preserve_size: (bool) Should it resize back to original dims?
        resample:       resampling method during rescale.

    Returns:
        PIL image of size crop_size, randomly cropped from `im`.
    """
    assert (min_scale < max_scale), "min_scale MUST be smaller than max_scale"
    width, height = im.size
    crop_width = np.random.randint(width*min_scale, width*max_scale)
    crop_height = np.random.randint(height*min_scale, height*max_scale)
    x_offset = np.random.randint(0, width - crop_width + 1)
    y_offset = np.random.randint(0, height - crop_height + 1)
    im2 = im.crop((x_offset, y_offset,
                   x_offset + crop_width,
                   y_offset + crop_height))
    if preserve_size:
        im2 = im2.resize(im.size, resample=resample)
    return im2

# ==============================================================================
#                                                         CROP_AND_PRESERVE_SIZE
# ==============================================================================
def crop_and_preserve_size(im, crop_dims, offset, resample=PIL.Image.NEAREST):
    """ Given a PIL image, the dimensions of the crop, and the offset of
        the crop, it crops the image, and resizes it back to the original
        dimensions.

    Args:
        im:         (PIL image)
        crop_dims:  Dimensions of the crop region [width, height]
        offset:     Position of the crop box from Top Left corner [x, y]
        resample:   resamplimg method
    """
    crop_width, crop_height = crop_dims
    x_offset, y_offset = offset
    im2 = im.crop((x_offset, y_offset,
                   x_offset + crop_width,
                   y_offset + crop_height))
    im2 = im2.resize(im.size, resample=resample)
    return im2

# ==============================================================================
#                                                                   RANDOM_SHIFT
# ==============================================================================
def random_shift(im, max=(5,5)):
    """ Randomly shifts an image.

    Args:
        im: (pil image)
        max: (tuple of two ints) max amount in each x y direction.
    """
    x_offset = np.random.randint(0, max[0])
    y_offset = np.random.randint(0, max[1])
    return ImageChops.offset(im, xoffset=x_offset, yoffset=y_offset)

# ==============================================================================
#                                                                    SHIFT_IMAGE
# ==============================================================================
def shift_image(im, shift):
    """ Shifts an image.

    Args:
        im
        shift:  [x,y]
    """
    return ImageChops.offset(im, xoffset=shift[0], yoffset=shift[1])

# ==============================================================================
#                                                              RANDOM_BRIGHTNESS
# ==============================================================================
def random_brightness(im, sd=0.5, min=0, max=20):
    """Creates a new image which randomly adjusts the brightness of `im` by
       randomly sampling a brightness value centered at 1, with a standard
       deviation of `sd` from a normal distribution. Clips values to a
       desired min and max range.

    Args:
        im:   PIL image
        sd:   (float) Standard deviation used for sampling brightness value.
        min:  (int or float) Clip contrast value to be no lower than this.
        max:  (int or float) Clip contrast value to be no higher than this.

    Returns:
        PIL image with brightness randomly adjusted.
    """
    brightness = np.clip(np.random.normal(loc=1, scale=sd), min, max)
    enhancer = ImageEnhance.Brightness(im)
    return enhancer.enhance(brightness)

# ==============================================================================
#                                                                RANDOM_CONTRAST
# ==============================================================================
def random_contrast(im, sd=0.5, min=0, max=10):
    """Creates a new image which randomly adjusts the contrast of `im` by
       randomly sampling a contrast value centered at 1, with a standard
       deviation of `sd` from a normal distribution. Clips values to a
       desired min and max range.

    Args:
        im:   PIL image
        sd:   (float) Standard deviation used for sampling contrast value.
        min:  (int or float) Clip contrast value to be no lower than this.
        max:  (int or float) Clip contrast value to be no higher than this.

    Returns:
        PIL image with contrast randomly adjusted.
    """
    contrast = np.clip(np.random.normal(loc=1, scale=sd), min, max)
    enhancer = ImageEnhance.Contrast(im)
    return enhancer.enhance(contrast)

# ==============================================================================
#                                                                    RANDOM_BLUR
# ==============================================================================
def random_blur(im, min=0, max=5):
    """ Creates a new image which applies a random amount of Gaussian Blur, with
        a blur radius that is randomly chosen to be in the range [min, max]
        inclusive.

    Args:
        im:   PIL image
        min:  (int) Min amount of blur desired.
        max:  (int) Max amount of blur desired.

    Returns:
        PIL image with random amount of blur applied.
    """
    blur_radius = randint(min, max+1)
    if blur_radius == 0:
        return im
    else:
        return im.filter(ImageFilter.GaussianBlur(radius=blur_radius))

# ==============================================================================
#                                                                   RANDOM_NOISE
# ==============================================================================
def random_noise(im, sd=5):
    """Creates a new image which has random noise.
       The intensity of the noise is determined by first randomly choosing the
       standard deviation of the noise as a value between 0 to `sd`.
       This value is then used as the standard deviation for randomly sampling
       individual pixel noise from a normal distribution.
       This random noise is added to the original image pixel values, and
       clipped to keep all values between 0-255.

    Args:
        im:   PIL image
        sd:   (int) Max Standard Deviation to select from.

    Returns:
        PIL image with random noise added.
    """
    mode = im.mode
    noise_sd = np.random.randint(0, sd)
    if noise_sd > 0:
        noise = np.random.normal(loc=0, scale=noise_sd, size=np.shape(im))
        im2 = np.clip(im + noise, 0, 255).astype(np.uint8)
        return array2pil(im2, mode=mode)
    else:
        return im

# ==============================================================================
#                                                                RANDOM_ROTATION
# ==============================================================================
def random_rotation(im, max=10, expand=True):
    """ Creates a new image which is rotated by a random amount between
        [-max, +max] inclusive.

    Args:
        im:     PIL image
        max:    (int) Max angle (in either direction).
        expand: (bool) expand image to prevent rotated edges being visible.

    Returns:
        PIL image with random rotation applied.
    """
    angle = randint(-max, max+1)
    if angle == 0:
        return im
    else:
        return im.rotate(angle, resample=Image.BILINEAR, expand=expand)

# ==============================================================================
#                                   RANDOM_TRANSFORMATIONS_FOR_SEGMENTATION_DATA
# ==============================================================================
def random_transformations_for_segmentation_data(X, Y, brightness=True, contrast=True, blur=3, crop=0.5, noise=10):
    """ Takes a batch of input images `X`, segmentation labels `Y` as arrays,
        and does random image transormations on them.

        Ensures that any tansformations that shift or scale the input images
        also have the same transormations applied to the label images.

    NOTE:  Assumes the pixels for input images are in the range of 0-255.

    Args:
        X:          (numpy array) batch of imput images
        Y:          (numpy array) batch of segmentation labels
        brightness: (bool)(default=True)
                    Apply random brightness?
        contrast:   (bool)(default=True)
                    Apply random contrast?
        blur:       (int or None)(default=3)
                    Amount of gaussian blur to apply (in pixels).
        crop:       (float or None)(default=0.5)
                    Smallest crop to take as a ratio of the dimensions of the
                    original image.
                    Eg, `crop=0.5` could crop a region of the image ranging
                    anywhere from 50x50 - 100x100 for an input image of 100x100.
        noise:      (int or None)(default=10)
                    Standard deviation (as a pixel intensity value) to use
                    for random normal noise to apply to each pixel.
                    (gets clipped to keep pixel values between 0-255)
    """
    images = np.zeros_like(X)
    labels = np.zeros_like(Y)
    n_images = len(images)
    for i in range(n_images):
        image = array2pil(X[i], mode="RGB")
        label = array2pil(Y[i], mode="L")

        if brightness:
            image = random_brightness(image, sd=0.5, min=0.2, max=4)
        if contrast:
            image = random_contrast(image, sd=0.5, min=0.2, max=5)
        if crop is not None:
            min_scale = crop
            width, height = np.array(np.shape(image)[:2])
            crop_width = np.random.randint(width*min_scale, width)
            crop_height = np.random.randint(height*min_scale, height)
            x_offset = np.random.randint(0, width - crop_width + 1)
            y_offset = np.random.randint(0, height - crop_height + 1)
            image = crop_and_preserve_size(image, crop_dims=[crop_width, crop_height], offset=[x_offset, y_offset])
            label = crop_and_preserve_size(label, crop_dims=[crop_width, crop_height], offset=[x_offset, y_offset])
        if blur is not None:
            image = random_blur(image, 0, blur)

        if noise is not None:
            image = random_noise(image, sd=noise)

        # Put into array
        images[i] = pil2array(image)
        labels[i] = pil2array(label)
    return images, labels
<|endoftext|>


<commit_before>import subprocess
from nose.tools import *
from hamcrest import *
import sys
from performanceplatform.collector.crontab import remove_existing_crontab_for_app

from tests.performanceplatform.collector.tools import temp_file
from performanceplatform.collector import crontab
from mock import patch

class TestRemoveExistingCrontab(object):

    def setUp(self):
        self.patcher = patch('socket.gethostname')
        self.mock_gethostname = self.patcher.start()
        self.mock_gethostname.return_value = 'nonnumerichostname'

    def test_jobs_are_not_removed_for_other_apps(self):
        crontab = [
            '# Begin performanceplatform.collector jobs for my-app',
            'foobar',
            '# End performanceplatform.collector jobs for my-app',
            '# Begin performanceplatform.collector jobs for other-app',
            'barfoo',
            '# End performanceplatform.collector jobs for other-app',
        ]
        new_crontab = remove_existing_crontab_for_app(crontab, 'my-app')
        assert_that(new_crontab,
                    has_item(contains_string('barfoo')))
        assert_that(new_crontab,
                    is_not(has_item(contains_string('foobar'))))

    def test_crontab_end_before_begin(self):
        crontab = [
            '# End performanceplatform.collector jobs for my-app',
            'foobar',
            '# Begin performanceplatform.collector jobs for my-app',
            'other'
        ]
        new_crontab = remove_existing_crontab_for_app(crontab, 'my-app')
        assert_that(new_crontab,
                    has_item(contains_string('foobar')))
        assert_that(new_crontab,
                    is_not(has_item(contains_string('other'))))

    def test_crontab_begin_with_no_end(self):
        crontab = [
            'foobar',
            '# Begin performanceplatform.collector jobs for my-app',
            'other',
        ]
        new_crontab = remove_existing_crontab_for_app(crontab, 'my-app')
        assert_that(new_crontab,
                    has_item(contains_string('foobar')))
        assert_that(new_crontab,
                    is_not(has_item(contains_string('other'))))

    def tearDown(self):
        self.patcher.stop()

class TestGenerateCrontab(object):

    def setUp(self):
        self.patcher = patch('socket.gethostname')
        self.mock_gethostname = self.patcher.start()
        self.mock_gethostname.return_value = 'nonnumerichostname'

    def test_other_cronjobs_are_preserved(self):
        with temp_file("") as jobs_path:
            generated_jobs = crontab.generate_crontab(
                ['other cronjob'],
                jobs_path,
                "/path/to/app",
                "some_id")
            assert_that(generated_jobs, has_item("other cronjob"))

    def test_some_cronjobs_are_added_between_containing_comments(self):
        with temp_file("schedule,query.json,creds.json,token.json,performanceplatform.json") as jobs_path:
            generated_jobs = crontab.generate_crontab(
                [],
                jobs_path,
                "/path/to/my-app",
                'some_id')
            assert_that(generated_jobs,
                        has_item('# Begin performanceplatform.collector jobs for some_id'))

            assert_that(generated_jobs,
                        has_item(contains_string("schedule")))
            assert_that(generated_jobs,
                        has_item(
                            contains_string("-q /path/to/my-app/config/query.json")))
            assert_that(generated_jobs,
                        has_item(
                            contains_string("-c /path/to/my-app/config/creds.json")))
            assert_that(generated_jobs,
                        has_item(
                            contains_string("-t /path/to/my-app/config/token.json")))
            assert_that(generated_jobs,
                        has_item(
                            contains_string("-b /path/to/my-app/config/performanceplatform.json")))

            assert_that(generated_jobs,
                        has_item('# End performanceplatform.collector jobs for some_id'))

    def test_added_jobs_run_the_crontab_module(self):
        with temp_file("schedule,query.json,creds.json,token.json,performanceplatform.json") as jobs_path:
            generated_jobs = crontab.generate_crontab(
                [],
                jobs_path,
                "/path/to/my-app",
                "some_id")
            assert_that(generated_jobs,
                        has_item(
                            contains_string("pp-collector")))

    def test_existing_pp_cronjobs_are_purged(self):
        with temp_file("schedule,query.json,creds.json,token.json,performanceplatform.json") as jobs_path:
            generated_jobs = crontab.generate_crontab(
                [
                    '# Begin performanceplatform.collector jobs for some_id',
                    'other job',
                    '# End performanceplatform.collector jobs for some_id'
                ],
                jobs_path,
                "/path/to/my-app",
                'some_id')
            assert_that(generated_jobs,
                        is_not(has_item(contains_string("other job"))))

    def test_invalid_jobs_file_causes_failure(self):
        with temp_file("afeg    ;efhv   2[\-r u1\-ry1r") as jobs_path:
            assert_raises(crontab.ParseError,
                          crontab.generate_crontab,
                          [],
                          jobs_path,
                          "/path/to/app",
                          "some_id")

    def test_can_use_id_for_generating_crontab_entries(self):
        with temp_file("something, something, something, something, dark side") as something:
            generated_jobs = crontab.generate_crontab(
                [],
                something,
                "/path/to/my-app",
                "unique-id-of-my-app"
            )
            assert_that(generated_jobs,
                        has_item('# Begin performanceplatform.collector jobs '
                                 'for unique-id-of-my-app'))
            assert_that(generated_jobs,
                        has_item('# End performanceplatform.collector jobs '
                                 'for unique-id-of-my-app'))

    def test_can_handle_whitespace_and_comments(self):
        temp_contents = ("# some comment\n"
                         "          \n"
                         "schedule,query,creds,token,performanceplatforn\n")

        with temp_file(temp_contents) as something:
            generated_jobs = crontab.generate_crontab(
                [],
                something,
                "/path/to/my-app",
                "unique-id-of-my-app"
            )
            job_contains = "pp-collector -q /path/to/my-app/config/query"
            assert_that(generated_jobs,
                        has_item(
                            contains_string(job_contains)))

    @patch('socket.gethostname')
    def test_jobs_based_on_hostname_with_1(self, hostname):
        hostname.return_value = 'development-1'

        temp_contents = ("schedule,query,creds,token,performanceplatforn\n"
                         "schedule2,query2,creds2,token2,performanceplatforn\n"
                         "schedule3,query2,creds3,token3,performanceplatforn\n"
                         )

        with temp_file(temp_contents) as something:
            generated_jobs = crontab.generate_crontab(
                [],
                something,
                "/path/to/my-app",
                "unique-id-of-my-app"
            )

            assert_that(
                removeCommentsFromCrontab(generated_jobs), has_length(1))

    @patch('socket.gethostname')
    def test_jobs_based_on_hostname_with_2(self, hostname):
        hostname.return_value = 'development-2'

        temp_contents = ("schedule,query,creds,token,performanceplatforn\n"
                         "schedule2,query2,creds2,token2,performanceplatforn\n"
                         "schedule3,query2,creds3,token3,performanceplatforn\n"
                         )

        with temp_file(temp_contents) as something:
            generated_jobs = crontab.generate_crontab(
                [],
                something,
                "/path/to/my-app",
                "unique-id-of-my-app"
            )

            assert_that(
                removeCommentsFromCrontab(generated_jobs), has_length(1))

    @patch('socket.gethostname')
    def test_jobs_based_on_hostname_with_3(self, hostname):
        hostname.return_value = 'development-3'

        temp_contents = ("schedule,query,creds,token,performanceplatforn\n"
                         "schedule2,query2,creds2,token2,performanceplatforn\n"
                         "schedule3,query2,creds3,token3,performanceplatforn\n"
                         )

        with temp_file(temp_contents) as something:
            generated_jobs = crontab.generate_crontab(
                [],
                something,
                "/path/to/my-app",
                "unique-id-of-my-app"
            )

            assert_that(
                removeCommentsFromCrontab(generated_jobs), has_length(1))

    def tearDown(self):
        self.patcher.stop()

def removeCommentsFromCrontab(generated_jobs):
    return filter(lambda job: not job.startswith("#"), generated_jobs)

class ProcessFailureError(StandardError):

    def __init__(self, code, command, output):
        self.code = code
        self.command = command
        self.output = output

class TestCrontabScript(object):

    def setUp(self):
        self.patcher = patch('socket.gethostname')
        self.mock_gethostname = self.patcher.start()
        self.mock_gethostname.return_value = 'nonnumerichostname'

    def run_crontab_script(self, current_crontab, path_to_app, path_to_jobs,
                           unique_id):
        with temp_file(current_crontab) as stdin_path:
            args = [
                sys.executable,
                '-m', 'performanceplatform.collector.crontab',
                path_to_app, path_to_jobs, unique_id
            ]
            # Bleh Python 2.6 :(
            proc = subprocess.Popen(args,
                                    stdin=open(stdin_path),
                                    stderr=subprocess.STDOUT,
                                    stdout=subprocess.PIPE)
            output = proc.communicate()
            if proc.returncode != 0:
                raise ProcessFailureError(
                    proc.returncode, ' '.join(args), output=output[0])
            return output[0]

    def test_happy_path(self):
        with temp_file('') as path_to_jobs:
            output = self.run_crontab_script(
                'current crontab', '/path/to/app', path_to_jobs, 'some_id')
            assert_that(output.strip(),
                        is_('current crontab'))

    def test_failure_results_in_non_zero_exit_status(self):
        assert_raises(ProcessFailureError,
                      self.run_crontab_script,
                      'current crontab', '/path/to/app', 'invalid path',
                      'some_id')

    def test_failure_returns_original_crontab(self):
        try:
            self.run_crontab_script(
                'current crontab', '/path/to/app', 'invalid path', 'some_id')
        except ProcessFailureError as e:
            assert_that(e.output, contains_string('current crontab'))

    def tearDown(self):
        self.patcher.stop()
<commit_msg>Add assertions around the sharding of jobs<commit_after>import subprocess
from nose.tools import *
from hamcrest import *
import sys
from performanceplatform.collector.crontab import remove_existing_crontab_for_app

from tests.performanceplatform.collector.tools import temp_file
from performanceplatform.collector import crontab
from mock import patch

class TestRemoveExistingCrontab(object):

    def setUp(self):
        self.patcher = patch('socket.gethostname')
        self.mock_gethostname = self.patcher.start()
        self.mock_gethostname.return_value = 'nonnumerichostname'

    def test_jobs_are_not_removed_for_other_apps(self):
        crontab = [
            '# Begin performanceplatform.collector jobs for my-app',
            'foobar',
            '# End performanceplatform.collector jobs for my-app',
            '# Begin performanceplatform.collector jobs for other-app',
            'barfoo',
            '# End performanceplatform.collector jobs for other-app',
        ]
        new_crontab = remove_existing_crontab_for_app(crontab, 'my-app')
        assert_that(new_crontab,
                    has_item(contains_string('barfoo')))
        assert_that(new_crontab,
                    is_not(has_item(contains_string('foobar'))))

    def test_crontab_end_before_begin(self):
        crontab = [
            '# End performanceplatform.collector jobs for my-app',
            'foobar',
            '# Begin performanceplatform.collector jobs for my-app',
            'other'
        ]
        new_crontab = remove_existing_crontab_for_app(crontab, 'my-app')
        assert_that(new_crontab,
                    has_item(contains_string('foobar')))
        assert_that(new_crontab,
                    is_not(has_item(contains_string('other'))))

    def test_crontab_begin_with_no_end(self):
        crontab = [
            'foobar',
            '# Begin performanceplatform.collector jobs for my-app',
            'other',
        ]
        new_crontab = remove_existing_crontab_for_app(crontab, 'my-app')
        assert_that(new_crontab,
                    has_item(contains_string('foobar')))
        assert_that(new_crontab,
                    is_not(has_item(contains_string('other'))))

    def tearDown(self):
        self.patcher.stop()

class TestGenerateCrontab(object):

    def setUp(self):
        self.patcher = patch('socket.gethostname')
        self.mock_gethostname = self.patcher.start()
        self.mock_gethostname.return_value = 'nonnumerichostname'

    def test_other_cronjobs_are_preserved(self):
        with temp_file("") as jobs_path:
            generated_jobs = crontab.generate_crontab(
                ['other cronjob'],
                jobs_path,
                "/path/to/app",
                "some_id")
            assert_that(generated_jobs, has_item("other cronjob"))

    def test_some_cronjobs_are_added_between_containing_comments(self):
        with temp_file("schedule,query.json,creds.json,token.json,performanceplatform.json") as jobs_path:
            generated_jobs = crontab.generate_crontab(
                [],
                jobs_path,
                "/path/to/my-app",
                'some_id')
            assert_that(generated_jobs,
                        has_item('# Begin performanceplatform.collector jobs for some_id'))

            assert_that(generated_jobs,
                        has_item(contains_string("schedule")))
            assert_that(generated_jobs,
                        has_item(
                            contains_string("-q /path/to/my-app/config/query.json")))
            assert_that(generated_jobs,
                        has_item(
                            contains_string("-c /path/to/my-app/config/creds.json")))
            assert_that(generated_jobs,
                        has_item(
                            contains_string("-t /path/to/my-app/config/token.json")))
            assert_that(generated_jobs,
                        has_item(
                            contains_string("-b /path/to/my-app/config/performanceplatform.json")))

            assert_that(generated_jobs,
                        has_item('# End performanceplatform.collector jobs for some_id'))

    def test_added_jobs_run_the_crontab_module(self):
        with temp_file("schedule,query.json,creds.json,token.json,performanceplatform.json") as jobs_path:
            generated_jobs = crontab.generate_crontab(
                [],
                jobs_path,
                "/path/to/my-app",
                "some_id")
            assert_that(generated_jobs,
                        has_item(
                            contains_string("pp-collector")))

    def test_existing_pp_cronjobs_are_purged(self):
        with temp_file("schedule,query.json,creds.json,token.json,performanceplatform.json") as jobs_path:
            generated_jobs = crontab.generate_crontab(
                [
                    '# Begin performanceplatform.collector jobs for some_id',
                    'other job',
                    '# End performanceplatform.collector jobs for some_id'
                ],
                jobs_path,
                "/path/to/my-app",
                'some_id')
            assert_that(generated_jobs,
                        is_not(has_item(contains_string("other job"))))

    def test_invalid_jobs_file_causes_failure(self):
        with temp_file("afeg    ;efhv   2[\-r u1\-ry1r") as jobs_path:
            assert_raises(crontab.ParseError,
                          crontab.generate_crontab,
                          [],
                          jobs_path,
                          "/path/to/app",
                          "some_id")

    def test_can_use_id_for_generating_crontab_entries(self):
        with temp_file("something, something, something, something, dark side") as something:
            generated_jobs = crontab.generate_crontab(
                [],
                something,
                "/path/to/my-app",
                "unique-id-of-my-app"
            )
            assert_that(generated_jobs,
                        has_item('# Begin performanceplatform.collector jobs '
                                 'for unique-id-of-my-app'))
            assert_that(generated_jobs,
                        has_item('# End performanceplatform.collector jobs '
                                 'for unique-id-of-my-app'))

    def test_can_handle_whitespace_and_comments(self):
        temp_contents = ("# some comment\n"
                         "          \n"
                         "schedule,query,creds,token,performanceplatforn\n")

        with temp_file(temp_contents) as something:
            generated_jobs = crontab.generate_crontab(
                [],
                something,
                "/path/to/my-app",
                "unique-id-of-my-app"
            )
            job_contains = "pp-collector -q /path/to/my-app/config/query"
            assert_that(generated_jobs,
                        has_item(
                            contains_string(job_contains)))

    @patch('socket.gethostname')
    def test_jobs_based_on_hostname_with_1(self, hostname):
        hostname.return_value = 'development-1'

        temp_contents = ("schedule,query,creds,token,performanceplatforn\n"
                         "schedule2,query2,creds2,token2,performanceplatforn\n"
                         "schedule3,query2,creds3,token3,performanceplatforn\n"
                         )

        with temp_file(temp_contents) as something:
            generated_jobs = crontab.generate_crontab(
                [],
                something,
                "/path/to/my-app",
                "unique-id-of-my-app"
            )

            assert_that(
                removeCommentsFromCrontab(generated_jobs), has_length(1))
            assert_that(removeCommentsFromCrontab(
                generated_jobs)[0], contains_string("schedule "))

    @patch('socket.gethostname')
    def test_jobs_based_on_hostname_with_2(self, hostname):
        hostname.return_value = 'development-2'

        temp_contents = ("schedule,query,creds,token,performanceplatforn\n"
                         "schedule2,query2,creds2,token2,performanceplatforn\n"
                         "schedule3,query2,creds3,token3,performanceplatforn\n"
                         )

        with temp_file(temp_contents) as something:
            generated_jobs = crontab.generate_crontab(
                [],
                something,
                "/path/to/my-app",
                "unique-id-of-my-app"
            )

            assert_that(
                removeCommentsFromCrontab(generated_jobs), has_length(1))
            assert_that(removeCommentsFromCrontab(
                generated_jobs)[0], contains_string("schedule3 "))

    @patch('socket.gethostname')
    def test_jobs_based_on_hostname_with_3(self, hostname):
        hostname.return_value = 'development-3'

        temp_contents = ("schedule,query,creds,token,performanceplatforn\n"
                         "schedule2,query2,creds2,token2,performanceplatforn\n"
                         "schedule3,query2,creds3,token3,performanceplatforn\n"
                         )

        with temp_file(temp_contents) as something:
            generated_jobs = crontab.generate_crontab(
                [],
                something,
                "/path/to/my-app",
                "unique-id-of-my-app"
            )

            assert_that(
                removeCommentsFromCrontab(generated_jobs), has_length(1))
            assert_that(removeCommentsFromCrontab(
                generated_jobs)[0], contains_string("schedule2 "))

    def tearDown(self):
        self.patcher.stop()

def removeCommentsFromCrontab(generated_jobs):
    return filter(lambda job: not job.startswith("#"), generated_jobs)

class ProcessFailureError(StandardError):

    def __init__(self, code, command, output):
        self.code = code
        self.command = command
        self.output = output

class TestCrontabScript(object):

    def setUp(self):
        self.patcher = patch('socket.gethostname')
        self.mock_gethostname = self.patcher.start()
        self.mock_gethostname.return_value = 'nonnumerichostname'

    def run_crontab_script(self, current_crontab, path_to_app, path_to_jobs,
                           unique_id):
        with temp_file(current_crontab) as stdin_path:
            args = [
                sys.executable,
                '-m', 'performanceplatform.collector.crontab',
                path_to_app, path_to_jobs, unique_id
            ]
            # Bleh Python 2.6 :(
            proc = subprocess.Popen(args,
                                    stdin=open(stdin_path),
                                    stderr=subprocess.STDOUT,
                                    stdout=subprocess.PIPE)
            output = proc.communicate()
            if proc.returncode != 0:
                raise ProcessFailureError(
                    proc.returncode, ' '.join(args), output=output[0])
            return output[0]

    def test_happy_path(self):
        with temp_file('') as path_to_jobs:
            output = self.run_crontab_script(
                'current crontab', '/path/to/app', path_to_jobs, 'some_id')
            assert_that(output.strip(),
                        is_('current crontab'))

    def test_failure_results_in_non_zero_exit_status(self):
        assert_raises(ProcessFailureError,
                      self.run_crontab_script,
                      'current crontab', '/path/to/app', 'invalid path',
                      'some_id')

    def test_failure_returns_original_crontab(self):
        try:
            self.run_crontab_script(
                'current crontab', '/path/to/app', 'invalid path', 'some_id')
        except ProcessFailureError as e:
            assert_that(e.output, contains_string('current crontab'))

    def tearDown(self):
        self.patcher.stop()
<|endoftext|>


<commit_before>"""
Regression tests for the Test Client, especially the customized assertions.

"""
from django.test import Client, TestCase
from django.core import mail
import os

class AssertContainsTests(TestCase):
    def test_contains(self):
        "Reponses can be inspected for content, including counting repeated substrings"
        response = self.client.get('/test_client_regress/no_template_view/')

        self.assertContains(response, 'once')
        self.assertContains(response, 'once', 1)
        self.assertContains(response, 'twice')
        self.assertContains(response, 'twice', 2)

        try:
            self.assertContains(response, 'once', 2)
        except AssertionError, e:
            self.assertEquals(str(e), "Found 1 instances of 'once' in response (expected 2)")

        try:
            self.assertContains(response, 'twice', 1)
        except AssertionError, e:
            self.assertEquals(str(e), "Found 2 instances of 'twice' in response (expected 1)")

        try:
            self.assertContains(response, 'thrice')
        except AssertionError, e:
            self.assertEquals(str(e), "Couldn't find 'thrice' in response")

        try:
            self.assertContains(response, 'thrice', 3)
        except AssertionError, e:
            self.assertEquals(str(e), "Found 0 instances of 'thrice' in response (expected 3)")

class AssertTemplateUsedTests(TestCase):
    fixtures = ['testdata.json']

    def test_no_context(self):
        "Template usage assertions work then templates aren't in use"
        response = self.client.get('/test_client_regress/no_template_view/')

        # Check that the no template case doesn't mess with the template assertions
        self.assertTemplateNotUsed(response, 'GET Template')

        try:
            self.assertTemplateUsed(response, 'GET Template')
        except AssertionError, e:
            self.assertEquals(str(e), "No templates used to render the response")

    def test_single_context(self):
        "Template assertions work when there is a single context"
        response = self.client.get('/test_client/post_view/', {})

        #
        try:
            self.assertTemplateNotUsed(response, 'Empty GET Template')
        except AssertionError, e:
            self.assertEquals(str(e), "Template 'Empty GET Template' was used unexpectedly in rendering the response")

        try:
            self.assertTemplateUsed(response, 'Empty POST Template')
        except AssertionError, e:
            self.assertEquals(str(e), "Template 'Empty POST Template' was not used to render the response. Actual template was 'Empty GET Template'")

    def test_multiple_context(self):
        "Template assertions work when there are multiple contexts"
        post_data = {
            'text': 'Hello World',
            'email': 'foo@example.com',
            'value': 37,
            'single': 'b',
            'multi': ('b','c','e')
        }
        response = self.client.post('/test_client/form_view_with_template/', post_data)
        self.assertContains(response, 'POST data OK')
        try:
            self.assertTemplateNotUsed(response, "form_view.html")
        except AssertionError, e:
            self.assertEquals(str(e), "Template 'form_view.html' was used unexpectedly in rendering the response")

        try:
            self.assertTemplateNotUsed(response, 'base.html')
        except AssertionError, e:
            self.assertEquals(str(e), "Template 'base.html' was used unexpectedly in rendering the response")

        try:
            self.assertTemplateUsed(response, "Valid POST Template")
        except AssertionError, e:
            self.assertEquals(str(e), "Template 'Valid POST Template' was not one of the templates used to render the response. Templates used: form_view.html, base.html")

class AssertRedirectsTests(TestCase):
    def test_redirect_page(self):
        "An assertion is raised if the original page couldn't be retrieved as expected"
        # This page will redirect with code 301, not 302
        response = self.client.get('/test_client/permanent_redirect_view/')
        try:
            self.assertRedirects(response, '/test_client/get_view/')
        except AssertionError, e:
            self.assertEquals(str(e), "Response didn't redirect as expected: Response code was 301 (expected 302)")

    def test_incorrect_target(self):
        "An assertion is raised if the response redirects to another target"
        response = self.client.get('/test_client/permanent_redirect_view/')
        try:
            # Should redirect to get_view
            self.assertRedirects(response, '/test_client/some_view/')
        except AssertionError, e:
            self.assertEquals(str(e), "Response didn't redirect as expected: Response code was 301 (expected 302)")

    def test_target_page(self):
        "An assertion is raised if the response redirect target cannot be retrieved as expected"
        response = self.client.get('/test_client/double_redirect_view/')
        try:
            # The redirect target responds with a 301 code, not 200
            self.assertRedirects(response, '/test_client/permanent_redirect_view/')
        except AssertionError, e:
            self.assertEquals(str(e), "Couldn't retrieve redirection page '/test_client/permanent_redirect_view/': response code was 301 (expected 200)")

class AssertFormErrorTests(TestCase):
    def test_unknown_form(self):
        "An assertion is raised if the form name is unknown"
        post_data = {
            'text': 'Hello World',
            'email': 'not an email address',
            'value': 37,
            'single': 'b',
            'multi': ('b','c','e')
        }
        response = self.client.post('/test_client/form_view/', post_data)
        self.assertEqual(response.status_code, 200)
        self.assertTemplateUsed(response, "Invalid POST Template")

        try:
            self.assertFormError(response, 'wrong_form', 'some_field', 'Some error.')
        except AssertionError, e:
            self.assertEqual(str(e), "The form 'wrong_form' was not used to render the response")

    def test_unknown_field(self):
        "An assertion is raised if the field name is unknown"
        post_data = {
            'text': 'Hello World',
            'email': 'not an email address',
            'value': 37,
            'single': 'b',
            'multi': ('b','c','e')
        }
        response = self.client.post('/test_client/form_view/', post_data)
        self.assertEqual(response.status_code, 200)
        self.assertTemplateUsed(response, "Invalid POST Template")

        try:
            self.assertFormError(response, 'form', 'some_field', 'Some error.')
        except AssertionError, e:
            self.assertEqual(str(e), "The form 'form' in context 0 does not contain the field 'some_field'")

    def test_noerror_field(self):
        "An assertion is raised if the field doesn't have any errors"
        post_data = {
            'text': 'Hello World',
            'email': 'not an email address',
            'value': 37,
            'single': 'b',
            'multi': ('b','c','e')
        }
        response = self.client.post('/test_client/form_view/', post_data)
        self.assertEqual(response.status_code, 200)
        self.assertTemplateUsed(response, "Invalid POST Template")

        try:
            self.assertFormError(response, 'form', 'value', 'Some error.')
        except AssertionError, e:
            self.assertEqual(str(e), "The field 'value' on form 'form' in context 0 contains no errors")

    def test_unknown_error(self):
        "An assertion is raised if the field doesn't contain the provided error"
        post_data = {
            'text': 'Hello World',
            'email': 'not an email address',
            'value': 37,
            'single': 'b',
            'multi': ('b','c','e')
        }
        response = self.client.post('/test_client/form_view/', post_data)
        self.assertEqual(response.status_code, 200)
        self.assertTemplateUsed(response, "Invalid POST Template")

        try:
            self.assertFormError(response, 'form', 'email', 'Some error.')
        except AssertionError, e:
            self.assertEqual(str(e), "The field 'email' on form 'form' in context 0 does not contain the error 'Some error.' (actual errors: [u'Enter a valid e-mail address.'])")

class AssertFileUploadTests(TestCase):
    def test_simple_upload(self):
        fd = open(os.path.join(os.path.dirname(__file__), "views.py"))
        post_data = {
            'name': 'Ringo',
            'file_field': fd,
        }
        response = self.client.post('/test_client_regress/file_upload/', post_data)
        self.assertEqual(response.status_code, 200)
<commit_msg>Fixed negligible typo in docstring in tests/regressiontests/test_client_regress/models.py from [5731]<commit_after>"""
Regression tests for the Test Client, especially the customized assertions.

"""
from django.test import Client, TestCase
from django.core import mail
import os

class AssertContainsTests(TestCase):
    def test_contains(self):
        "Responses can be inspected for content, including counting repeated substrings"
        response = self.client.get('/test_client_regress/no_template_view/')

        self.assertContains(response, 'once')
        self.assertContains(response, 'once', 1)
        self.assertContains(response, 'twice')
        self.assertContains(response, 'twice', 2)

        try:
            self.assertContains(response, 'once', 2)
        except AssertionError, e:
            self.assertEquals(str(e), "Found 1 instances of 'once' in response (expected 2)")

        try:
            self.assertContains(response, 'twice', 1)
        except AssertionError, e:
            self.assertEquals(str(e), "Found 2 instances of 'twice' in response (expected 1)")

        try:
            self.assertContains(response, 'thrice')
        except AssertionError, e:
            self.assertEquals(str(e), "Couldn't find 'thrice' in response")

        try:
            self.assertContains(response, 'thrice', 3)
        except AssertionError, e:
            self.assertEquals(str(e), "Found 0 instances of 'thrice' in response (expected 3)")

class AssertTemplateUsedTests(TestCase):
    fixtures = ['testdata.json']

    def test_no_context(self):
        "Template usage assertions work then templates aren't in use"
        response = self.client.get('/test_client_regress/no_template_view/')

        # Check that the no template case doesn't mess with the template assertions
        self.assertTemplateNotUsed(response, 'GET Template')

        try:
            self.assertTemplateUsed(response, 'GET Template')
        except AssertionError, e:
            self.assertEquals(str(e), "No templates used to render the response")

    def test_single_context(self):
        "Template assertions work when there is a single context"
        response = self.client.get('/test_client/post_view/', {})

        #
        try:
            self.assertTemplateNotUsed(response, 'Empty GET Template')
        except AssertionError, e:
            self.assertEquals(str(e), "Template 'Empty GET Template' was used unexpectedly in rendering the response")

        try:
            self.assertTemplateUsed(response, 'Empty POST Template')
        except AssertionError, e:
            self.assertEquals(str(e), "Template 'Empty POST Template' was not used to render the response. Actual template was 'Empty GET Template'")

    def test_multiple_context(self):
        "Template assertions work when there are multiple contexts"
        post_data = {
            'text': 'Hello World',
            'email': 'foo@example.com',
            'value': 37,
            'single': 'b',
            'multi': ('b','c','e')
        }
        response = self.client.post('/test_client/form_view_with_template/', post_data)
        self.assertContains(response, 'POST data OK')
        try:
            self.assertTemplateNotUsed(response, "form_view.html")
        except AssertionError, e:
            self.assertEquals(str(e), "Template 'form_view.html' was used unexpectedly in rendering the response")

        try:
            self.assertTemplateNotUsed(response, 'base.html')
        except AssertionError, e:
            self.assertEquals(str(e), "Template 'base.html' was used unexpectedly in rendering the response")

        try:
            self.assertTemplateUsed(response, "Valid POST Template")
        except AssertionError, e:
            self.assertEquals(str(e), "Template 'Valid POST Template' was not one of the templates used to render the response. Templates used: form_view.html, base.html")

class AssertRedirectsTests(TestCase):
    def test_redirect_page(self):
        "An assertion is raised if the original page couldn't be retrieved as expected"
        # This page will redirect with code 301, not 302
        response = self.client.get('/test_client/permanent_redirect_view/')
        try:
            self.assertRedirects(response, '/test_client/get_view/')
        except AssertionError, e:
            self.assertEquals(str(e), "Response didn't redirect as expected: Response code was 301 (expected 302)")

    def test_incorrect_target(self):
        "An assertion is raised if the response redirects to another target"
        response = self.client.get('/test_client/permanent_redirect_view/')
        try:
            # Should redirect to get_view
            self.assertRedirects(response, '/test_client/some_view/')
        except AssertionError, e:
            self.assertEquals(str(e), "Response didn't redirect as expected: Response code was 301 (expected 302)")

    def test_target_page(self):
        "An assertion is raised if the response redirect target cannot be retrieved as expected"
        response = self.client.get('/test_client/double_redirect_view/')
        try:
            # The redirect target responds with a 301 code, not 200
            self.assertRedirects(response, '/test_client/permanent_redirect_view/')
        except AssertionError, e:
            self.assertEquals(str(e), "Couldn't retrieve redirection page '/test_client/permanent_redirect_view/': response code was 301 (expected 200)")

class AssertFormErrorTests(TestCase):
    def test_unknown_form(self):
        "An assertion is raised if the form name is unknown"
        post_data = {
            'text': 'Hello World',
            'email': 'not an email address',
            'value': 37,
            'single': 'b',
            'multi': ('b','c','e')
        }
        response = self.client.post('/test_client/form_view/', post_data)
        self.assertEqual(response.status_code, 200)
        self.assertTemplateUsed(response, "Invalid POST Template")

        try:
            self.assertFormError(response, 'wrong_form', 'some_field', 'Some error.')
        except AssertionError, e:
            self.assertEqual(str(e), "The form 'wrong_form' was not used to render the response")

    def test_unknown_field(self):
        "An assertion is raised if the field name is unknown"
        post_data = {
            'text': 'Hello World',
            'email': 'not an email address',
            'value': 37,
            'single': 'b',
            'multi': ('b','c','e')
        }
        response = self.client.post('/test_client/form_view/', post_data)
        self.assertEqual(response.status_code, 200)
        self.assertTemplateUsed(response, "Invalid POST Template")

        try:
            self.assertFormError(response, 'form', 'some_field', 'Some error.')
        except AssertionError, e:
            self.assertEqual(str(e), "The form 'form' in context 0 does not contain the field 'some_field'")

    def test_noerror_field(self):
        "An assertion is raised if the field doesn't have any errors"
        post_data = {
            'text': 'Hello World',
            'email': 'not an email address',
            'value': 37,
            'single': 'b',
            'multi': ('b','c','e')
        }
        response = self.client.post('/test_client/form_view/', post_data)
        self.assertEqual(response.status_code, 200)
        self.assertTemplateUsed(response, "Invalid POST Template")

        try:
            self.assertFormError(response, 'form', 'value', 'Some error.')
        except AssertionError, e:
            self.assertEqual(str(e), "The field 'value' on form 'form' in context 0 contains no errors")

    def test_unknown_error(self):
        "An assertion is raised if the field doesn't contain the provided error"
        post_data = {
            'text': 'Hello World',
            'email': 'not an email address',
            'value': 37,
            'single': 'b',
            'multi': ('b','c','e')
        }
        response = self.client.post('/test_client/form_view/', post_data)
        self.assertEqual(response.status_code, 200)
        self.assertTemplateUsed(response, "Invalid POST Template")

        try:
            self.assertFormError(response, 'form', 'email', 'Some error.')
        except AssertionError, e:
            self.assertEqual(str(e), "The field 'email' on form 'form' in context 0 does not contain the error 'Some error.' (actual errors: [u'Enter a valid e-mail address.'])")

class AssertFileUploadTests(TestCase):
    def test_simple_upload(self):
        fd = open(os.path.join(os.path.dirname(__file__), "views.py"))
        post_data = {
            'name': 'Ringo',
            'file_field': fd,
        }
        response = self.client.post('/test_client_regress/file_upload/', post_data)
        self.assertEqual(response.status_code, 200)
<|endoftext|>


<commit_before>"""
This test checks the behavior of openfile when filehandles should be exhausted.
"""

#pragma repy

MAX_FILES = 5

files = listfiles()
filehandles = [None, None, None, None, None]

# Open random files
for x in xrange(MAX_FILES):
  filehandles[x] = openfile(files[x], False)

# Try to open this file with create, we should get an error
# and the file should not be created
TRY_FILE = "trying.to.create.this.file"

try:
  fileh = openfile(TRY_FILE, True)
except ResourceExhaustedError:
  pass
else:
  log("Opened file with no handles available!",'\n')

# Check if the file exists
if TRY_FILE in listfiles():
  log("The file was created with no handles available!",'\n')

<commit_msg>Fix #48, test exhausting file handles<commit_after>"""
This test checks the behavior of openfile when filehandles should be exhausted.
"""

#pragma repy
filehandles = []
filenames = []

RAND_FILE = "random.files.to.create.number"

# Open random files, as many as the resource restrictions allows
for number in xrange(getresources()[0]['filesopened']):
  filenames.append(RAND_FILE + str(number + 1))
  filehandles.append(openfile((RAND_FILE + str(number + 1)), True))

# Try to open this file with create, we should get an error
# and the file should not be created
TRYFILE = "trying.to.create.this.file"

try:
  fileh = openfile(TRYFILE, True)
except ResourceExhaustedError:
  pass
else:
  log("Opened file with no handles available!",'\n')
# Check if the file exists
if TRYFILE in listfiles():
  log("The file was created with no handles available!",'\n')
  fileh.close()
  removefile(TRYFILE)

#Close all random files opened
for randfile in filehandles:
  randfile.close()

#Clean up all random files
for name in filenames:
  removefile(name)<|endoftext|>


<commit_before>from .environment import Environment

from viper.parser.ast.nodes import AST, Parameter

from inspect import signature
from typing import Callable, List

class Value:
    pass

class TupleVal(Value):
    def __init__(self, *vals: Value):
        self.vals = list(vals)

    def __repr__(self) -> str:
        return f"TupleVal({', '.join(map(str, self.vals))})"

class NumVal(Value):
    def __init__(self, val: str):
        self.val = val

    def __repr__(self) -> str:
        return f"NumVal({self.val})"

class CloVal(Value):
    def __init__(self, params: List[Parameter], code: AST, env: Environment):
        self.params = params
        self.code = code
        self.env = env

    def __repr__(self) -> str:
        return f"CloVal(({', '.join(map(lambda p: p.internal, self.params))}), {self.env})"

class ForeignCloVal(Value):
    def __init__(self, func: Callable, env: Environment):
        self.params: List[str] = list(signature(func).parameters)
        self.func = func
        self.env = env

    def __repr__(self) -> str:
        return f"ForeignCloVal(({', '.join(self.params)}), {self.env})"

class BoolVal(Value):
    def __repr__(self) -> str:
        return "BoolVal"

class TrueVal(BoolVal):
    def __repr__(self) -> str:
        return "TrueVal"

class FalseVal(BoolVal):
    def __repr__(self) -> str:
        return "FalseVal"

class UnitVal(Value):
    def __repr__(self) -> str:
        return "UnitVal"

class BottomVal(Value):
    def __repr__(self) -> str:
        return "BottomVal"

class EllipsisVal(Value):
    def __repr__(self) -> str:
        return "EllipsisVal"

def val_to_python(val: Value):
    if isinstance(val, NumVal):
        num_val = float(val.val)
        if num_val.is_integer():
            num_val = int(num_val)
        return num_val
    elif isinstance(val, TrueVal):
        return True
    elif isinstance(val, FalseVal):
        return False
    else:
        raise RuntimeError(f"Cannot convert value to native Python value: {val}")
<commit_msg>Add function to convert Python values to Value<commit_after>from .environment import Environment

from viper.parser.ast.nodes import AST, Parameter

from inspect import signature
from typing import Callable, List

class Value:
    pass

class TupleVal(Value):
    def __init__(self, *vals: Value):
        self.vals = list(vals)

    def __repr__(self) -> str:
        return f"TupleVal({', '.join(map(str, self.vals))})"

class NumVal(Value):
    def __init__(self, val: str):
        self.val = val

    def __repr__(self) -> str:
        return f"NumVal({self.val})"

class CloVal(Value):
    def __init__(self, params: List[Parameter], code: AST, env: Environment):
        self.params = params
        self.code = code
        self.env = env

    def __repr__(self) -> str:
        return f"CloVal(({', '.join(map(lambda p: p.internal, self.params))}), {self.env})"

class ForeignCloVal(Value):
    def __init__(self, func: Callable, env: Environment):
        self.params: List[str] = list(signature(func).parameters)
        self.func = func
        self.env = env

    def __repr__(self) -> str:
        return f"ForeignCloVal(({', '.join(self.params)}), {self.env})"

class BoolVal(Value):
    def __repr__(self) -> str:
        return "BoolVal"

class TrueVal(BoolVal):
    def __repr__(self) -> str:
        return "TrueVal"

class FalseVal(BoolVal):
    def __repr__(self) -> str:
        return "FalseVal"

class UnitVal(Value):
    def __repr__(self) -> str:
        return "UnitVal"

class BottomVal(Value):
    def __repr__(self) -> str:
        return "BottomVal"

class EllipsisVal(Value):
    def __repr__(self) -> str:
        return "EllipsisVal"

def val_to_python(val: Value):
    if isinstance(val, NumVal):
        num_val = float(val.val)
        if num_val.is_integer():
            num_val = int(num_val)
        return num_val
    elif isinstance(val, TrueVal):
        return True
    elif isinstance(val, FalseVal):
        return False
    else:
        raise RuntimeError(f"Cannot convert Value to native Python value: {val}")  # TODO: Use custom error.

def python_to_val(py) -> Value:
    if isinstance(py, int) or isinstance(py, float):
        return NumVal(str(py))
    elif isinstance(py, bool):
        if py:
            return TrueVal()
        else:
            return FalseVal()
    else:
        raise RuntimeError(f"Cannot convert native Python value to Value: {py}")  # TODO: Use custom error.
<|endoftext|>