<commit_before>from __future__ import annotations

import random
from abc import ABC, abstractmethod
import logging
from typing import Iterable, Dict

import numpy
from typing_extensions import Protocol
import sklearn.linear_model

import dedupe.core as core
import dedupe.training as training
import dedupe.datamodel as datamodel
from dedupe._typing import TrainingExample, RecordID
from dedupe.predicates import Predicate

logger = logging.getLogger(__name__)

class ActiveLearner(ABC):
    @abstractmethod
    def transform(self) -> None:
        pass

    @abstractmethod
    def pop(self) -> TrainingExample:
        pass

    @abstractmethod
    def mark(self) -> None:
        pass

    @abstractmethod
    def __len__(self) -> int:
        pass

class HasDataModel(Protocol):

    data_model: datamodel.DataModel

class RLRLearner(sklearn.linear_model.LogisticRegression, ActiveLearner):
    def __init__(self, data_model):
        super().__init__()
        self.data_model = data_model
        self._candidates: list[TrainingExample]

    @property
    def candidates(self) -> list[TrainingExample]:
        return self._candidates

    @candidates.setter
    def candidates(self, new_candidates):
        self._candidates = new_candidates

        self.distances = self.transform(self._candidates)

        random_pair = random.choice(self._candidates)
        exact_match = (random_pair[0], random_pair[0])
        self.fit_transform([exact_match, random_pair], [1, 0])

    def transform(self, pairs):
        return self.data_model.distances(pairs)

    def fit(self, X, y):

        self.y = numpy.array(y)
        self.X = X

        super().fit(self.X, self.y)

    def fit_transform(self, pairs, y):
        self.fit(self.transform(pairs), y)

    def pop(self) -> TrainingExample:
        if not len(self.candidates):
            raise IndexError("No more unlabeled examples to label")

        target_uncertainty = self._bias()

        probabilities = self.candidate_scores()

        distance_to_target = numpy.abs(target_uncertainty - probabilities)
        uncertain_index = distance_to_target.argmin()

        self.distances = numpy.delete(self.distances, uncertain_index, axis=0)

        uncertain_pair = self.candidates.pop(uncertain_index)

        return uncertain_pair

    def _remove(self, index):
        self.distances = numpy.delete(self.distances, index, axis=0)

    def mark(self, pairs, y):

        self.y = numpy.concatenate([self.y, y])
        self.X = numpy.vstack([self.X, self.transform(pairs)])

        self.fit(self.X, self.y)

    def _bias(self):
        positive = numpy.sum(self.y == 1)
        n_examples = len(self.y)

        bias = 1 - (positive / n_examples if positive else 0)

        # When we have just a few examples we are okay with getting
        # examples where the model strongly believes the example is
        # going to be positive or negative. As we get more examples,
        # prefer to ask for labels of examples the model is more
        # uncertain of.
        uncertainty_weight = min(positive, n_examples - positive)
        bias_weight = 10

        weighted_bias = 0.5 * uncertainty_weight + bias * bias_weight
        weighted_bias /= uncertainty_weight + bias_weight

        return weighted_bias

    def candidate_scores(self):
        return self.predict_proba(self.distances)[:, 1].reshape(-1, 1)

    def __len__(self):
        return len(self.candidates)

class BlockLearner(object):
    def __init__(self, data_model: datamodel.DataModel, *args):
        self.data_model = data_model

        self.current_predicates: tuple[Predicate, ...] = ()

        self._cached_labels: numpy.ndarray | None = None
        self._old_dupes = []

        self.block_learner: training.BlockLearner

    def fit_transform(self, pairs, y):
        dupes = [pair for label, pair in zip(y, pairs) if label]

        new_dupes = [pair for pair in dupes if pair not in self._old_dupes]
        new_uncovered = not all(self.predict(new_dupes))

        if new_uncovered:
            self.current_predicates = self.block_learner.learn(dupes, recall=1.0)
            self._cached_labels = None
            self._old_dupes = dupes

    def candidate_scores(self):
        if self._cached_labels is None:
            labels = self.predict(self.candidates)
            self._cached_labels = numpy.array(labels).reshape(-1, 1)

        return self._cached_labels

    def predict(self, candidates):
        labels = []
        for record_1, record_2 in candidates:

            for predicate in self.current_predicates:
                keys = predicate(record_2, target=True)
                if keys:
                    if set(predicate(record_1)) & set(keys):
                        labels.append(1)
                        break
            else:
                labels.append(0)

        return labels

    def _remove(self, index):
        if self._cached_labels is not None:
            self._cached_labels = numpy.delete(self._cached_labels, index, axis=0)

    def _sample_indices(self, sample_size: int) -> Iterable[tuple[RecordID, RecordID]]:

        weights: Dict[tuple[RecordID, RecordID], float] = {}
        for predicate, covered in self.block_learner.comparison_cover.items():
            # each predicate gets to vote for every record pair it covers. the
            # strength of that vote is in inverse proportion to the number of
            # records the predicate covers.
            #
            # if a predicate only covers a few record pairs, the value of
            # the vote it puts on those few pairs will be worth more than
            # a predicate that covers almost all the record pairs
            weight: float = 1 / len(covered)
            for pair in covered:
                weights[pair] = weights.get(pair, 0.0) + weight

        if sample_size < len(weights):
            # consider using a reservoir sampling strategy, which would
            # be more memory efficient and probably about as fast
            normalized_weights = numpy.fromiter(weights.values(), dtype=float) / sum(
                weights.values()
            )
            rng = numpy.random.default_rng()
            sample_indices = rng.choice(
                len(weights), size=sample_size, replace=False, p=normalized_weights
            )
            keys = list(weights.keys())
            sample_ids = ((keys[i][0], keys[i][1]) for i in sample_indices)
        else:
            sample_ids = weights.keys()

        return sample_ids

class DedupeBlockLearner(BlockLearner):
    def __init__(self, data_model: datamodel.DataModel, data, index_include):
        super().__init__(data_model)

        N_SAMPLED_RECORDS = 5000
        N_SAMPLED_RECORD_PAIRS = 10000

        index_data = Sample(data, 50000)
        sampled_records = Sample(index_data, N_SAMPLED_RECORDS)
        preds = self.data_model.predicates()

        self.block_learner = training.DedupeBlockLearner(
            preds, sampled_records, index_data
        )

        self.candidates = self._sample(sampled_records, N_SAMPLED_RECORD_PAIRS)
        examples_to_index = self.candidates.copy()

        if index_include:
            examples_to_index += index_include

        self._index_predicates(examples_to_index)

    def _index_predicates(self, candidates):

        blocker = self.block_learner.blocker

        records = core.unique((record for pair in candidates for record in pair))

        for field in blocker.index_fields:
            unique_fields = {record[field] for record in records}
            blocker.index(unique_fields, field)

        for pred in blocker.index_predicates:
            pred.freeze(records)

    def _sample(self, data, sample_size):

        sample_indices = self._sample_indices(sample_size)

        sample = [(data[id_1], data[id_2]) for id_1, id_2 in sample_indices]

        return sample

class RecordLinkBlockLearner(BlockLearner):
    def __init__(self, data_model, data_1, data_2, index_include):

        super().__init__(data_model)

        N_SAMPLED_RECORDS = 1000
        N_SAMPLED_RECORD_PAIRS = 5000

        sampled_records_1 = Sample(data_1, N_SAMPLED_RECORDS)
        index_data = Sample(data_2, 50000)
        sampled_records_2 = Sample(index_data, N_SAMPLED_RECORDS)

        preds = self.data_model.predicates(canopies=False)

        self.block_learner = training.RecordLinkBlockLearner(
            preds, sampled_records_1, sampled_records_2, index_data
        )

        self.candidates = self._sample(
            sampled_records_1, sampled_records_2, N_SAMPLED_RECORD_PAIRS
        )
        examples_to_index = self.candidates.copy()

        if index_include:
            examples_to_index += index_include

        self._index_predicates(examples_to_index)

    def _index_predicates(self, candidates):

        blocker = self.block_learner.blocker

        A, B = zip(*candidates)
        A = core.unique(A)
        B = core.unique(B)

        for field in blocker.index_fields:
            unique_fields = {record[field] for record in B}
            blocker.index(unique_fields, field)

        for pred in blocker.index_predicates:
            pred.freeze(A, B)

    def _sample(self, data_1, data_2, sample_size):

        sample_indices = self._sample_indices(sample_size)

        sample = [(data_1[id_1], data_2[id_2]) for id_1, id_2 in sample_indices]

        return sample

class DisagreementLearner(ActiveLearner):

    classifier: RLRLearner
    blocker: BlockLearner
    candidates: list[TrainingExample]

    def _common_init(self):

        self.learners = (self.classifier, self.blocker)
        self.y = numpy.array([])
        self.pairs = []

    def pop(self) -> TrainingExample:
        if not len(self.candidates):
            raise IndexError("No more unlabeled examples to label")

        probs_l = []
        for learner in self.learners:
            probabilities = learner.candidate_scores()
            probs_l.append(probabilities)

        probs = numpy.concatenate(probs_l, axis=1)

        # where do the classifers disagree?
        disagreement = numpy.std(probs > 0.5, axis=1).astype(bool)

        if disagreement.any():
            conflicts = disagreement.nonzero()[0]
            target = numpy.random.uniform(size=1)
            uncertain_index = conflicts[numpy.argmax(probs[conflicts][:, 0] - target)]
        else:
            uncertain_index = numpy.std(probs, axis=1).argmax()

        logger.debug(
            "Classifier: %.2f, Covered: %s",
            probs[uncertain_index][0],
            bool(probs[uncertain_index][1]),
        )

        uncertain_pair = self.candidates.pop(uncertain_index)

        for learner in self.learners:
            learner._remove(uncertain_index)

        return uncertain_pair

    def mark(self, pairs, y):

        self.y = numpy.concatenate([self.y, y])
        self.pairs.extend(pairs)

        for learner in self.learners:
            learner.fit_transform(self.pairs, self.y)

    def __len__(self):
        return len(self.candidates)

    def transform(self):
        pass

    def learn_predicates(self, recall, index_predicates):
        dupes = [pair for label, pair in zip(self.y, self.pairs) if label]

        if not index_predicates:
            old_preds = self.blocker.block_learner.blocker.predicates.copy()

            no_index_predicates = [
                pred for pred in old_preds if not hasattr(pred, "index")
            ]
            self.blocker.block_learner.blocker.predicates = no_index_predicates

            learned_preds = self.blocker.block_learner.learn(
                dupes, recall=recall, candidate_types="random forest"
            )

            self.blocker.block_learner.blocker.predicates = old_preds

        else:
            learned_preds = self.blocker.block_learner.learn(
                dupes, recall=recall, candidate_types="random forest"
            )

        return learned_preds

class DedupeDisagreementLearner(DisagreementLearner):
    def __init__(
        self, data_model, data, blocked_proportion, sample_size, index_include
    ):

        self.data_model = data_model

        data = core.index(data)

        random_pair = (
            random.choice(list(data.values())),
            random.choice(list(data.values())),
        )
        exact_match = (random_pair[0], random_pair[0])

        index_include = index_include.copy()
        index_include.append(exact_match)

        self.blocker = DedupeBlockLearner(data_model, data, index_include)

        self.candidates = self.blocker.candidates

        self.classifier = RLRLearner(self.data_model)
        self.classifier.candidates = self.candidates

        self._common_init()

        self.mark([exact_match] * 4 + [random_pair], [1] * 4 + [0])

class RecordLinkDisagreementLearner(DisagreementLearner):
    def __init__(
        self, data_model, data_1, data_2, blocked_proportion, sample_size, index_include
    ):

        self.data_model = data_model

        data_1 = core.index(data_1)

        offset = len(data_1)
        data_2 = core.index(data_2, offset)

        random_pair = (
            random.choice(list(data_1.values())),
            random.choice(list(data_2.values())),
        )
        exact_match = (random_pair[0], random_pair[0])

        index_include = index_include.copy()
        index_include.append(exact_match)

        self.blocker = RecordLinkBlockLearner(data_model, data_1, data_2, index_include)
        self.candidates = self.blocker.candidates

        self.classifier = RLRLearner(self.data_model)
        self.classifier.candidates = self.candidates

        self._common_init()

        self.mark([exact_match] * 4 + [random_pair], [1] * 4 + [0])

class Sample(dict):
    def __init__(self, d, sample_size):
        if len(d) <= sample_size:
            super().__init__(d)
        else:
            sample = random.sample(d.keys(), sample_size)
            super().__init__({k: d[k] for k in sample})
<commit_msg>type fix<commit_after>from __future__ import annotations

import random
from abc import ABC, abstractmethod
import logging
from typing import Iterable, Dict

import numpy
from typing_extensions import Protocol
import sklearn.linear_model

import dedupe.core as core
import dedupe.training as training
import dedupe.datamodel as datamodel
from dedupe._typing import TrainingExample, RecordID, RecordDict
from dedupe.predicates import Predicate

logger = logging.getLogger(__name__)

class ActiveLearner(ABC):
    @abstractmethod
    def transform(self) -> None:
        pass

    @abstractmethod
    def pop(self) -> TrainingExample:
        pass

    @abstractmethod
    def mark(self) -> None:
        pass

    @abstractmethod
    def __len__(self) -> int:
        pass

class HasDataModel(Protocol):

    data_model: datamodel.DataModel

class RLRLearner(sklearn.linear_model.LogisticRegression, ActiveLearner):
    def __init__(self, data_model):
        super().__init__()
        self.data_model = data_model
        self._candidates: list[TrainingExample]

    @property
    def candidates(self) -> list[TrainingExample]:
        return self._candidates

    @candidates.setter
    def candidates(self, new_candidates):
        self._candidates = new_candidates

        self.distances = self.transform(self._candidates)

        random_pair = random.choice(self._candidates)
        exact_match = (random_pair[0], random_pair[0])
        self.fit_transform([exact_match, random_pair], [1, 0])

    def transform(self, pairs):
        return self.data_model.distances(pairs)

    def fit(self, X, y):

        self.y = numpy.array(y)
        self.X = X

        super().fit(self.X, self.y)

    def fit_transform(self, pairs, y):
        self.fit(self.transform(pairs), y)

    def pop(self) -> TrainingExample:
        if not len(self.candidates):
            raise IndexError("No more unlabeled examples to label")

        target_uncertainty = self._bias()

        probabilities = self.candidate_scores()

        distance_to_target = numpy.abs(target_uncertainty - probabilities)
        uncertain_index = distance_to_target.argmin()

        self.distances = numpy.delete(self.distances, uncertain_index, axis=0)

        uncertain_pair = self.candidates.pop(uncertain_index)

        return uncertain_pair

    def _remove(self, index):
        self.distances = numpy.delete(self.distances, index, axis=0)

    def mark(self, pairs, y):

        self.y = numpy.concatenate([self.y, y])
        self.X = numpy.vstack([self.X, self.transform(pairs)])

        self.fit(self.X, self.y)

    def _bias(self):
        positive = numpy.sum(self.y == 1)
        n_examples = len(self.y)

        bias = 1 - (positive / n_examples if positive else 0)

        # When we have just a few examples we are okay with getting
        # examples where the model strongly believes the example is
        # going to be positive or negative. As we get more examples,
        # prefer to ask for labels of examples the model is more
        # uncertain of.
        uncertainty_weight = min(positive, n_examples - positive)
        bias_weight = 10

        weighted_bias = 0.5 * uncertainty_weight + bias * bias_weight
        weighted_bias /= uncertainty_weight + bias_weight

        return weighted_bias

    def candidate_scores(self):
        return self.predict_proba(self.distances)[:, 1].reshape(-1, 1)

    def __len__(self):
        return len(self.candidates)

class BlockLearner(object):
    def __init__(self, data_model: datamodel.DataModel, *args):
        self.data_model = data_model

        self.current_predicates: tuple[Predicate, ...] = ()

        self._cached_labels: numpy.ndarray | None = None
        self._old_dupes: list[tuple[RecordDict, RecordDict]] = []

        self.block_learner: training.BlockLearner

    def fit_transform(self, pairs, y):
        dupes = [pair for label, pair in zip(y, pairs) if label]

        new_dupes = [pair for pair in dupes if pair not in self._old_dupes]
        new_uncovered = not all(self.predict(new_dupes))

        if new_uncovered:
            self.current_predicates = self.block_learner.learn(dupes, recall=1.0)
            self._cached_labels = None
            self._old_dupes = dupes

    def candidate_scores(self):
        if self._cached_labels is None:
            labels = self.predict(self.candidates)
            self._cached_labels = numpy.array(labels).reshape(-1, 1)

        return self._cached_labels

    def predict(self, candidates):
        labels = []
        for record_1, record_2 in candidates:

            for predicate in self.current_predicates:
                keys = predicate(record_2, target=True)
                if keys:
                    if set(predicate(record_1)) & set(keys):
                        labels.append(1)
                        break
            else:
                labels.append(0)

        return labels

    def _remove(self, index):
        if self._cached_labels is not None:
            self._cached_labels = numpy.delete(self._cached_labels, index, axis=0)

    def _sample_indices(self, sample_size: int) -> Iterable[tuple[RecordID, RecordID]]:

        weights: Dict[tuple[RecordID, RecordID], float] = {}
        for predicate, covered in self.block_learner.comparison_cover.items():
            # each predicate gets to vote for every record pair it covers. the
            # strength of that vote is in inverse proportion to the number of
            # records the predicate covers.
            #
            # if a predicate only covers a few record pairs, the value of
            # the vote it puts on those few pairs will be worth more than
            # a predicate that covers almost all the record pairs
            weight: float = 1 / len(covered)
            for pair in covered:
                weights[pair] = weights.get(pair, 0.0) + weight

        sample_ids: Iterable[tuple[RecordID, RecordID]]
        if sample_size < len(weights):
            # consider using a reservoir sampling strategy, which would
            # be more memory efficient and probably about as fast
            normalized_weights = numpy.fromiter(weights.values(), dtype=float) / sum(
                weights.values()
            )
            rng = numpy.random.default_rng()
            sample_indices = rng.choice(
                len(weights), size=sample_size, replace=False, p=normalized_weights
            )
            keys = list(weights.keys())
            sample_ids = ((keys[i][0], keys[i][1]) for i in sample_indices)
        else:
            sample_ids = weights.keys()

        return sample_ids

class DedupeBlockLearner(BlockLearner):
    def __init__(self, data_model: datamodel.DataModel, data, index_include):
        super().__init__(data_model)

        N_SAMPLED_RECORDS = 5000
        N_SAMPLED_RECORD_PAIRS = 10000

        index_data = Sample(data, 50000)
        sampled_records = Sample(index_data, N_SAMPLED_RECORDS)
        preds = self.data_model.predicates()

        self.block_learner = training.DedupeBlockLearner(
            preds, sampled_records, index_data
        )

        self.candidates = self._sample(sampled_records, N_SAMPLED_RECORD_PAIRS)
        examples_to_index = self.candidates.copy()

        if index_include:
            examples_to_index += index_include

        self._index_predicates(examples_to_index)

    def _index_predicates(self, candidates):

        blocker = self.block_learner.blocker

        records = core.unique((record for pair in candidates for record in pair))

        for field in blocker.index_fields:
            unique_fields = {record[field] for record in records}
            blocker.index(unique_fields, field)

        for pred in blocker.index_predicates:
            pred.freeze(records)

    def _sample(self, data, sample_size):

        sample_indices = self._sample_indices(sample_size)

        sample = [(data[id_1], data[id_2]) for id_1, id_2 in sample_indices]

        return sample

class RecordLinkBlockLearner(BlockLearner):
    def __init__(self, data_model, data_1, data_2, index_include):

        super().__init__(data_model)

        N_SAMPLED_RECORDS = 1000
        N_SAMPLED_RECORD_PAIRS = 5000

        sampled_records_1 = Sample(data_1, N_SAMPLED_RECORDS)
        index_data = Sample(data_2, 50000)
        sampled_records_2 = Sample(index_data, N_SAMPLED_RECORDS)

        preds = self.data_model.predicates(canopies=False)

        self.block_learner = training.RecordLinkBlockLearner(
            preds, sampled_records_1, sampled_records_2, index_data
        )

        self.candidates = self._sample(
            sampled_records_1, sampled_records_2, N_SAMPLED_RECORD_PAIRS
        )
        examples_to_index = self.candidates.copy()

        if index_include:
            examples_to_index += index_include

        self._index_predicates(examples_to_index)

    def _index_predicates(self, candidates):

        blocker = self.block_learner.blocker

        A, B = zip(*candidates)
        A = core.unique(A)
        B = core.unique(B)

        for field in blocker.index_fields:
            unique_fields = {record[field] for record in B}
            blocker.index(unique_fields, field)

        for pred in blocker.index_predicates:
            pred.freeze(A, B)

    def _sample(self, data_1, data_2, sample_size):

        sample_indices = self._sample_indices(sample_size)

        sample = [(data_1[id_1], data_2[id_2]) for id_1, id_2 in sample_indices]

        return sample

class DisagreementLearner(ActiveLearner):

    classifier: RLRLearner
    blocker: BlockLearner
    candidates: list[TrainingExample]

    def _common_init(self):

        self.learners = (self.classifier, self.blocker)
        self.y = numpy.array([])
        self.pairs = []

    def pop(self) -> TrainingExample:
        if not len(self.candidates):
            raise IndexError("No more unlabeled examples to label")

        probs_l = []
        for learner in self.learners:
            probabilities = learner.candidate_scores()
            probs_l.append(probabilities)

        probs = numpy.concatenate(probs_l, axis=1)

        # where do the classifers disagree?
        disagreement = numpy.std(probs > 0.5, axis=1).astype(bool)

        if disagreement.any():
            conflicts = disagreement.nonzero()[0]
            target = numpy.random.uniform(size=1)
            uncertain_index = conflicts[numpy.argmax(probs[conflicts][:, 0] - target)]
        else:
            uncertain_index = numpy.std(probs, axis=1).argmax()

        logger.debug(
            "Classifier: %.2f, Covered: %s",
            probs[uncertain_index][0],
            bool(probs[uncertain_index][1]),
        )

        uncertain_pair = self.candidates.pop(uncertain_index)

        for learner in self.learners:
            learner._remove(uncertain_index)

        return uncertain_pair

    def mark(self, pairs, y):

        self.y = numpy.concatenate([self.y, y])
        self.pairs.extend(pairs)

        for learner in self.learners:
            learner.fit_transform(self.pairs, self.y)

    def __len__(self):
        return len(self.candidates)

    def transform(self):
        pass

    def learn_predicates(self, recall, index_predicates):
        dupes = [pair for label, pair in zip(self.y, self.pairs) if label]

        if not index_predicates:
            old_preds = self.blocker.block_learner.blocker.predicates.copy()

            no_index_predicates = [
                pred for pred in old_preds if not hasattr(pred, "index")
            ]
            self.blocker.block_learner.blocker.predicates = no_index_predicates

            learned_preds = self.blocker.block_learner.learn(
                dupes, recall=recall, candidate_types="random forest"
            )

            self.blocker.block_learner.blocker.predicates = old_preds

        else:
            learned_preds = self.blocker.block_learner.learn(
                dupes, recall=recall, candidate_types="random forest"
            )

        return learned_preds

class DedupeDisagreementLearner(DisagreementLearner):
    def __init__(
        self, data_model, data, blocked_proportion, sample_size, index_include
    ):

        self.data_model = data_model

        data = core.index(data)

        random_pair = (
            random.choice(list(data.values())),
            random.choice(list(data.values())),
        )
        exact_match = (random_pair[0], random_pair[0])

        index_include = index_include.copy()
        index_include.append(exact_match)

        self.blocker = DedupeBlockLearner(data_model, data, index_include)

        self.candidates = self.blocker.candidates

        self.classifier = RLRLearner(self.data_model)
        self.classifier.candidates = self.candidates

        self._common_init()

        self.mark([exact_match] * 4 + [random_pair], [1] * 4 + [0])

class RecordLinkDisagreementLearner(DisagreementLearner):
    def __init__(
        self, data_model, data_1, data_2, blocked_proportion, sample_size, index_include
    ):

        self.data_model = data_model

        data_1 = core.index(data_1)

        offset = len(data_1)
        data_2 = core.index(data_2, offset)

        random_pair = (
            random.choice(list(data_1.values())),
            random.choice(list(data_2.values())),
        )
        exact_match = (random_pair[0], random_pair[0])

        index_include = index_include.copy()
        index_include.append(exact_match)

        self.blocker = RecordLinkBlockLearner(data_model, data_1, data_2, index_include)
        self.candidates = self.blocker.candidates

        self.classifier = RLRLearner(self.data_model)
        self.classifier.candidates = self.candidates

        self._common_init()

        self.mark([exact_match] * 4 + [random_pair], [1] * 4 + [0])

class Sample(dict):
    def __init__(self, d, sample_size):
        if len(d) <= sample_size:
            super().__init__(d)
        else:
            sample = random.sample(d.keys(), sample_size)
            super().__init__({k: d[k] for k in sample})
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
bin/restart.py

Find the current scgi process id and kill that pid's child processes
as a "gentle" restart for SCGI.  Should be smart enough only to kill
children of the local scgi_handler running at pid named in './scgi.pid'.

for an explanation of how it works in SCGI see

  http://mail.mems-exchange.org/pipermail/quixote-users/2003-February/001243.html

save_popen() taken from a python cookbook recipe comment at

  http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52296

"""

import os, tempfile

def save_popen (command):
    """This is a deadlock save version of popen2 (no stdin), that returns
    an object with errorlevel,out, and err"""

    outfile=tempfile.mktemp()
    errfile=tempfile.mktemp()
    errorlevel=os.system("( %s ) > %s 2> %s" %
        (command,outfile,errfile)) >> 8
    out=open(outfile,"r").read()
    err=open(errfile,"r").read()
    os.remove(outfile)
    os.remove(errfile)
    return out.strip()

my_pid = open('scgi.pid').read().strip()

# 'grep python' because the first grep might show up in the list
pid_set = save_popen("""ps -f | grep scgi_handler | grep python | awk '{print $2, $3}'""")
pid_lines = pid_set.split('\n')

for line in pid_lines:
    pid, ppid = line.split(' ')
    if pid == my_pid:
        print 'Finding children for pid', pid
    elif ppid == my_pid:
        print 'Killing child pid', pid
        os.system('kill %s' % pid)
<commit_msg>Use python os.kill(pid, sig), rather than os.system('kill -HUP %s')<commit_after>#!/usr/bin/env python

"""
bin/restart.py

Find the current scgi process id and kill that pid's child processes
as a "gentle" restart for SCGI.  Should be smart enough only to kill
children of the local scgi_handler running at pid named in './scgi.pid'.

for an explanation of how it works in SCGI see

  http://mail.mems-exchange.org/pipermail/quixote-users/2003-February/001243.html

save_popen() taken from a python cookbook recipe comment at

  http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52296

"""

import os, signal, tempfile

def save_popen (command):
    """This is a deadlock save version of popen2 (no stdin), that returns
    an object with errorlevel,out, and err"""

    outfile=tempfile.mktemp()
    errfile=tempfile.mktemp()
    errorlevel=os.system("( %s ) > %s 2> %s" %
        (command,outfile,errfile)) >> 8
    out=open(outfile,"r").read()
    err=open(errfile,"r").read()
    os.remove(outfile)
    os.remove(errfile)
    return out.strip()

my_pid = open('scgi.pid').read().strip()

# 'grep python' because the first grep might show up in the list
pid_set = save_popen("""ps -f | grep scgi_handler | grep python | awk '{print $2, $3}'""")
pid_lines = pid_set.split('\n')

for line in pid_lines:
    pid, ppid = line.split(' ')
    if pid == my_pid:
        print 'Finding children for pid', pid
    elif ppid == my_pid:
        print 'Killing child pid', pid
        os.kill(int(pid), signal.SIGHUP)
<|endoftext|>


<commit_before>#
# Tests the AnalyserEquationAst class bindings
#
import unittest

class AnalyserEquationAstTestCase(unittest.TestCase):

    def test_create_destroy(self):
        from libcellml import AnalyserEquationAst

        x = AnalyserEquationAst()
        del x

    def test_analyser_equation_ast(self):
        from libcellml import AnalyserEquationAst

        # Create an equation AST and check its default settings.
        ast = AnalyserEquationAst()
        self.assertEqual(AnalyserEquationAst.Type.ASSIGNMENT, ast.type())
        self.assertEqual('', ast.value())
        self.assertIsNone(ast.variable())
        self.assertIsNone(ast.parent())
        self.assertIsNone(ast.leftChild())
        self.assertIsNone(ast.rightChild())

        del ast

if __name__ == '__main__':
    unittest.main()
<commit_msg>Some minor cleaning up.<commit_after>#
# Tests the AnalyserEquationAst class bindings
#
import unittest

class AnalyserEquationAstTestCase(unittest.TestCase):

    def test_create_destroy(self):
        from libcellml import AnalyserEquationAst

        x = AnalyserEquationAst()
        del x

    def test_analyser_equation_ast(self):
        from libcellml import AnalyserEquationAst

        # Create an equation AST and check its default settings.
        ast = AnalyserEquationAst()
        self.assertEqual(AnalyserEquationAst.Type.ASSIGNMENT, ast.type())
        self.assertEqual('', ast.value())
        self.assertIsNone(ast.variable())
        self.assertIsNone(ast.parent())
        self.assertIsNone(ast.leftChild())
        self.assertIsNone(ast.rightChild())

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>import unittest

import cupy
from cupy.random import distributions
from cupy import testing

import numpy

_regular_float_dtypes = (numpy.float64, numpy.float32)
_float_dtypes = _regular_float_dtypes + (numpy.float16,)
_signed_dtypes = tuple(numpy.dtype(i).type for i in 'bhilq')
_unsigned_dtypes = tuple(numpy.dtype(i).type for i in 'BHILQ')
_int_dtypes = _signed_dtypes + _unsigned_dtypes

class RandomDistributionsTestCase(unittest.TestCase):
    def check_distribution(self, dist_name, params, dtype):
        cp_params = {k: cupy.asarray(params[k]) for k in params}
        np_out = numpy.random.__dict__[dist_name](
            size=self.shape, **params).astype(dtype)
        cp_out = distributions.__dict__[dist_name](
            size=self.shape, dtype=dtype, **cp_params)
        self.assertEqual(cp_out.shape, np_out.shape)
        self.assertEqual(cp_out.dtype, np_out.dtype)

@testing.parameterize(*testing.product({
    'shape': [(4, 3, 2), (3, 2)],
    'a_shape': [(), (3, 2)],
    'b_shape': [(), (3, 2)],
    'dtype': _float_dtypes,  # to escape timeout
})
)
@testing.gpu
class TestDistributionsBeta(RandomDistributionsTestCase):

    @cupy.testing.for_float_dtypes('a_dtype')
    @cupy.testing.for_float_dtypes('b_dtype')
    def test_beta(self, a_dtype, b_dtype):
        a = 3. * numpy.ones(self.a_shape, dtype=a_dtype)
        b = 3. * numpy.ones(self.b_shape, dtype=b_dtype)
        self.check_distribution('beta',
                                {'a': a, 'b': b}, self.dtype)

@testing.parameterize(*testing.product({
    'shape': [(4, 3, 2), (3, 2)],
    'n_shape': [(), (3, 2)],
    'p_shape': [(), (3, 2)],
    'dtype': _int_dtypes,  # to escape timeout
})
)
@testing.gpu
class TestDistributionsBinomial(RandomDistributionsTestCase):

    @cupy.testing.for_signed_dtypes('n_dtype')
    @cupy.testing.for_float_dtypes('p_dtype')
    def test_binomial(self, n_dtype, p_dtype):
        n = 5 * numpy.ones(self.n_shape, dtype=n_dtype)
        p = 0.5 * numpy.ones(self.p_shape, dtype=p_dtype)
        self.check_distribution('binomial',
                                {'n': n, 'p': p}, self.dtype)

@testing.parameterize(*testing.product({
    'shape': [(4, 3, 2, 3), (3, 2, 3)],
    'alpha_shape': [(3,)],
})
)
@testing.gpu
class TestDistributionsDirichlet(RandomDistributionsTestCase):

    @cupy.testing.for_float_dtypes('alpha_dtype')
    @cupy.testing.for_float_dtypes('dtype')
    def test_dirichlet(self, alpha_dtype, dtype):
        alpha = numpy.ones(self.alpha_shape, dtype=alpha_dtype)
        self.check_distribution('dirichlet',
                                {'alpha': alpha}, dtype)

@testing.parameterize(*testing.product({
    'shape': [(4, 3, 2), (3, 2)],
    'loc_shape': [(), (3, 2)],
    'scale_shape': [(), (3, 2)],
})
)
@testing.gpu
class TestDistributionsGumbel(RandomDistributionsTestCase):

    @cupy.testing.for_float_dtypes('dtype', no_float16=True)
    @cupy.testing.for_float_dtypes('loc_dtype')
    @cupy.testing.for_float_dtypes('scale_dtype')
    def test_gumbel(self, loc_dtype, scale_dtype, dtype):
        loc = numpy.ones(self.loc_shape, dtype=loc_dtype)
        scale = numpy.ones(self.scale_shape, dtype=scale_dtype)
        self.check_distribution('gumbel',
                                {'loc': loc, 'scale': scale}, dtype)

@testing.parameterize(*testing.product({
    'shape': [(4, 3, 2), (3, 2)],
    'loc_shape': [(), (3, 2)],
    'scale_shape': [(), (3, 2)],
})
)
@testing.gpu
class TestDistributionsLaplace(RandomDistributionsTestCase):

    @cupy.testing.for_float_dtypes('dtype', no_float16=True)
    @cupy.testing.for_float_dtypes('loc_dtype')
    @cupy.testing.for_float_dtypes('scale_dtype')
    def test_laplace(self, loc_dtype, scale_dtype, dtype):
        loc = numpy.ones(self.loc_shape, dtype=loc_dtype)
        scale = numpy.ones(self.scale_shape, dtype=scale_dtype)
        self.check_distribution('laplace',
                                {'loc': loc, 'scale': scale}, dtype)

@testing.parameterize(*testing.product({
    'shape': [(4, 3, 2), (3, 2)],
    'mean_shape': [()],
    'sigma_shape': [()],
})
)
@testing.gpu
class TestDistributionsLognormal(RandomDistributionsTestCase):

    @cupy.testing.for_float_dtypes('dtype', no_float16=True)
    @cupy.testing.for_float_dtypes('mean_dtype')
    @cupy.testing.for_float_dtypes('sigma_dtype')
    def test_lognormal(self, mean_dtype, sigma_dtype, dtype):
        mean = numpy.ones(self.mean_shape, dtype=mean_dtype)
        sigma = numpy.ones(self.sigma_shape, dtype=sigma_dtype)
        self.check_distribution('lognormal',
                                {'mean': mean, 'sigma': sigma}, dtype)

@testing.parameterize(*testing.product({
    'shape': [(4, 3, 2), (3, 2)],
    'loc_shape': [(), (3, 2)],
    'scale_shape': [(), (3, 2)],
})
)
@testing.gpu
class TestDistributionsNormal(RandomDistributionsTestCase):

    @cupy.testing.for_float_dtypes('dtype', no_float16=True)
    @cupy.testing.for_float_dtypes('loc_dtype')
    @cupy.testing.for_float_dtypes('scale_dtype')
    def test_normal(self, loc_dtype, scale_dtype, dtype):
        loc = numpy.ones(self.loc_shape, dtype=loc_dtype)
        scale = numpy.ones(self.scale_shape, dtype=scale_dtype)
        self.check_distribution('normal',
                                {'loc': loc, 'scale': scale}, dtype)

@testing.parameterize(*testing.product({
    'shape': [(4, 3, 2), (3, 2)],
})
)
@testing.gpu
class TestDistributionsStandardNormal(RandomDistributionsTestCase):

    @cupy.testing.for_float_dtypes('dtype', no_float16=True)
    def test_standardnormal(self, dtype):
        self.check_distribution('standard_normal', {}, dtype)

@testing.parameterize(*testing.product({
    'shape': [(4, 3, 2), (3, 2)],
    'low_shape': [(), (3, 2)],
    'high_shape': [(), (3, 2)],
})
)
@testing.gpu
class TestDistributionsUniform(RandomDistributionsTestCase):

    @cupy.testing.for_float_dtypes('dtype', no_float16=True)
    @cupy.testing.for_float_dtypes('low_dtype')
    @cupy.testing.for_float_dtypes('high_dtype')
    def test_uniform(self, low_dtype, high_dtype, dtype):
        low = numpy.ones(self.low_shape, dtype=low_dtype)
        high = numpy.ones(self.high_shape, dtype=high_dtype) * 2.
        self.check_distribution('uniform',
                                {'low': low, 'high': high}, dtype)
<commit_msg>[fix] based on the comment<commit_after>import unittest

import cupy
from cupy.random import distributions
from cupy import testing

import numpy

_regular_float_dtypes = (numpy.float64, numpy.float32)
_float_dtypes = _regular_float_dtypes + (numpy.float16,)
_signed_dtypes = tuple(numpy.dtype(i).type for i in 'bhilq')
_unsigned_dtypes = tuple(numpy.dtype(i).type for i in 'BHILQ')
_int_dtypes = _signed_dtypes + _unsigned_dtypes

class RandomDistributionsTestCase(unittest.TestCase):
    def check_distribution(self, dist_name, params, dtype):
        cp_params = {k: cupy.asarray(params[k]) for k in params}
        np_out = getattr(numpy.random, dist_name)(
            size=self.shape, **params).astype(dtype)
        cp_out = getattr(distributions, dist_name)(
            size=self.shape, dtype=dtype, **cp_params)
        self.assertEqual(cp_out.shape, np_out.shape)
        self.assertEqual(cp_out.dtype, np_out.dtype)

@testing.parameterize(*testing.product({
    'shape': [(4, 3, 2), (3, 2)],
    'a_shape': [(), (3, 2)],
    'b_shape': [(), (3, 2)],
    'dtype': _float_dtypes,  # to escape timeout
})
)
@testing.gpu
class TestDistributionsBeta(RandomDistributionsTestCase):

    @cupy.testing.for_float_dtypes('a_dtype')
    @cupy.testing.for_float_dtypes('b_dtype')
    def test_beta(self, a_dtype, b_dtype):
        a = 3. * numpy.ones(self.a_shape, dtype=a_dtype)
        b = 3. * numpy.ones(self.b_shape, dtype=b_dtype)
        self.check_distribution('beta',
                                {'a': a, 'b': b}, self.dtype)

@testing.parameterize(*testing.product({
    'shape': [(4, 3, 2), (3, 2)],
    'n_shape': [(), (3, 2)],
    'p_shape': [(), (3, 2)],
    'dtype': _int_dtypes,  # to escape timeout
})
)
@testing.gpu
class TestDistributionsBinomial(RandomDistributionsTestCase):

    @cupy.testing.for_signed_dtypes('n_dtype')
    @cupy.testing.for_float_dtypes('p_dtype')
    def test_binomial(self, n_dtype, p_dtype):
        n = 5 * numpy.ones(self.n_shape, dtype=n_dtype)
        p = 0.5 * numpy.ones(self.p_shape, dtype=p_dtype)
        self.check_distribution('binomial',
                                {'n': n, 'p': p}, self.dtype)

@testing.parameterize(*testing.product({
    'shape': [(4, 3, 2, 3), (3, 2, 3)],
    'alpha_shape': [(3,)],
})
)
@testing.gpu
class TestDistributionsDirichlet(RandomDistributionsTestCase):

    @cupy.testing.for_float_dtypes('alpha_dtype')
    @cupy.testing.for_float_dtypes('dtype')
    def test_dirichlet(self, alpha_dtype, dtype):
        alpha = numpy.ones(self.alpha_shape, dtype=alpha_dtype)
        self.check_distribution('dirichlet',
                                {'alpha': alpha}, dtype)

@testing.parameterize(*testing.product({
    'shape': [(4, 3, 2), (3, 2)],
    'loc_shape': [(), (3, 2)],
    'scale_shape': [(), (3, 2)],
})
)
@testing.gpu
class TestDistributionsGumbel(RandomDistributionsTestCase):

    @cupy.testing.for_float_dtypes('dtype', no_float16=True)
    @cupy.testing.for_float_dtypes('loc_dtype')
    @cupy.testing.for_float_dtypes('scale_dtype')
    def test_gumbel(self, loc_dtype, scale_dtype, dtype):
        loc = numpy.ones(self.loc_shape, dtype=loc_dtype)
        scale = numpy.ones(self.scale_shape, dtype=scale_dtype)
        self.check_distribution('gumbel',
                                {'loc': loc, 'scale': scale}, dtype)

@testing.parameterize(*testing.product({
    'shape': [(4, 3, 2), (3, 2)],
    'loc_shape': [(), (3, 2)],
    'scale_shape': [(), (3, 2)],
})
)
@testing.gpu
class TestDistributionsLaplace(RandomDistributionsTestCase):

    @cupy.testing.for_float_dtypes('dtype', no_float16=True)
    @cupy.testing.for_float_dtypes('loc_dtype')
    @cupy.testing.for_float_dtypes('scale_dtype')
    def test_laplace(self, loc_dtype, scale_dtype, dtype):
        loc = numpy.ones(self.loc_shape, dtype=loc_dtype)
        scale = numpy.ones(self.scale_shape, dtype=scale_dtype)
        self.check_distribution('laplace',
                                {'loc': loc, 'scale': scale}, dtype)

@testing.parameterize(*testing.product({
    'shape': [(4, 3, 2), (3, 2)],
    'mean_shape': [()],
    'sigma_shape': [()],
})
)
@testing.gpu
class TestDistributionsLognormal(RandomDistributionsTestCase):

    @cupy.testing.for_float_dtypes('dtype', no_float16=True)
    @cupy.testing.for_float_dtypes('mean_dtype')
    @cupy.testing.for_float_dtypes('sigma_dtype')
    def test_lognormal(self, mean_dtype, sigma_dtype, dtype):
        mean = numpy.ones(self.mean_shape, dtype=mean_dtype)
        sigma = numpy.ones(self.sigma_shape, dtype=sigma_dtype)
        self.check_distribution('lognormal',
                                {'mean': mean, 'sigma': sigma}, dtype)

@testing.parameterize(*testing.product({
    'shape': [(4, 3, 2), (3, 2)],
    'loc_shape': [(), (3, 2)],
    'scale_shape': [(), (3, 2)],
})
)
@testing.gpu
class TestDistributionsNormal(RandomDistributionsTestCase):

    @cupy.testing.for_float_dtypes('dtype', no_float16=True)
    @cupy.testing.for_float_dtypes('loc_dtype')
    @cupy.testing.for_float_dtypes('scale_dtype')
    def test_normal(self, loc_dtype, scale_dtype, dtype):
        loc = numpy.ones(self.loc_shape, dtype=loc_dtype)
        scale = numpy.ones(self.scale_shape, dtype=scale_dtype)
        self.check_distribution('normal',
                                {'loc': loc, 'scale': scale}, dtype)

@testing.parameterize(*testing.product({
    'shape': [(4, 3, 2), (3, 2)],
})
)
@testing.gpu
class TestDistributionsStandardNormal(RandomDistributionsTestCase):

    @cupy.testing.for_float_dtypes('dtype', no_float16=True)
    def test_standardnormal(self, dtype):
        self.check_distribution('standard_normal', {}, dtype)

@testing.parameterize(*testing.product({
    'shape': [(4, 3, 2), (3, 2)],
    'low_shape': [(), (3, 2)],
    'high_shape': [(), (3, 2)],
})
)
@testing.gpu
class TestDistributionsUniform(RandomDistributionsTestCase):

    @cupy.testing.for_float_dtypes('dtype', no_float16=True)
    @cupy.testing.for_float_dtypes('low_dtype')
    @cupy.testing.for_float_dtypes('high_dtype')
    def test_uniform(self, low_dtype, high_dtype, dtype):
        low = numpy.ones(self.low_shape, dtype=low_dtype)
        high = numpy.ones(self.high_shape, dtype=high_dtype) * 2.
        self.check_distribution('uniform',
                                {'low': low, 'high': high}, dtype)
<|endoftext|>


<commit_before>from __future__ import absolute_import

from bokeh.io import save
from bokeh.plotting import figure
from bokeh.models import CustomJS, Range1d
from selenium.webdriver.common.action_chains import ActionChains

import pytest
pytestmark = pytest.mark.integration

def make_pan_plot_with_callback(range_min=None, range_max=None):
    x_range = Range1d(0, 3, bounds=(range_min, range_max))
    x_callback = CustomJS(args=dict(x_range=x_range), code="""
        window.get_x_range_start = function() {
            return x_range.get('start');
        }
        window.get_x_range_end = function() {
            return x_range.get('end');
        }
    """)
    x_range.callback = x_callback

    y_range = Range1d(0, 3, bounds=(range_min, range_max))
    y_callback = CustomJS(args=dict(y_range=y_range), code="""
        window.get_y_range_start = function() {
            return y_range.get('start');
        }
        window.get_y_range_end = function() {
            return y_range.get('end');
        }
    """)
    y_range.callback = y_callback

    plot = figure(
        height=400, width=400, tools='pan,reset', x_range=x_range, y_range=y_range
    )
    plot.min_border = 0
    plot.rect(x=[1, 2], y=[1, 1], width=0.9, height=0.9)
    return plot

def pan_plot(selenium, pan_x=None, pan_y=None):
    canvas = selenium.find_element_by_tag_name('canvas')
    actions = ActionChains(selenium)
    actions.move_to_element_with_offset(canvas, 200, 200)
    actions.click_and_hold()
    actions.move_by_offset(pan_x, pan_y)
    actions.release()
    actions.perform()

def test_range_with_callback_triggers_alert(output_file_url, selenium):
    # Simple test to ensure range callbacks are working
    # Rest of tests in this file depend on range callback.

    plot = make_pan_plot_with_callback()
    initial_start = plot.x_range.start
    save(plot)
    selenium.get(output_file_url)

    # Pan plot and test for new range value
    pan_plot(selenium, pan_x=100, pan_y=100)
    new_range_start = float(selenium.execute_script("""alert(window.get_x_range_start())"""))
    selenium.switch_to_alert().dismiss()  # This is not necessary but assists debugging
    assert new_range_start < initial_start

def test_x_range_does_not_pan_left_of_x_min(output_file_url, selenium):
    x_range_min = -1
    plot = make_pan_plot_with_callback(range_min=x_range_min)
    save(plot)
    selenium.get(output_file_url)

    # Pan plot and test for new range value
    pan_plot(selenium, pan_x=200, pan_y=0)
    new_range_start = float(selenium.execute_script("""alert(window.get_x_range_start())"""))
    selenium.switch_to_alert().dismiss()  # This is not necessary but assists debugging
    assert new_range_start == x_range_min

def test_x_range_does_not_pan_right_of_x_max(output_file_url, selenium):
    x_range_max = 4
    plot = make_pan_plot_with_callback(range_max=x_range_max)
    save(plot)
    selenium.get(output_file_url)

    # Pan plot and test for new range value
    pan_plot(selenium, pan_x=-200, pan_y=0)
    new_range_end = float(selenium.execute_script("""alert(window.get_x_range_end())"""))
    selenium.switch_to_alert().dismiss()  # This is not necessary but assists debugging
    assert new_range_end == x_range_max

def test_y_range_does_not_pan_below_y_min(output_file_url, selenium):
    y_range_min = -1
    plot = make_pan_plot_with_callback(range_min=y_range_min)
    save(plot)
    selenium.get(output_file_url)

    # Pan plot and test for new range value
    pan_plot(selenium, pan_x=50, pan_y=-150)
    new_range_start = float(selenium.execute_script("""alert(window.get_y_range_start())"""))
    selenium.switch_to_alert().dismiss()  # This is not necessary but assists debugging
    assert new_range_start == y_range_min

def test_y_range_does_not_pan_above_y_max(output_file_url, selenium):
    y_range_max = 4
    plot = make_pan_plot_with_callback(range_max=y_range_max)
    save(plot)
    selenium.get(output_file_url)

    # Pan plot and test for new range value
    pan_plot(selenium, pan_x=50, pan_y=150)
    new_range_end = float(selenium.execute_script("""alert(window.get_y_range_end())"""))
    selenium.switch_to_alert().dismiss()  # This is not necessary but assists debugging
    assert new_range_end == y_range_max
<commit_msg>Tweak name of test method<commit_after>from __future__ import absolute_import

from bokeh.io import save
from bokeh.plotting import figure
from bokeh.models import CustomJS, Range1d
from selenium.webdriver.common.action_chains import ActionChains

import pytest
pytestmark = pytest.mark.integration

def make_range1d_pan_plot_with_callback(range_min=None, range_max=None):
    x_range = Range1d(0, 3, bounds=(range_min, range_max))
    x_callback = CustomJS(args=dict(x_range=x_range), code="""
        window.get_x_range_start = function() {
            return x_range.get('start');
        }
        window.get_x_range_end = function() {
            return x_range.get('end');
        }
    """)
    x_range.callback = x_callback

    y_range = Range1d(0, 3, bounds=(range_min, range_max))
    y_callback = CustomJS(args=dict(y_range=y_range), code="""
        window.get_y_range_start = function() {
            return y_range.get('start');
        }
        window.get_y_range_end = function() {
            return y_range.get('end');
        }
    """)
    y_range.callback = y_callback

    plot = figure(
        height=400, width=400, tools='pan,reset', x_range=x_range, y_range=y_range
    )
    plot.min_border = 0
    plot.rect(x=[1, 2], y=[1, 1], width=0.9, height=0.9)
    return plot

def pan_plot(selenium, pan_x=None, pan_y=None):
    canvas = selenium.find_element_by_tag_name('canvas')
    actions = ActionChains(selenium)
    actions.move_to_element_with_offset(canvas, 200, 200)
    actions.click_and_hold()
    actions.move_by_offset(pan_x, pan_y)
    actions.release()
    actions.perform()

def test_range_with_callback_triggers_alert(output_file_url, selenium):
    # Simple test to ensure range callbacks are working
    # Rest of tests in this file depend on range callback.

    plot = make_range1d_pan_plot_with_callback()
    initial_start = plot.x_range.start
    save(plot)
    selenium.get(output_file_url)

    # Pan plot and test for new range value
    pan_plot(selenium, pan_x=100, pan_y=100)
    new_range_start = float(selenium.execute_script("""alert(window.get_x_range_start())"""))
    selenium.switch_to_alert().dismiss()  # This is not necessary but assists debugging
    assert new_range_start < initial_start

def test_x_range_does_not_pan_left_of_x_min(output_file_url, selenium):
    x_range_min = -1
    plot = make_range1d_pan_plot_with_callback(range_min=x_range_min)
    save(plot)
    selenium.get(output_file_url)

    # Pan plot and test for new range value
    pan_plot(selenium, pan_x=200, pan_y=0)
    new_range_start = float(selenium.execute_script("""alert(window.get_x_range_start())"""))
    selenium.switch_to_alert().dismiss()  # This is not necessary but assists debugging
    assert new_range_start == x_range_min

def test_x_range_does_not_pan_right_of_x_max(output_file_url, selenium):
    x_range_max = 4
    plot = make_range1d_pan_plot_with_callback(range_max=x_range_max)
    save(plot)
    selenium.get(output_file_url)

    # Pan plot and test for new range value
    pan_plot(selenium, pan_x=-200, pan_y=0)
    new_range_end = float(selenium.execute_script("""alert(window.get_x_range_end())"""))
    selenium.switch_to_alert().dismiss()  # This is not necessary but assists debugging
    assert new_range_end == x_range_max

def test_y_range_does_not_pan_below_y_min(output_file_url, selenium):
    y_range_min = -1
    plot = make_range1d_pan_plot_with_callback(range_min=y_range_min)
    save(plot)
    selenium.get(output_file_url)

    # Pan plot and test for new range value
    pan_plot(selenium, pan_x=50, pan_y=-150)
    new_range_start = float(selenium.execute_script("""alert(window.get_y_range_start())"""))
    selenium.switch_to_alert().dismiss()  # This is not necessary but assists debugging
    assert new_range_start == y_range_min

def test_y_range_does_not_pan_above_y_max(output_file_url, selenium):
    y_range_max = 4
    plot = make_range1d_pan_plot_with_callback(range_max=y_range_max)
    save(plot)
    selenium.get(output_file_url)

    # Pan plot and test for new range value
    pan_plot(selenium, pan_x=50, pan_y=150)
    new_range_end = float(selenium.execute_script("""alert(window.get_y_range_end())"""))
    selenium.switch_to_alert().dismiss()  # This is not necessary but assists debugging
    assert new_range_end == y_range_max
<|endoftext|>


<commit_before>import models
import forms
import tables
import field_filters

from django.views.generic import DetailView, FormView
from django.utils.translation import ugettext_lazy as _
from django.http.response import HttpResponseRedirect

from urljects import U, URLView, pk
from dal import autocomplete

from core import generic_views
from comments.views import CommentViewGeneric

class PublisherView(generic_views.LoginMixin):
    view_name = 'publishers'
    model = models.Publisher

class AddPublisher(PublisherView, FormView, URLView):
    form_class = forms.PublisherForm
    view_name = 'publisher_add'
    template_name = 'add_form.html'
    title = _('Add publisher')

    url = U / 'add'
    url_name = 'add'

    def form_valid(self, form):
        publisher, contact = form.save()
        return HttpResponseRedirect(publisher.get_absolute_url())

class Detail(PublisherView, DetailView, CommentViewGeneric, URLView):
    template_name = 'publisher.html'

    url = U / pk / 'detail'
    url_name = 'detail'

class Edit(PublisherView, generic_views.EditView, URLView):
    form_class = forms.PublisherEditForm

    url = U / pk / 'edit'
    url_name = 'edit'

class History(PublisherView, generic_views.HistoryView, URLView):
    """
        History of changes to publishers
    """

    url = U / pk / 'history'
    url_name = 'history'

class ListView(PublisherView, generic_views.FilteredListView, URLView):
    title = _('Publishers')
    table_class = tables.PublisherTable
    filter_class = field_filters.PublisherFilter

    url = U
    url_name = 'list'

class EditContacts(PublisherView, FormView, generic_views.ObjectMixinFixed, URLView):  # noqa
    form_class = forms.ContactFormset
    template_name = 'formset_verbose.html'
    title = _('Edit contacts')

    url = U / pk / 'contacts'
    url_name = 'edit_contacts'

    def get_form_kwargs(self):
        kwargs = super(EditContacts, self).get_form_kwargs()
        kwargs['queryset'] = self.object.contactperson_set.all()
        return kwargs

    def form_valid(self, form):
        contacts = form.save(commit=False)
        for contact in contacts:
            contact.publisher = self.object
            contact.save()
        for obj in form.deleted_objects:
            obj.delete()

        return HttpResponseRedirect(self.object.get_absolute_url())

class PublisherAutocomplete(autocomplete.Select2QuerySetView, URLView):
    url_name = 'autocomplete'
    url = U / 'autocomplete'

    def get_queryset(self):
        if not self.request.user.is_authenticated():
            return models.Publisher.objects.none()
        qs = models.Publisher.objects.all()
        if self.q:
            qs = qs.filter(name__istartswith=self.q)
        return qs

class PublisherContactAutocomplete(autocomplete.Select2QuerySetView, URLView):
    url_name = 'contact_autocomplete'
    url = U / 'contact_autocomplete'

    def get_queryset(self):
        if not self.request.user.is_authenticated():
            return models.ContactPerson.objects.none()
        qs = models.ContactPerson.objects.all()
        publisher = self.forwarded.get('publisher', None)
        if publisher:
            qs = qs.filter(publisher=publisher)

        if self.q:
            qs = qs.filter(name__istartswith=self.q)
        return qs
<commit_msg>Change publisher autocomplete to icontains filter<commit_after>import models
import forms
import tables
import field_filters

from django.views.generic import DetailView, FormView
from django.utils.translation import ugettext_lazy as _
from django.http.response import HttpResponseRedirect

from urljects import U, URLView, pk
from dal import autocomplete

from core import generic_views
from comments.views import CommentViewGeneric

class PublisherView(generic_views.LoginMixin):
    view_name = 'publishers'
    model = models.Publisher

class AddPublisher(PublisherView, FormView, URLView):
    form_class = forms.PublisherForm
    view_name = 'publisher_add'
    template_name = 'add_form.html'
    title = _('Add publisher')

    url = U / 'add'
    url_name = 'add'

    def form_valid(self, form):
        publisher, contact = form.save()
        return HttpResponseRedirect(publisher.get_absolute_url())

class Detail(PublisherView, DetailView, CommentViewGeneric, URLView):
    template_name = 'publisher.html'

    url = U / pk / 'detail'
    url_name = 'detail'

class Edit(PublisherView, generic_views.EditView, URLView):
    form_class = forms.PublisherEditForm

    url = U / pk / 'edit'
    url_name = 'edit'

class History(PublisherView, generic_views.HistoryView, URLView):
    """
        History of changes to publishers
    """

    url = U / pk / 'history'
    url_name = 'history'

class ListView(PublisherView, generic_views.FilteredListView, URLView):
    title = _('Publishers')
    table_class = tables.PublisherTable
    filter_class = field_filters.PublisherFilter

    url = U
    url_name = 'list'

class EditContacts(PublisherView, FormView, generic_views.ObjectMixinFixed, URLView):  # noqa
    form_class = forms.ContactFormset
    template_name = 'formset_verbose.html'
    title = _('Edit contacts')

    url = U / pk / 'contacts'
    url_name = 'edit_contacts'

    def get_form_kwargs(self):
        kwargs = super(EditContacts, self).get_form_kwargs()
        kwargs['queryset'] = self.object.contactperson_set.all()
        return kwargs

    def form_valid(self, form):
        contacts = form.save(commit=False)
        for contact in contacts:
            contact.publisher = self.object
            contact.save()
        for obj in form.deleted_objects:
            obj.delete()

        return HttpResponseRedirect(self.object.get_absolute_url())

class PublisherAutocomplete(autocomplete.Select2QuerySetView, URLView):
    url_name = 'autocomplete'
    url = U / 'autocomplete'

    def get_queryset(self):
        if not self.request.user.is_authenticated():
            return models.Publisher.objects.none()
        qs = models.Publisher.objects.all()
        if self.q:
            qs = qs.filter(name__icontains=self.q)
        return qs

class PublisherContactAutocomplete(autocomplete.Select2QuerySetView, URLView):
    url_name = 'contact_autocomplete'
    url = U / 'contact_autocomplete'

    def get_queryset(self):
        if not self.request.user.is_authenticated():
            return models.ContactPerson.objects.none()
        qs = models.ContactPerson.objects.all()
        publisher = self.forwarded.get('publisher', None)
        if publisher:
            qs = qs.filter(publisher=publisher)

        if self.q:
            qs = qs.filter(name__icontains=self.q)
        return qs
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import json
import os
from os import environ
from os.path import exists

import pytest

from pyscaffold.api import create_project

pytestmark = [pytest.mark.slow, pytest.mark.system]

@pytest.fixture(autouse=True)
def dont_load_dotenv():
    """pytest-virtualenv creates a `.env` directory by default, but `.env`
    entries in the file system are loaded by Pipenv as dotenv files.

    To prevent errors for happening we have to disable this feature.

    Additionally, it seems that env vars have to be changed before using
    venv, so an autouse fixture is required (cannot put this part in the
    beginning of the test function.
    """
    environ['PIPENV_DONT_LOAD_ENV'] = '1'

@pytest.fixture(autouse=True)
def cwd(tmpdir):
    """Guarantee a blank folder as workspace"""

    with tmpdir.as_cwd():
        yield tmpdir

def test_pipenv_works_with_pyscaffold(cwd, venv_path, venv_run):
    def win10_decorator(func):
        """pipenv has colored output even if PIPENV_COLORBLIND=1

        We pipe to /dev/null to avoid strange color symbols in output
        """
        def wrapper(cmd, **kwargs):
            if cmd.startswith('pipenv'):
                func.venv.env['PIPENV_IGNORE_VIRTUALENVS'] = '1'
                # put output to /dev/null
                cmd = "{} {}".format(cmd, '>NUL 2>&1')
            return func(cmd, **kwargs)
        return wrapper

    if os.name == 'nt':  # Windows 10
        # Do some nasty workaround because Kenneth Reitz is not accessible
        # for arguments: https://github.com/pypa/pipenv/issues/545
        venv_run = win10_decorator(venv_run)

    # Given a project is created with pyscaffold
    # and it has some dependencies in setup.cfg
    create_project(project='myproj', requirements=['appdirs'])
    with cwd.join('myproj').as_cwd():
        # When we install pipenv,
        venv_run('pip install -v pipenv')
        venv_run('pipenv --bare install certifi')
        # use it to proxy setup.cfg
        venv_run('pipenv --bare install -e .')
        # and install things to the dev env,
        venv_run('pipenv --bare install --dev flake8')

        # Then it should be able to generate a Pipfile.lock
        venv_run('pipenv lock')
        assert exists('Pipfile.lock')

        # with the correct dependencies
        with open('Pipfile.lock') as fp:
            content = json.load(fp)
            assert content['default']['appdirs']
            assert content['develop']['flake8']

        # and run things from inside pipenv's venv
        assert venv_path in venv_run('pipenv run which flake8')
        venv_run('pipenv --bare run flake8 src/myproj/skeleton.py')
<commit_msg>Deactivate pipenv unittest under Windows 10<commit_after># -*- coding: utf-8 -*-
import json
import os
from os import environ
from os.path import exists

import pytest

from pyscaffold.api import create_project

pytestmark = [pytest.mark.slow, pytest.mark.system]

@pytest.fixture(autouse=True)
def dont_load_dotenv():
    """pytest-virtualenv creates a `.env` directory by default, but `.env`
    entries in the file system are loaded by Pipenv as dotenv files.

    To prevent errors for happening we have to disable this feature.

    Additionally, it seems that env vars have to be changed before using
    venv, so an autouse fixture is required (cannot put this part in the
    beginning of the test function.
    """
    environ['PIPENV_DONT_LOAD_ENV'] = '1'

@pytest.fixture(autouse=True)
def cwd(tmpdir):
    """Guarantee a blank folder as workspace"""

    with tmpdir.as_cwd():
        yield tmpdir

@pytest.mark.skipif(
    os.name == 'nt',
    reason="pipenv fails due to colors (non-utf8) under Windows 10")
def test_pipenv_works_with_pyscaffold(cwd, venv_path, venv_run):
    # Given a project is created with pyscaffold
    # and it has some dependencies in setup.cfg
    create_project(project='myproj', requirements=['appdirs'])
    with cwd.join('myproj').as_cwd():
        # When we install pipenv,
        venv_run('pip install -v pipenv')
        venv_run('pipenv --bare install certifi')
        # use it to proxy setup.cfg
        venv_run('pipenv --bare install -e .')
        # and install things to the dev env,
        venv_run('pipenv --bare install --dev flake8')

        # Then it should be able to generate a Pipfile.lock
        venv_run('pipenv lock')
        assert exists('Pipfile.lock')

        # with the correct dependencies
        with open('Pipfile.lock') as fp:
            content = json.load(fp)
            assert content['default']['appdirs']
            assert content['develop']['flake8']

        # and run things from inside pipenv's venv
        assert venv_path in venv_run('pipenv run which flake8')
        venv_run('pipenv --bare run flake8 src/myproj/skeleton.py')
<|endoftext|>


<commit_before># -*- coding:utf-8 -*-

import os
import logging

from socket import gethostname
from multiprocessing import cpu_count
from tornado.options import define, options, parse_command_line
from tornado.log import enable_pretty_logging

try:
    from configparser import ConfigParser
except ImportError:
    from ConfigParser import ConfigParser

enable_pretty_logging()
log = logging.getLogger(__name__)

DEFAULT_SERVER_CERT = "server.crt"
DEFAULT_SERVER_KEY = "server.key"
DEFAULT_PORT = 7007
DEFAULT_CONSUL = "http://localhost:8500"

if 'serverkey' not in options:
    define('serverkey', default=DEFAULT_SERVER_KEY,
           help='SSL key')

if 'servercert' not in options:
    define('servercert', default=DEFAULT_SERVER_CERT,
           help='SSL cert')

if 'port' not in options:
    define('port', default=DEFAULT_PORT, type=int, help="listen port")

if 'nproc' not in options:
    define('nproc', default=1, type=int,
           help="Number of cores")

if 'debug' not in options:
    define('debug', default=False, type=bool)

if 'sd' not in options:
    define('sd', default=os.environ.get('SD', DEFAULT_CONSUL), type=str)

if 'registryHost' not in options:
    define("registryHost", default="localhost",
           help="Redis host")

if 'registryPort' not in options:
    define("registryPort", default=6379, type=int, help="Redis port")

def makeDefaultConfig():
    """builds the default config for ServiceDiscovery"""
    config = ConfigParser()
    config.add_section('ServiceDiscovery')
    config.set('ServiceDiscovery', 'nproc', str(options.nproc))
    config.set('ServiceDiscovery', 'secret',
               os.environ.get('SECRET', 'secret0000000000'))

    config.set('ServiceDiscovery', 'protocol', 'https')
    config.set('ServiceDiscovery', 'address', gethostname())
    config.set('ServiceDiscovery', 'port', str(options.port))
    config.set('ServiceDiscovery', 'servicename', 'ServiceDiscovery')

    config.set('ServiceDiscovery', 'servercert', options.servercert)
    config.set('ServiceDiscovery', 'serverkey', options.serverkey)
    config.set('ServiceDiscovery', 'sd', options.sd)

    config.set('ServiceDiscovery', 'registryHost', options.registryHost)
    config.set('ServiceDiscovery', 'registryPort', str(options.registryPort))

    log.info("Rebuilt config")
    for section in config.sections():
        for key, value in config.items(section):
            if key == 'secret':
                value = len(value) * '*'
            log.info("[%s] %s = %s", section, key, value)
    return config

_config = None

def config():
    global _config
    if _config is None:
        try:
            parse_command_line(final=False)
        except Exception as e:
            # FIXME: this is junk
            log.debug(e)
            pass
        _config = makeDefaultConfig()
    return _config
<commit_msg>update ports<commit_after># -*- coding:utf-8 -*-

import os
import logging

from socket import gethostname
from multiprocessing import cpu_count
from tornado.options import define, options, parse_command_line
from tornado.log import enable_pretty_logging

try:
    from configparser import ConfigParser
except ImportError:
    from ConfigParser import ConfigParser

enable_pretty_logging()
log = logging.getLogger(__name__)

DEFAULT_SERVER_CERT = "server.crt"
DEFAULT_SERVER_KEY = "server.key"
DEFAULT_PORT = 70070
DEFAULT_CONSUL = "http://localhost:8500"

if 'serverkey' not in options:
    define('serverkey', default=DEFAULT_SERVER_KEY,
           help='SSL key')

if 'servercert' not in options:
    define('servercert', default=DEFAULT_SERVER_CERT,
           help='SSL cert')

if 'port' not in options:
    define('port', default=DEFAULT_PORT, type=int, help="listen port")

if 'nproc' not in options:
    define('nproc', default=1, type=int,
           help="Number of cores")

if 'debug' not in options:
    define('debug', default=False, type=bool)

if 'sd' not in options:
    define('sd', default=os.environ.get('SD', DEFAULT_CONSUL), type=str)

if 'registryHost' not in options:
    define("registryHost", default="localhost",
           help="Redis host")

if 'registryPort' not in options:
    define("registryPort", default=6379, type=int, help="Redis port")

def makeDefaultConfig():
    """builds the default config for ServiceDiscovery"""
    config = ConfigParser()
    config.add_section('ServiceDiscovery')
    config.set('ServiceDiscovery', 'nproc', str(options.nproc))
    config.set('ServiceDiscovery', 'secret',
               os.environ.get('SECRET', 'secret0000000000'))

    config.set('ServiceDiscovery', 'protocol', 'https')
    config.set('ServiceDiscovery', 'address', gethostname())
    config.set('ServiceDiscovery', 'port', str(options.port))
    config.set('ServiceDiscovery', 'servicename', 'ServiceDiscovery')

    config.set('ServiceDiscovery', 'servercert', options.servercert)
    config.set('ServiceDiscovery', 'serverkey', options.serverkey)
    config.set('ServiceDiscovery', 'sd', options.sd)

    config.set('ServiceDiscovery', 'registryHost', options.registryHost)
    config.set('ServiceDiscovery', 'registryPort', str(options.registryPort))

    log.info("Rebuilt config")
    for section in config.sections():
        for key, value in config.items(section):
            if key == 'secret':
                value = len(value) * '*'
            log.info("[%s] %s = %s", section, key, value)
    return config

_config = None

def config():
    global _config
    if _config is None:
        try:
            parse_command_line(final=False)
        except Exception as e:
            # FIXME: this is junk
            log.debug(e)
            pass
        _config = makeDefaultConfig()
    return _config
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import os
import codecs

import pytest

from cookiecutter import main

@pytest.fixture
def template():
    return 'tests/test-extensions/custom-extension'

@pytest.fixture
def output_dir(tmpdir):
    return str(tmpdir.mkdir('hello'))

@pytest.fixture(autouse=True)
def modify_syspath(monkeypatch):
    # Make sure that the custom extension can be loaded
    monkeypatch.syspath_prepend(
        'tests/test-extensions/hello_extension'
    )

def test_pre_generate_hook(template, output_dir):
    project_dir = main.cookiecutter(
        template,
        no_input=True,
        output_dir=output_dir,
        extra_context={
            'project_slug': 'foobar',
            'name': 'Cookiemonster',
        },
    )

    readme_file = os.path.join(project_dir, 'README.rst')

    with codecs.open(readme_file, encoding='utf8') as f:
        readme = f.read().strip()

    assert readme == 'Hello Cookiemonster!'
<commit_msg>Mark test as xfail and refer to issue #850<commit_after># -*- coding: utf-8 -*-

import os
import codecs

import pytest

from cookiecutter import main

@pytest.fixture
def template():
    return 'tests/test-extensions/custom-extension'

@pytest.fixture
def output_dir(tmpdir):
    return str(tmpdir.mkdir('hello'))

@pytest.fixture(autouse=True)
def modify_syspath(monkeypatch):
    # Make sure that the custom extension can be loaded
    monkeypatch.syspath_prepend(
        'tests/test-extensions/hello_extension'
    )

@pytest.mark.xfail(reason='issue #850')
def test_pre_generate_hook(template, output_dir):
    project_dir = main.cookiecutter(
        template,
        no_input=True,
        output_dir=output_dir,
        extra_context={
            'project_slug': 'foobar',
            'name': 'Cookiemonster',
        },
    )

    readme_file = os.path.join(project_dir, 'README.rst')

    with codecs.open(readme_file, encoding='utf8') as f:
        readme = f.read().strip()

    assert readme == 'Hello Cookiemonster!'
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import json
import demjson
import yaml
import re
import unwrapper

def unwrap_and_load(content):
    cleaned = clean_json(unwrapper.unwrap_raw(content))
    return loads(cleaned)

def loads(content):
    cleaned = clean_json(content)
    try:
        # strip out HTML content and unescaped chars
        return json.loads(cleaned)
    except Exception:
        # try using demjson to decode a non-strict json string
        return demjson.decode(cleaned)

def clean_json(content):
    return remove_html(remove_bad_double_quotes(remove_invalid_escapes(add_missing_commas(content))))

def process_repl(match):
    return match.group(2)

def remove_bad_blocks(block_pairs, content):
    x = content
    for pair in block_pairs:
        x = remove_bad_block(pair[0], pair[1], x)

    return x

def remove_bad_block(key_name_to_rem, key_to_stop_rem_at, content):
    if '"{}"'.format(key_name_to_rem) in content:
        x = re.sub('(\"{}\":.{{1,}})(\"{}\")'.format(key_name_to_rem, key_to_stop_rem_at), process_repl, content, flags=re.DOTALL)
        return x

    return content

def sub_first(match):
    return match.group(1) + '"'

def sub_last(match):
    return '"' + match.group(1)

def remove_bad_double_quotes(content):
    # JSON requires values to be surrounded in " 's, ie `{"foo": "bar"}`. This handles
    # cases where the JSON is like `{"foo":""bar""}`--- json.loads() won't like this case !
    return re.sub('([^\:])\"\"', sub_first, re.sub(r'\"\"([^,])', sub_last, content))

def remove_html(content):
    return re.sub('<[^<]+?>', '', content)

def remove_invalid_escapes(content):
    return content.replace("\\", "")

def add_missing_commas(content):
    return content.replace('"}', '",}')
<commit_msg>update core behavior<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import json
import demjson
import yaml
import re
import unwrapper

def unwrap_and_load(content):
    cleaned = clean_json(unwrapper.unwrap_raw(content))
    return loads(cleaned)

def loads(content, try_yaml=False):
    try:
        json.loads(content)
    except Exception:
        cleaned = clean_json(content)
        try:
            # strip out HTML content and unescaped chars
            return json.loads(cleaned)
        except Exception:
            # try using demjson to decode a non-strict json string
            try:
                return demjson.decode(cleaned)
            except Exception:
                if try_yaml:
                    # try loading as yaml-- yaml is a superset of json..this could be dangerous in cases
                    return yaml.load(cleaned)
                raise

def clean_json(content):
    return remove_html(remove_bad_double_quotes(remove_invalid_escapes(add_missing_commas(content))))

def process_repl(match):
    return match.group(2)

def remove_bad_blocks(block_pairs, content):
    x = content
    for pair in block_pairs:
        x = remove_bad_block(pair[0], pair[1], x)

    return x

def remove_bad_block(key_name_to_rem, key_to_stop_rem_at, content):
    if '"{}"'.format(key_name_to_rem) in content:
        x = re.sub('(\"{}\":.{{1,}})(\"{}\")'.format(key_name_to_rem, key_to_stop_rem_at), process_repl, content, flags=re.DOTALL)
        return x

    return content

def sub_first(match):
    return match.group(1) + '"'

def sub_last(match):
    return '"' + match.group(1)

def remove_bad_double_quotes(content):
    # JSON requires values to be surrounded in " 's, ie `{"foo": "bar"}`. This handles
    # cases where the JSON is like `{"foo":""bar""}`--- json.loads() won't like this case !
    return re.sub('([^\:])\"\"', sub_first, re.sub(r'\"\"([^,])', sub_last, content))

def remove_html(content):
    return re.sub('<[^<]+?>', '', content)

def remove_invalid_escapes(content):
    return content.replace("\\", "")

def add_missing_commas(content):
    return content.replace('"}', '",}')
<|endoftext|>


<commit_before>#!/usr/bin/env python

import glob
import os
from io import StringIO
import sys

import pandas as pd

sys.path.insert(0, os.pardir)
from testing_harness import PyAPITestHarness
import openmc

class DiffTallyTestHarness(PyAPITestHarness):
    def _build_inputs(self):
        # Build default materials/geometry
        self._input_set.build_default_materials_and_geometry()

        # Set settings explicitly
        self._input_set.settings.batches = 3
        self._input_set.settings.inactive = 0
        self._input_set.settings.particles = 100
        self._input_set.settings.source = openmc.Source(space=openmc.stats.Box(
            [-160, -160, -183], [160, 160, 183]))
        self._input_set.settings.temperature['multipole'] = True

        self._input_set.tallies = openmc.Tallies()

        filt_mats = openmc.MaterialFilter((1, 3))
        filt_eout = openmc.EnergyoutFilter((0.0, 0.625, 20.0e6))

        # We want density derivatives for both water and fuel to get coverage
        # for both fissile and non-fissile materials.
        d1 = openmc.TallyDerivative(derivative_id=1)
        d1.variable = 'density'
        d1.material = 3
        d2 = openmc.TallyDerivative(derivative_id=2)
        d2.variable = 'density'
        d2.material = 1

        # O-16 is a good nuclide to test against because it is present in both
        # water and fuel.  Some routines need to recognize that they have the
        # perturbed nuclide but not the perturbed material.
        d3 = openmc.TallyDerivative(derivative_id=3)
        d3.variable = 'nuclide_density'
        d3.material = 1
        d3.nuclide = 'O16'

        # A fissile nuclide, just for good measure.
        d4 = openmc.TallyDerivative(derivative_id=4)
        d4.variable = 'nuclide_density'
        d4.material = 1
        d4.nuclide = 'U235'

        # Temperature derivatives.
        d5 = openmc.TallyDerivative(derivative_id=5)
        d5.variable = 'temperature'
        d5.material = 1

        derivs = [d1, d2, d3, d4, d5]

        # Cover the flux score.
        for i in range(5):
            t = openmc.Tally()
            t.add_score('flux')
            t.add_filter(filt_mats)
            t.derivative = derivs[i]
            self._input_set.tallies.append(t)

        # Cover supported scores with a collision estimator.
        for i in range(5):
            t = openmc.Tally()
            t.add_score('total')
            t.add_score('absorption')
            t.add_score('scatter')
            t.add_score('fission')
            t.add_score('nu-fission')
            t.add_filter(filt_mats)
            t.add_nuclide('total')
            t.add_nuclide('U235')
            t.derivative = derivs[i]
            self._input_set.tallies.append(t)

        # Cover an analog estimator.
        for i in range(5):
            t = openmc.Tally()
            t.add_score('absorption')
            t.add_filter(filt_mats)
            t.estimator = 'analog'
            t.derivative = derivs[i]
            self._input_set.tallies.append(t)

        # Energyout filter and total nuclide for the density derivatives.
        for i in range(2):
            t = openmc.Tally()
            t.add_score('nu-fission')
            t.add_score('scatter')
            t.add_filter(filt_mats)
            t.add_filter(filt_eout)
            t.add_nuclide('total')
            t.add_nuclide('U235')
            t.derivative = derivs[i]
            self._input_set.tallies.append(t)

        # Energyout filter without total nuclide for other derivatives.
        for i in range(2, 5):
            t = openmc.Tally()
            t.add_score('nu-fission')
            t.add_score('scatter')
            t.add_filter(filt_mats)
            t.add_filter(filt_eout)
            t.add_nuclide('U235')
            t.derivative = derivs[i]
            self._input_set.tallies.append(t)

        self._input_set.export()

    def _get_results(self):
        # Read the statepoint and summary files.
        statepoint = glob.glob(os.path.join(os.getcwd(), self._sp_name))[0]
        sp = openmc.StatePoint(statepoint)

        # Extract the tally data as a Pandas DataFrame.
        df = pd.DataFrame()
        for t in sp.tallies.values():
            df = df.append(t.get_pandas_dataframe(), ignore_index=True)

        # Extract the relevant data as a CSV string.
        out = StringIO()
        cols = ('d_material', 'd_nuclide', 'd_variable', 'score', 'mean',
                'std. dev.')
        df.to_csv(out, columns=cols, index=False, float_format='%.7e')

        return out.getvalue()

    def _cleanup(self):
        super(DiffTallyTestHarness, self)._cleanup()
        f = os.path.join(os.getcwd(), 'tallies.xml')
        if os.path.exists(f): os.remove(f)

if __name__ == '__main__':
    harness = DiffTallyTestHarness('statepoint.3.h5', True)
    harness.main()
<commit_msg>Go back to try/except on test_diff_tally StringIO<commit_after>#!/usr/bin/env python

import glob
import os
try:
    from StringIO import StringIO
except:
    from io import StringIO
import sys

import pandas as pd

sys.path.insert(0, os.pardir)
from testing_harness import PyAPITestHarness
import openmc

class DiffTallyTestHarness(PyAPITestHarness):
    def _build_inputs(self):
        # Build default materials/geometry
        self._input_set.build_default_materials_and_geometry()

        # Set settings explicitly
        self._input_set.settings.batches = 3
        self._input_set.settings.inactive = 0
        self._input_set.settings.particles = 100
        self._input_set.settings.source = openmc.Source(space=openmc.stats.Box(
            [-160, -160, -183], [160, 160, 183]))
        self._input_set.settings.temperature['multipole'] = True

        self._input_set.tallies = openmc.Tallies()

        filt_mats = openmc.MaterialFilter((1, 3))
        filt_eout = openmc.EnergyoutFilter((0.0, 0.625, 20.0e6))

        # We want density derivatives for both water and fuel to get coverage
        # for both fissile and non-fissile materials.
        d1 = openmc.TallyDerivative(derivative_id=1)
        d1.variable = 'density'
        d1.material = 3
        d2 = openmc.TallyDerivative(derivative_id=2)
        d2.variable = 'density'
        d2.material = 1

        # O-16 is a good nuclide to test against because it is present in both
        # water and fuel.  Some routines need to recognize that they have the
        # perturbed nuclide but not the perturbed material.
        d3 = openmc.TallyDerivative(derivative_id=3)
        d3.variable = 'nuclide_density'
        d3.material = 1
        d3.nuclide = 'O16'

        # A fissile nuclide, just for good measure.
        d4 = openmc.TallyDerivative(derivative_id=4)
        d4.variable = 'nuclide_density'
        d4.material = 1
        d4.nuclide = 'U235'

        # Temperature derivatives.
        d5 = openmc.TallyDerivative(derivative_id=5)
        d5.variable = 'temperature'
        d5.material = 1

        derivs = [d1, d2, d3, d4, d5]

        # Cover the flux score.
        for i in range(5):
            t = openmc.Tally()
            t.add_score('flux')
            t.add_filter(filt_mats)
            t.derivative = derivs[i]
            self._input_set.tallies.append(t)

        # Cover supported scores with a collision estimator.
        for i in range(5):
            t = openmc.Tally()
            t.add_score('total')
            t.add_score('absorption')
            t.add_score('scatter')
            t.add_score('fission')
            t.add_score('nu-fission')
            t.add_filter(filt_mats)
            t.add_nuclide('total')
            t.add_nuclide('U235')
            t.derivative = derivs[i]
            self._input_set.tallies.append(t)

        # Cover an analog estimator.
        for i in range(5):
            t = openmc.Tally()
            t.add_score('absorption')
            t.add_filter(filt_mats)
            t.estimator = 'analog'
            t.derivative = derivs[i]
            self._input_set.tallies.append(t)

        # Energyout filter and total nuclide for the density derivatives.
        for i in range(2):
            t = openmc.Tally()
            t.add_score('nu-fission')
            t.add_score('scatter')
            t.add_filter(filt_mats)
            t.add_filter(filt_eout)
            t.add_nuclide('total')
            t.add_nuclide('U235')
            t.derivative = derivs[i]
            self._input_set.tallies.append(t)

        # Energyout filter without total nuclide for other derivatives.
        for i in range(2, 5):
            t = openmc.Tally()
            t.add_score('nu-fission')
            t.add_score('scatter')
            t.add_filter(filt_mats)
            t.add_filter(filt_eout)
            t.add_nuclide('U235')
            t.derivative = derivs[i]
            self._input_set.tallies.append(t)

        self._input_set.export()

    def _get_results(self):
        # Read the statepoint and summary files.
        statepoint = glob.glob(os.path.join(os.getcwd(), self._sp_name))[0]
        sp = openmc.StatePoint(statepoint)

        # Extract the tally data as a Pandas DataFrame.
        df = pd.DataFrame()
        for t in sp.tallies.values():
            df = df.append(t.get_pandas_dataframe(), ignore_index=True)

        # Extract the relevant data as a CSV string.
        out = StringIO()
        cols = ('d_material', 'd_nuclide', 'd_variable', 'score', 'mean',
                'std. dev.')
        df.to_csv(out, columns=cols, index=False, float_format='%.7e')

        return out.getvalue()

    def _cleanup(self):
        super(DiffTallyTestHarness, self)._cleanup()
        f = os.path.join(os.getcwd(), 'tallies.xml')
        if os.path.exists(f): os.remove(f)

if __name__ == '__main__':
    harness = DiffTallyTestHarness('statepoint.3.h5', True)
    harness.main()
<|endoftext|>


<commit_before>"""
.. module:: dj-stripe.tests.test_event_processing_exception
   :synopsis: dj-stripe EventProcessingException Model Tests.

.. moduleauthor:: Alex Kavanaugh (@kavdev)

"""

from copy import deepcopy

from django.test import TestCase

from djstripe.models import EventProcessingException, Event

class TestEventProcessingException(TestCase):

    def setUp(self):
        self.msg = {
            "created": 1363911708,
            "data": {
                "object": {
                    "account_balance": 0,
                    "active_card": None,
                    "created": 1363911708,
                    "delinquent": False,
                    "description": None,
                    "discount": None,
                    "email": "xxxxxxxxxx@yahoo.com",
                    "id": "cus_xxxxxxxxxxxxxxx",
                    "livemode": True,
                    "object": "customer",
                    "subscription": None
                }
            },
            "id": "evt_xxxxxxxxxxxxx",
            "livemode": True,
            "object": "event",
            "pending_webhooks": 1,
            "request": "bla",
            "api_version": "2015-07-28",
            "type": "ping"
        }

        self.invalid_customer_msg = {
            "created": 1363911708,
            "data": {
                "object": {
                    "account_balance": 0,
                    "active_card": None,
                    "created": 1363911708,
                    "delinquent": False,
                    "description": None,
                    "discount": None,
                    "email": "xxxxxxxxxx@yahoo.com",
                    "customer": "cus_xxxxxxxxxxxxxxx",
                    "livemode": True,
                    "object": "customer",
                    "subscription": None
                }
            },
            "id": "evt_xxxxxxxxxxxxx",
            "livemode": True,
            "object": "event",
            "request": "bla",
            "api_version": "2015-07-28",
            "pending_webhooks": 1,
            "type": "ping"
        }

        self.event = Event.objects.create(
            stripe_id=self.msg["id"],
            type="ping",
            webhook_message=self.msg,
            message=self.msg
        )

    def test_tostring(self):
        # Not sure if this is normal, but self.exception returns:
        # AssertionError: '<IOError, pk=1, Event=ping - evt_xxxxxxxxxxxxx>' != '<Error in transmission., pk=1, Event=ping - evt_xxxxxxxxxxxxx>'
        # - <IOError, pk=1, Event=ping - evt_xxxxxxxxxxxxx>
        # ?  --
        # + <Error in transmission., pk=1, Event=<ping, stripe_id=evt_xxxxxxxxxxxxx>>
        # ?       ++++++++++++++++++

        try:
            raise IOError("Error in transmission.")
        except IOError as error:
            EventProcessingException.log(data=self.msg["data"], exception=error, event=self.event)
            exception = EventProcessingException.objects.get(event=self.event)

        # It may be too strict to assert the pk? Maybe incr field not reset in some psql implementations?
        # self.assertIn('<Error in transmission., pk=1, Event=<ping, stripe_id=evt_xxxxxxxxxxxxx>>', str(exception))
        self.assertIn('<Error in transmission., pk=', str(exception))
        self.assertIn(', Event=<type=ping, stripe_id=evt_xxxxxxxxxxxxx>>', str(exception))

    def test_non_crud_link_customer_on_non_customer(self):
        self.event.process()
        self.assertEqual(None, self.event.customer)

    def test_non_crud_link_customer_on_invalid_customer(self):
        event_copy = deepcopy(self.event)
        event_copy.validated_message = self.invalid_customer_msg
        event_copy.save()
        event_copy.process()
        self.assertEqual(None, event_copy.customer)
<commit_msg>fixed event test bug<commit_after>"""
.. module:: dj-stripe.tests.test_event_processing_exception
   :synopsis: dj-stripe EventProcessingException Model Tests.

.. moduleauthor:: Alex Kavanaugh (@kavdev)

"""

from copy import deepcopy

from django.test import TestCase

from djstripe.models import EventProcessingException, Event

class TestEventProcessingException(TestCase):

    def setUp(self):
        self.msg = {
            "created": 1363911708,
            "data": {
                "object": {
                    "account_balance": 0,
                    "active_card": None,
                    "created": 1363911708,
                    "delinquent": False,
                    "description": None,
                    "discount": None,
                    "email": "xxxxxxxxxx@yahoo.com",
                    "id": "cus_xxxxxxxxxxxxxxx",
                    "livemode": True,
                    "object": "customer",
                    "subscription": None
                }
            },
            "id": "evt_xxxxxxxxxxxxx",
            "livemode": True,
            "object": "event",
            "pending_webhooks": 1,
            "request": "bla",
            "api_version": "2015-07-28",
            "type": "ping"
        }

        self.invalid_customer_msg = {
            "created": 1363911708,
            "data": {
                "object": {
                    "account_balance": 0,
                    "active_card": None,
                    "created": 1363911708,
                    "delinquent": False,
                    "description": None,
                    "discount": None,
                    "email": "xxxxxxxxxx@yahoo.com",
                    "customer": "cus_xxxxxxxxxxxxxxx",
                    "livemode": True,
                    "object": "customer",
                    "subscription": None
                }
            },
            "id": "evt_xxxxxxxxxxxxx",
            "livemode": True,
            "object": "event",
            "request": "bla",
            "api_version": "2015-07-28",
            "pending_webhooks": 1,
            "type": "ping"
        }

        self.event = Event.objects.create(
            stripe_id=self.msg["id"],
            type="ping",
            webhook_message=self.msg,
        )

    def test_tostring(self):
        # Not sure if this is normal, but self.exception returns:
        # AssertionError: '<IOError, pk=1, Event=ping - evt_xxxxxxxxxxxxx>' != '<Error in transmission., pk=1, Event=ping - evt_xxxxxxxxxxxxx>'
        # - <IOError, pk=1, Event=ping - evt_xxxxxxxxxxxxx>
        # ?  --
        # + <Error in transmission., pk=1, Event=<ping, stripe_id=evt_xxxxxxxxxxxxx>>
        # ?       ++++++++++++++++++

        try:
            raise IOError("Error in transmission.")
        except IOError as error:
            EventProcessingException.log(data=self.msg["data"], exception=error, event=self.event)
            exception = EventProcessingException.objects.get(event=self.event)

        # It may be too strict to assert the pk? Maybe incr field not reset in some psql implementations?
        # self.assertIn('<Error in transmission., pk=1, Event=<ping, stripe_id=evt_xxxxxxxxxxxxx>>', str(exception))
        self.assertIn('<Error in transmission., pk=', str(exception))
        self.assertIn(', Event=<type=ping, stripe_id=evt_xxxxxxxxxxxxx>>', str(exception))

    def test_non_crud_link_customer_on_non_customer(self):
        self.event.process()
        self.assertEqual(None, self.event.customer)

    def test_non_crud_link_customer_on_invalid_customer(self):
        event_copy = deepcopy(self.event)
        event_copy.validated_message = self.invalid_customer_msg
        event_copy.save()
        event_copy.process()
        self.assertEqual(None, event_copy.customer)
<|endoftext|>