<commit_before># -*- encoding: utf-8 -*-
#
#  dnstamper
#  *********
#
#  The test reports censorship if the cardinality of the intersection of
#  the query result set from the control server and the query result set
#  from the experimental server is zero, which is to say, if the two sets
#  have no matching results whatsoever.
#
#  NOTE: This test frequently results in false positives due to GeoIP-based
#  load balancing on major global sites such as google, facebook, and
#  youtube, etc.
#
# :authors: Arturo Filastò, Isis Lovecruft
# :licence: see LICENSE

import pdb

from twisted.python import usage
from twisted.internet import defer

from ooni.templates import dnst

from ooni import nettest
from ooni.utils import log

class UsageOptions(usage.Options):
    optParameters = [['backend', 'b', '8.8.8.8:53',
                        'The OONI backend that runs the DNS resolver'],
                     ['testresolvers', 'T', None,
                        'File containing list of DNS resolvers to test against'],
                     ['testresolver', 't', None,
                         'Specify a single test resolver to use for testing']
                    ]

class DNSTamperTest(dnst.DNSTest):

    name = "DNS tamper"
    description = "DNS censorship detection test"
    version = "0.4"
    authors = "Arturo Filastò, Isis Lovecruft"
    requirements = None

    inputFile = ['file', 'f', None,
                 'Input file of list of hostnames to attempt to resolve']

    usageOptions = UsageOptions
    requiredOptions = ['backend', 'file']

    def setUp(self):
        if (not self.localOptions['testresolvers'] and \
                not self.localOptions['testresolver']):
            raise usage.UsageError("You did not specify a testresolver")

        elif self.localOptions['testresolvers']:
            test_resolvers_file = self.localOptions['testresolvers']

        elif self.localOptions['testresolver']:
            self.test_resolvers = [self.localOptions['testresolver']]

        try:
            with open(test_resolvers_file) as f:
                self.test_resolvers = [x.split('#')[0].strip() for x in f.readlines()]
                self.report['test_resolvers'] = self.test_resolvers
            fp.close()

        except IOError:
            log.exception(e)
            raise usage.UsageError("Invalid test resolvers file")

        except NameError:
            log.debug("No test resolver file configured")

        dns_ip, dns_port = self.localOptions['backend'].split(':')
        self.control_dns_server = (dns_ip, int(dns_port))

        self.report['control_resolver'] = self.control_dns_server

    @defer.inlineCallbacks
    def test_a_lookup(self):
        """
        We perform an A lookup on the DNS test servers for the domains to be
        tested and an A lookup on the known good DNS server.

        We then compare the results from test_resolvers and that from
        control_resolver and see if the match up.
        If they match up then no censorship is happening (tampering: false).

        If they do not we do a reverse lookup (PTR) on the test_resolvers and
        the control resolver for every IP address we got back and check to see
        if anyone of them matches the control ones.

        If they do then we take not of the fact that censorship is probably not
        happening (tampering: reverse-match).

        If they do not match then censorship is probably going on (tampering:
        true).
        """
        log.msg("Doing the test lookups on %s" % self.input)
        list_of_ds = []
        hostname = self.input

        self.report['tampering'] = {}

        control_answers = yield self.performALookup(hostname, self.control_dns_server)
        if not control_answers:
                log.err("Got no response from control DNS server %s," \
                        " perhaps the DNS resolver is down?" % self.control_dns_server[0])
                self.report['tampering'][self.control_dns_server] = 'no_answer'
                return

        for test_resolver in self.test_resolvers:
            log.msg("Testing %s test resolver" % test_resolver)
            test_dns_server = (test_resolver, 53)

            experiment_answers = yield self.performALookup(hostname, test_dns_server)
            log.debug("Got the following A lookup answers %s for %s" % (experiment_answers, test_resolver))

            if not experiment_answers:
                log.err("Got no response, perhaps the DNS resolver is down?")
                self.report['tampering'][test_resolver] = 'no_answer'
                continue

            def lookup_details():
                """
                A closure useful for printing test details.
                """
                log.msg("test resolver: %s" % test_resolver)
                log.msg("experiment answers: %s" % experiment_answers)
                log.msg("control answers: %s" % control_answers)

            log.debug("Comparing %s with %s" % (experiment_answers, control_answers))
            if set(experiment_answers) & set(control_answers):
                lookup_details()
                log.msg("tampering: false")
                self.report['tampering'][test_resolver] = False
            else:
                log.msg("Trying to do reverse lookup")

                experiment_reverse = yield self.performPTRLookup(experiment_answers[0], test_dns_server)
                control_reverse = yield self.performPTRLookup(control_answers[0], self.control_dns_server)

                if experiment_reverse == control_reverse:
                    log.msg("Further testing has eliminated false positives")
                    lookup_details()
                    log.msg("tampering: reverse_match")
                    self.report['tampering'][test_resolver] = 'reverse_match'
                else:
                    log.msg("Reverse lookups do not match")
                    lookup_details()
                    log.msg("tampering: true")
                    self.report['tampering'][test_resolver] = True
<commit_msg>Fix variable name<commit_after># -*- encoding: utf-8 -*-
#
#  dnstamper
#  *********
#
#  The test reports censorship if the cardinality of the intersection of
#  the query result set from the control server and the query result set
#  from the experimental server is zero, which is to say, if the two sets
#  have no matching results whatsoever.
#
#  NOTE: This test frequently results in false positives due to GeoIP-based
#  load balancing on major global sites such as google, facebook, and
#  youtube, etc.
#
# :authors: Arturo Filastò, Isis Lovecruft
# :licence: see LICENSE

import pdb

from twisted.python import usage
from twisted.internet import defer

from ooni.templates import dnst

from ooni import nettest
from ooni.utils import log

class UsageOptions(usage.Options):
    optParameters = [['backend', 'b', '8.8.8.8:53',
                        'The OONI backend that runs the DNS resolver'],
                     ['testresolvers', 'T', None,
                        'File containing list of DNS resolvers to test against'],
                     ['testresolver', 't', None,
                         'Specify a single test resolver to use for testing']
                    ]

class DNSTamperTest(dnst.DNSTest):

    name = "DNS tamper"
    description = "DNS censorship detection test"
    version = "0.4"
    authors = "Arturo Filastò, Isis Lovecruft"
    requirements = None

    inputFile = ['file', 'f', None,
                 'Input file of list of hostnames to attempt to resolve']

    usageOptions = UsageOptions
    requiredOptions = ['backend', 'file']

    def setUp(self):
        if (not self.localOptions['testresolvers'] and \
                not self.localOptions['testresolver']):
            raise usage.UsageError("You did not specify a testresolver")

        elif self.localOptions['testresolvers']:
            test_resolvers_file = self.localOptions['testresolvers']

        elif self.localOptions['testresolver']:
            self.test_resolvers = [self.localOptions['testresolver']]

        try:
            with open(test_resolvers_file) as f:
                self.test_resolvers = [x.split('#')[0].strip() for x in f.readlines()]
                self.report['test_resolvers'] = self.test_resolvers
            f.close()

        except IOError:
            log.exception(e)
            raise usage.UsageError("Invalid test resolvers file")

        except NameError:
            log.debug("No test resolver file configured")

        dns_ip, dns_port = self.localOptions['backend'].split(':')
        self.control_dns_server = (dns_ip, int(dns_port))

        self.report['control_resolver'] = self.control_dns_server

    @defer.inlineCallbacks
    def test_a_lookup(self):
        """
        We perform an A lookup on the DNS test servers for the domains to be
        tested and an A lookup on the known good DNS server.

        We then compare the results from test_resolvers and that from
        control_resolver and see if the match up.
        If they match up then no censorship is happening (tampering: false).

        If they do not we do a reverse lookup (PTR) on the test_resolvers and
        the control resolver for every IP address we got back and check to see
        if anyone of them matches the control ones.

        If they do then we take not of the fact that censorship is probably not
        happening (tampering: reverse-match).

        If they do not match then censorship is probably going on (tampering:
        true).
        """
        log.msg("Doing the test lookups on %s" % self.input)
        list_of_ds = []
        hostname = self.input

        self.report['tampering'] = {}

        control_answers = yield self.performALookup(hostname, self.control_dns_server)
        if not control_answers:
                log.err("Got no response from control DNS server %s," \
                        " perhaps the DNS resolver is down?" % self.control_dns_server[0])
                self.report['tampering'][self.control_dns_server] = 'no_answer'
                return

        for test_resolver in self.test_resolvers:
            log.msg("Testing %s test resolver" % test_resolver)
            test_dns_server = (test_resolver, 53)

            experiment_answers = yield self.performALookup(hostname, test_dns_server)
            log.debug("Got the following A lookup answers %s for %s" % (experiment_answers, test_resolver))

            if not experiment_answers:
                log.err("Got no response, perhaps the DNS resolver is down?")
                self.report['tampering'][test_resolver] = 'no_answer'
                continue

            def lookup_details():
                """
                A closure useful for printing test details.
                """
                log.msg("test resolver: %s" % test_resolver)
                log.msg("experiment answers: %s" % experiment_answers)
                log.msg("control answers: %s" % control_answers)

            log.debug("Comparing %s with %s" % (experiment_answers, control_answers))
            if set(experiment_answers) & set(control_answers):
                lookup_details()
                log.msg("tampering: false")
                self.report['tampering'][test_resolver] = False
            else:
                log.msg("Trying to do reverse lookup")

                experiment_reverse = yield self.performPTRLookup(experiment_answers[0], test_dns_server)
                control_reverse = yield self.performPTRLookup(control_answers[0], self.control_dns_server)

                if experiment_reverse == control_reverse:
                    log.msg("Further testing has eliminated false positives")
                    lookup_details()
                    log.msg("tampering: reverse_match")
                    self.report['tampering'][test_resolver] = 'reverse_match'
                else:
                    log.msg("Reverse lookups do not match")
                    lookup_details()
                    log.msg("tampering: true")
                    self.report['tampering'][test_resolver] = True
<|endoftext|>


<commit_before>"""Module which impements the sequence class and a few of its subclasses."""

from typing import List, Dict

import tensorflow as tf
from typeguard import check_argument_types

from neuralmonkey.dataset import Dataset
from neuralmonkey.decorators import tensor
from neuralmonkey.model.feedable import FeedDict
from neuralmonkey.model.model_part import ModelPart
from neuralmonkey.model.parameterized import InitializerSpecs
from neuralmonkey.model.stateful import TemporalStateful
from neuralmonkey.tf_utils import get_variable
from neuralmonkey.vocabulary import Vocabulary, pad_batch, sentence_mask

# pylint: disable=abstract-method
class Sequence(ModelPart, TemporalStateful):
    """Base class for a data sequence.

    This abstract class represents a batch of sequences of Tensors of possibly
    different lengths.

    Sequence is essentialy a temporal stateful object whose states and mask
    are fed, or computed from fed values. It is also a ModelPart, and
    therefore, it can store variables such as embedding matrices.
    """

    def __init__(self,
                 name: str,
                 max_length: int = None,
                 reuse: ModelPart = None,
                 save_checkpoint: str = None,
                 load_checkpoint: str = None,
                 initializers: InitializerSpecs = None) -> None:
        """Construct a new `Sequence` object.

        Arguments:
            name: The name for the `ModelPart` object
            max_length: Maximum length of sequences in the object (not checked)
            save_checkpoint: The save_checkpoint parameter for `ModelPart`
            load_checkpoint: The load_checkpoint parameter for `ModelPart`
        """
        ModelPart.__init__(self, name, reuse, save_checkpoint, load_checkpoint,
                           initializers)

        self.max_length = max_length
        if self.max_length is not None and self.max_length <= 0:
            raise ValueError("Max sequence length must be a positive integer.")
# pylint: enable=abstract-method

class EmbeddedFactorSequence(Sequence):
    """A sequence that stores one or more embedded inputs (factors)."""

    # pylint: disable=too-many-arguments
    def __init__(self,
                 name: str,
                 vocabularies: List[Vocabulary],
                 data_ids: List[str],
                 embedding_sizes: List[int],
                 max_length: int = None,
                 add_start_symbol: bool = False,
                 add_end_symbol: bool = False,
                 scale_embeddings_by_depth: bool = False,
                 embeddings_source: "EmbeddedFactorSequence" = None,
                 reuse: ModelPart = None,
                 save_checkpoint: str = None,
                 load_checkpoint: str = None,
                 initializers: InitializerSpecs = None) -> None:
        """Construct a new instance of `EmbeddedFactorSequence`.

        Takes three lists of vocabularies, data series IDs, and embedding
        sizes and construct a `Sequence` object. The supplied lists must be
        equal in length and the indices to these lists must correspond
        to each other

        Arguments:
            name: The name for the `ModelPart` object
            vocabularies: A list of `Vocabulary` objects used for each factor
            data_ids: A list of strings identifying the data series used for
                each factor
            embedding_sizes: A list of integers specifying the size of the
                embedding vector for each factor
            max_length: The maximum length of the sequences
            add_start_symbol: Includes <s> in the sequence
            add_end_symbol: Includes </s> in the sequence
            scale_embeddings_by_depth: Set to True for T2T import compatibility
            embeddings_source: EmbeddedSequence from which the embeedings will
                be reused.
            save_checkpoint: The save_checkpoint parameter for `ModelPart`
            load_checkpoint: The load_checkpoint parameter for `ModelPart`
        """
        check_argument_types()
        Sequence.__init__(
            self, name, max_length, reuse, save_checkpoint, load_checkpoint,
            initializers)

        self.vocabularies = vocabularies
        self.vocabulary_sizes = [len(vocab) for vocab in self.vocabularies]
        self.data_ids = data_ids
        self.embedding_sizes = embedding_sizes
        self.add_start_symbol = add_start_symbol
        self.add_end_symbol = add_end_symbol
        self.scale_embeddings_by_depth = scale_embeddings_by_depth
        self.embeddings_source = embeddings_source

        if not (len(self.data_ids)
                == len(self.vocabularies)
                == len(self.embedding_sizes)):
            raise ValueError("data_ids, vocabularies, and embedding_sizes "
                             "lists need to have the same length")

        if any([esize <= 0 for esize in self.embedding_sizes]):
            raise ValueError("Embedding size must be a positive integer.")

        if embeddings_source is not None:
            if not all(v1 == v2 for v1, v2 in zip(
                    self.vocabularies, embeddings_source.vocabularies)):
                raise ValueError(
                    "When reusing embeedings, vocabularies must be the same.")
            if not all(s1 == s2 for s1, s2 in zip(
                    self.embedding_sizes, embeddings_source.embedding_sizes)):
                raise ValueError(
                    "When reusing embeedings, embeddings sizes must be equal.")

        self._variable_scope.set_initializer(
            tf.random_normal_initializer(stddev=0.001))
    # pylint: enable=too-many-arguments

    @property
    def input_types(self) -> Dict[str, tf.DType]:
        return {d_id: tf.string for d_id in self.data_ids}

    @property
    def input_shapes(self) -> Dict[str, tf.TensorShape]:
        return {d_id: tf.TensorShape([None, None]) for d_id in self.data_ids}

    @tensor
    def input_factor_indices(self) -> List[tf.Tensor]:
        return [vocab.strings_to_indices(factor) for
                vocab, factor in zip(self.vocabularies, self.input_factors)]

    @tensor
    def input_factors(self) -> List[tf.Tensor]:
        return [self.dataset[s_id] for s_id in self.data_ids]

    @tensor
    def embedding_matrices(self) -> List[tf.Tensor]:
        """Return a list of embedding matrices for each factor."""

        # Note: Embedding matrices are numbered rather than named by the data
        # id so the data_id string does not need to be the same across
        # experiments

        if self.embeddings_source is not None:
            return self.embeddings_source.embedding_matrices

        return [
            get_variable(
                name="embedding_matrix_{}".format(i),
                shape=[vocab_size, emb_size])
            for i, (data_id, vocab_size, emb_size) in enumerate(zip(
                self.data_ids, self.vocabulary_sizes, self.embedding_sizes))]

    @tensor
    def temporal_states(self) -> tf.Tensor:
        """Return the embedded factors.

        A 3D Tensor of shape (batch, time, dimension),
        where dimension is the sum of the embedding sizes supplied to the
        constructor.
        """
        embedded_factors = []
        for (factor, embedding_matrix) in zip(
                self.input_factor_indices, self.embedding_matrices):
            emb_factor = tf.nn.embedding_lookup(embedding_matrix, factor)

            # github.com/tensorflow/tensor2tensor/blob/v1.5.6/tensor2tensor/
            #            layers/modalities.py#L104
            if self.scale_embeddings_by_depth:
                emb_size = embedding_matrix.shape.as_list()[-1]
                emb_factor *= emb_size**0.5

            # We explicitly set paddings to zero-value vectors
            # TODO: remove unnecessary masking in the subesquent modules
            emb_factor = emb_factor * tf.expand_dims(self.temporal_mask, -1)
            embedded_factors.append(emb_factor)

        return tf.concat(embedded_factors, 2)

    # pylint: disable=unsubscriptable-object
    @tensor
    def temporal_mask(self) -> tf.Tensor:
        return sentence_mask(self.input_factor_indices[0])
    # pylint: enable=unsubscriptable-object

    def feed_dict(self, dataset: Dataset, train: bool = False) -> FeedDict:
        """Feed the placholders with the data.

        Arguments:
            dataset: The dataset.
            train: A flag whether the train mode is enabled.

        Returns:
            The constructed feed dictionary that contains the factor data and
            the mask.
        """
        fd = ModelPart.feed_dict(self, dataset, train)

        # for checking the lengths of individual factors
        for factor_plc, name in zip(self.input_factors, self.data_ids):
            sentences = dataset.get_series(name)
            fd[factor_plc] = pad_batch(
                list(sentences), self.max_length, self.add_start_symbol,
                self.add_end_symbol)

        return fd

class EmbeddedSequence(EmbeddedFactorSequence):
    """A sequence of embedded inputs (for a single factor)."""

    # pylint: disable=too-many-arguments
    def __init__(self,
                 name: str,
                 vocabulary: Vocabulary,
                 data_id: str,
                 embedding_size: int,
                 max_length: int = None,
                 add_start_symbol: bool = False,
                 add_end_symbol: bool = False,
                 scale_embeddings_by_depth: bool = False,
                 embeddings_source: "EmbeddedSequence" = None,
                 reuse: ModelPart = None,
                 save_checkpoint: str = None,
                 load_checkpoint: str = None,
                 initializers: InitializerSpecs = None) -> None:
        """Construct a new instance of `EmbeddedSequence`.

        Arguments:
            name: The name for the `ModelPart` object
            vocabulary: A `Vocabulary` object used for the sequence data
            data_id: A string that identifies the data series used for
                the sequence data
            embedding_sizes: An integer that specifies the size of the
                embedding vector for the sequence data
            max_length: The maximum length of the sequences
            add_start_symbol: Includes <s> in the sequence
            add_end_symbol: Includes </s> in the sequence
            scale_embeddings_by_depth: Set to True for T2T import compatibility
            embeddings_source: `EmbeddedSequence` from which the embeedings
                will be reused.
            save_checkpoint: The save_checkpoint parameter for `ModelPart`
            load_checkpoint: The load_checkpoint parameter for `ModelPart`
        """
        EmbeddedFactorSequence.__init__(
            self,
            name=name,
            vocabularies=[vocabulary],
            data_ids=[data_id],
            embedding_sizes=[embedding_size],
            max_length=max_length,
            add_start_symbol=add_start_symbol,
            add_end_symbol=add_end_symbol,
            scale_embeddings_by_depth=scale_embeddings_by_depth,
            embeddings_source=embeddings_source,
            reuse=reuse,
            save_checkpoint=save_checkpoint,
            load_checkpoint=load_checkpoint,
            initializers=initializers)
    # pylint: enable=too-many-arguments

    # pylint: disable=unsubscriptable-object
    @property
    def inputs(self) -> tf.Tensor:
        """Return a 2D placeholder for the sequence inputs."""
        return self.input_factor_indices[0]

    @property
    def embedding_matrix(self) -> tf.Tensor:
        """Return the embedding matrix for the sequence."""
        return self.embedding_matrices[0]
    # pylint: enable=unsubscriptable-object

    @property
    def vocabulary(self) -> Vocabulary:
        """Return the input vocabulary."""
        return self.vocabularies[0]

    @property
    def data_id(self) -> str:
        """Return the input data series indentifier."""
        return self.data_ids[0]
<commit_msg>sequence trainable option<commit_after>"""Module which impements the sequence class and a few of its subclasses."""

from typing import List, Dict

import tensorflow as tf
from typeguard import check_argument_types

from neuralmonkey.dataset import Dataset
from neuralmonkey.decorators import tensor
from neuralmonkey.model.feedable import FeedDict
from neuralmonkey.model.model_part import ModelPart
from neuralmonkey.model.parameterized import InitializerSpecs
from neuralmonkey.model.stateful import TemporalStateful
from neuralmonkey.tf_utils import get_variable
from neuralmonkey.vocabulary import Vocabulary, pad_batch, sentence_mask

# pylint: disable=abstract-method
class Sequence(ModelPart, TemporalStateful):
    """Base class for a data sequence.

    This abstract class represents a batch of sequences of Tensors of possibly
    different lengths.

    Sequence is essentialy a temporal stateful object whose states and mask
    are fed, or computed from fed values. It is also a ModelPart, and
    therefore, it can store variables such as embedding matrices.
    """

    def __init__(self,
                 name: str,
                 max_length: int = None,
                 reuse: ModelPart = None,
                 save_checkpoint: str = None,
                 load_checkpoint: str = None,
                 initializers: InitializerSpecs = None) -> None:
        """Construct a new `Sequence` object.

        Arguments:
            name: The name for the `ModelPart` object
            max_length: Maximum length of sequences in the object (not checked)
            save_checkpoint: The save_checkpoint parameter for `ModelPart`
            load_checkpoint: The load_checkpoint parameter for `ModelPart`
        """
        ModelPart.__init__(self, name, reuse, save_checkpoint, load_checkpoint,
                           initializers)

        self.max_length = max_length
        if self.max_length is not None and self.max_length <= 0:
            raise ValueError("Max sequence length must be a positive integer.")
# pylint: enable=abstract-method

class EmbeddedFactorSequence(Sequence):
    """A sequence that stores one or more embedded inputs (factors)."""

    # pylint: disable=too-many-arguments
    def __init__(self,
                 name: str,
                 vocabularies: List[Vocabulary],
                 data_ids: List[str],
                 embedding_sizes: List[int],
                 max_length: int = None,
                 add_start_symbol: bool = False,
                 add_end_symbol: bool = False,
                 scale_embeddings_by_depth: bool = False,
                 trainable: bool = True,
                 embeddings_source: "EmbeddedFactorSequence" = None,
                 reuse: ModelPart = None,
                 save_checkpoint: str = None,
                 load_checkpoint: str = None,
                 initializers: InitializerSpecs = None) -> None:
        """Construct a new instance of `EmbeddedFactorSequence`.

        Takes three lists of vocabularies, data series IDs, and embedding
        sizes and construct a `Sequence` object. The supplied lists must be
        equal in length and the indices to these lists must correspond
        to each other

        Arguments:
            name: The name for the `ModelPart` object
            vocabularies: A list of `Vocabulary` objects used for each factor
            data_ids: A list of strings identifying the data series used for
                each factor
            embedding_sizes: A list of integers specifying the size of the
                embedding vector for each factor
            max_length: The maximum length of the sequences
            add_start_symbol: Includes <s> in the sequence
            add_end_symbol: Includes </s> in the sequence
            scale_embeddings_by_depth: Set to True for T2T import compatibility
            embeddings_source: EmbeddedSequence from which the embeedings will
                be reused.
            save_checkpoint: The save_checkpoint parameter for `ModelPart`
            load_checkpoint: The load_checkpoint parameter for `ModelPart`
        """
        check_argument_types()
        Sequence.__init__(
            self, name, max_length, reuse, save_checkpoint, load_checkpoint,
            initializers)

        self.vocabularies = vocabularies
        self.vocabulary_sizes = [len(vocab) for vocab in self.vocabularies]
        self.data_ids = data_ids
        self.embedding_sizes = embedding_sizes
        self.add_start_symbol = add_start_symbol
        self.add_end_symbol = add_end_symbol
        self.scale_embeddings_by_depth = scale_embeddings_by_depth
        self.embeddings_source = embeddings_source
        self.trainable = trainable

        if not (len(self.data_ids)
                == len(self.vocabularies)
                == len(self.embedding_sizes)):
            raise ValueError("data_ids, vocabularies, and embedding_sizes "
                             "lists need to have the same length")

        if any([esize <= 0 for esize in self.embedding_sizes]):
            raise ValueError("Embedding size must be a positive integer.")

        if embeddings_source is not None:
            if not all(v1 == v2 for v1, v2 in zip(
                    self.vocabularies, embeddings_source.vocabularies)):
                raise ValueError(
                    "When reusing embeedings, vocabularies must be the same.")
            if not all(s1 == s2 for s1, s2 in zip(
                    self.embedding_sizes, embeddings_source.embedding_sizes)):
                raise ValueError(
                    "When reusing embeedings, embeddings sizes must be equal.")

        self._variable_scope.set_initializer(
            tf.random_normal_initializer(stddev=0.001))
    # pylint: enable=too-many-arguments

    @property
    def input_types(self) -> Dict[str, tf.DType]:
        return {d_id: tf.string for d_id in self.data_ids}

    @property
    def input_shapes(self) -> Dict[str, tf.TensorShape]:
        return {d_id: tf.TensorShape([None, None]) for d_id in self.data_ids}

    @tensor
    def input_factor_indices(self) -> List[tf.Tensor]:
        return [vocab.strings_to_indices(factor) for
                vocab, factor in zip(self.vocabularies, self.input_factors)]

    @tensor
    def input_factors(self) -> List[tf.Tensor]:
        return [self.dataset[s_id] for s_id in self.data_ids]

    @tensor
    def embedding_matrices(self) -> List[tf.Tensor]:
        """Return a list of embedding matrices for each factor."""

        # Note: Embedding matrices are numbered rather than named by the data
        # id so the data_id string does not need to be the same across
        # experiments

        if self.embeddings_source is not None:
            return self.embeddings_source.embedding_matrices

        return [
            get_variable(
                name="embedding_matrix_{}".format(i),
                shape=[vocab_size, emb_size],
                trainable=self.trainable)
            for i, (data_id, vocab_size, emb_size) in enumerate(zip(
                self.data_ids, self.vocabulary_sizes, self.embedding_sizes))]

    @tensor
    def temporal_states(self) -> tf.Tensor:
        """Return the embedded factors.

        A 3D Tensor of shape (batch, time, dimension),
        where dimension is the sum of the embedding sizes supplied to the
        constructor.
        """
        embedded_factors = []
        for (factor, embedding_matrix) in zip(
                self.input_factor_indices, self.embedding_matrices):
            emb_factor = tf.nn.embedding_lookup(embedding_matrix, factor)

            # github.com/tensorflow/tensor2tensor/blob/v1.5.6/tensor2tensor/
            #            layers/modalities.py#L104
            if self.scale_embeddings_by_depth:
                emb_size = embedding_matrix.shape.as_list()[-1]
                emb_factor *= emb_size**0.5

            # We explicitly set paddings to zero-value vectors
            # TODO: remove unnecessary masking in the subesquent modules
            emb_factor = emb_factor * tf.expand_dims(self.temporal_mask, -1)
            embedded_factors.append(emb_factor)

        return tf.concat(embedded_factors, 2)

    # pylint: disable=unsubscriptable-object
    @tensor
    def temporal_mask(self) -> tf.Tensor:
        return sentence_mask(self.input_factor_indices[0])
    # pylint: enable=unsubscriptable-object

    def feed_dict(self, dataset: Dataset, train: bool = False) -> FeedDict:
        """Feed the placholders with the data.

        Arguments:
            dataset: The dataset.
            train: A flag whether the train mode is enabled.

        Returns:
            The constructed feed dictionary that contains the factor data and
            the mask.
        """
        fd = ModelPart.feed_dict(self, dataset, train)

        # for checking the lengths of individual factors
        for factor_plc, name in zip(self.input_factors, self.data_ids):
            sentences = dataset.get_series(name)
            fd[factor_plc] = pad_batch(
                list(sentences), self.max_length, self.add_start_symbol,
                self.add_end_symbol)

        return fd

class EmbeddedSequence(EmbeddedFactorSequence):
    """A sequence of embedded inputs (for a single factor)."""

    # pylint: disable=too-many-arguments
    def __init__(self,
                 name: str,
                 vocabulary: Vocabulary,
                 data_id: str,
                 embedding_size: int,
                 max_length: int = None,
                 add_start_symbol: bool = False,
                 add_end_symbol: bool = False,
                 scale_embeddings_by_depth: bool = False,
                 trainable: bool = True,
                 embeddings_source: "EmbeddedSequence" = None,
                 reuse: ModelPart = None,
                 save_checkpoint: str = None,
                 load_checkpoint: str = None,
                 initializers: InitializerSpecs = None) -> None:
        """Construct a new instance of `EmbeddedSequence`.

        Arguments:
            name: The name for the `ModelPart` object
            vocabulary: A `Vocabulary` object used for the sequence data
            data_id: A string that identifies the data series used for
                the sequence data
            embedding_sizes: An integer that specifies the size of the
                embedding vector for the sequence data
            max_length: The maximum length of the sequences
            add_start_symbol: Includes <s> in the sequence
            add_end_symbol: Includes </s> in the sequence
            scale_embeddings_by_depth: Set to True for T2T import compatibility
            embeddings_source: `EmbeddedSequence` from which the embeedings
                will be reused.
            save_checkpoint: The save_checkpoint parameter for `ModelPart`
            load_checkpoint: The load_checkpoint parameter for `ModelPart`
        """
        EmbeddedFactorSequence.__init__(
            self,
            name=name,
            vocabularies=[vocabulary],
            data_ids=[data_id],
            embedding_sizes=[embedding_size],
            max_length=max_length,
            add_start_symbol=add_start_symbol,
            add_end_symbol=add_end_symbol,
            scale_embeddings_by_depth=scale_embeddings_by_depth,
            trainable=trainable,
            embeddings_source=embeddings_source,
            reuse=reuse,
            save_checkpoint=save_checkpoint,
            load_checkpoint=load_checkpoint,
            initializers=initializers)
    # pylint: enable=too-many-arguments

    # pylint: disable=unsubscriptable-object
    @property
    def inputs(self) -> tf.Tensor:
        """Return a 2D placeholder for the sequence inputs."""
        return self.input_factor_indices[0]

    @property
    def embedding_matrix(self) -> tf.Tensor:
        """Return the embedding matrix for the sequence."""
        return self.embedding_matrices[0]
    # pylint: enable=unsubscriptable-object

    @property
    def vocabulary(self) -> Vocabulary:
        """Return the input vocabulary."""
        return self.vocabularies[0]

    @property
    def data_id(self) -> str:
        """Return the input data series indentifier."""
        return self.data_ids[0]
<|endoftext|>


<commit_before>#=========================================================================
# ValRdyBundle_test.py
#=========================================================================
# Unit tests for the ValRdy PortBundle.

from new_pymtl    import *
from ValRdyBundle import InValRdyBundle, OutValRdyBundle
from new_pmlib    import TestVectorSimulator

#-------------------------------------------------------------------------
# ValRdyQueue
#-------------------------------------------------------------------------
# Test Queue using the ValRdy interface.
class ValRdyQueue( Model ):

  def __init__( s, nbits ):

    s.enq   = InValRdyBundle( nbits )
    s.deq   = OutValRdyBundle( nbits )

    s.full  = Wire( 1 )
    s.wen   = Wire( 1 )

  def elaborate_logic( s ):

    @s.combinational
    def comb():

      s.wen.v      = ~s.full & s.enq.val
      s.enq.rdy.v  = ~s.full
      s.deq.val.v  = s.full

    @s.posedge_clk
    def seq():

      # Data Register
      if s.wen:
        s.deq.msg.next = s.enq.msg

      # Full Bit
      if s.reset:
        s.full.next = 0
      elif   s.deq.rdy and s.deq.val:
        s.full.next = 0
      elif s.enq.rdy and s.enq.val:
        s.full.next = 1
      else:
        s.full.next = s.full

  def line_trace( s ):

    return "{} () {}".format( s.enq, s.deq )

#-------------------------------------------------------------------------
# test_valrdy_sim
#-------------------------------------------------------------------------
def test_valrdy_sim( dump_vcd ):

  test_vectors = [

    # Enqueue one element and then dequeue it
    # enq_val enq_rdy enq_bits deq_val deq_rdy deq_bits
    [ 1,      1,      0x0001,  0,      1,      '?'    ],
    [ 0,      0,      0x0000,  1,      1,      0x0001 ],
    [ 0,      1,      0x0000,  0,      0,      '?'    ],

    # Fill in the queue and enq/deq at the same time
    # enq_val enq_rdy enq_bits deq_val deq_rdy deq_bits
    [ 1,      1,      0x0002,  0,      0,      '?'    ],
    [ 1,      0,      0x0003,  1,      0,      0x0002 ],
    [ 0,      0,      0x0003,  1,      0,      0x0002 ],
    [ 1,      0,      0x0003,  1,      1,      0x0002 ],
    [ 1,      1,      0x0003,  0,      1,      '?'    ],
    [ 1,      0,      0x0004,  1,      1,      0x0003 ],
    [ 1,      1,      0x0004,  0,      1,      '?'    ],
    [ 0,      0,      0x0004,  1,      1,      0x0004 ],
    [ 0,      1,      0x0004,  0,      1,      '?'    ],

  ]

  # Instantiate and elaborate the model

  model = ValRdyQueue( 16 )
  model.elaborate()

  # Define functions mapping the test vector to ports in model

  def tv_in( model, test_vector ):

    model.enq.val.value = test_vector[0]
    model.enq.msg.value = test_vector[2]
    model.deq.rdy.value = test_vector[4]

  def tv_out( model, test_vector ):

    assert model.enq.rdy.value == test_vector[1]
    assert model.deq.val.value == test_vector[3]
    if not test_vector[5] == '?':
      assert model.deq.msg.value == test_vector[5]

  # Run the test

  sim = TestVectorSimulator( model, test_vectors, tv_in, tv_out )
  #if dump_vcd:
  #  sim.dump_vcd( "valrdy_test.vcd" )
  sim.run_test()

##-------------------------------------------------------------------------
## Test Translation
##-------------------------------------------------------------------------
#
#def test_valrdy_translation( ):
#
#    # Create temporary file to write out Verilog
#
#    temp_file = "valrdy_test.v"
#    compile_cmd = ("iverilog -g2005 -Wall -Wno-sensitivity-entire-vector"
#                    "-Wno-sensitivity-entire-array " + temp_file)
#    fd = open( temp_file, 'w' )
#
#    # Instantiate and elaborate model
#
#    model = ValRdyQueue( 16 )
#    model.elaborate()
#
#    # Translate
#
#    code = VerilogTranslationTool( model, fd )
#    fd.close()
#
#    # Make sure translation compiles
#    # TODO: figure out a way to group PortBundles during translation?
#
#    x = os.system( compile_cmd )
#    assert x == 0

<commit_msg>[new_pmlib] add verilog testing to ValRdyBundle<commit_after>#=========================================================================
# ValRdyBundle_test.py
#=========================================================================
# Unit tests for the ValRdy PortBundle.

from new_pymtl    import *
from ValRdyBundle import InValRdyBundle, OutValRdyBundle
from new_pmlib    import TestVectorSimulator

from new_pymtl.translation_tools.verilator_sim import get_verilated

#-------------------------------------------------------------------------
# ValRdyQueue
#-------------------------------------------------------------------------
# Test Queue using the ValRdy interface.
class ValRdyQueue( Model ):

  def __init__( s, nbits ):

    s.enq   = InValRdyBundle ( nbits )
    s.deq   = OutValRdyBundle( nbits )

    s.full  = Wire( 1 )
    s.wen   = Wire( 1 )

  def elaborate_logic( s ):

    @s.combinational
    def comb():

      s.wen.v      = ~s.full & s.enq.val
      s.enq.rdy.v  = ~s.full
      s.deq.val.v  = s.full

    @s.posedge_clk
    def seq():

      # Data Register
      if s.wen:
        s.deq.msg.next = s.enq.msg

      # Full Bit
      if s.reset:
        s.full.next = 0
      elif   s.deq.rdy and s.deq.val:
        s.full.next = 0
      elif s.enq.rdy and s.enq.val:
        s.full.next = 1
      else:
        s.full.next = s.full

  def line_trace( s ):

    return "{} () {}".format( s.enq, s.deq )

#-------------------------------------------------------------------------
# test_valrdy_sim
#-------------------------------------------------------------------------
def test_valrdy_sim( dump_vcd, test_verilog ):

  test_vectors = [

    # Enqueue one element and then dequeue it
    # enq_val enq_rdy enq_bits deq_val deq_rdy deq_bits
    [ 1,      1,      0x0001,  0,      1,      '?'    ],
    [ 0,      0,      0x0000,  1,      1,      0x0001 ],
    [ 0,      1,      0x0000,  0,      0,      '?'    ],

    # Fill in the queue and enq/deq at the same time
    # enq_val enq_rdy enq_bits deq_val deq_rdy deq_bits
    [ 1,      1,      0x0002,  0,      0,      '?'    ],
    [ 1,      0,      0x0003,  1,      0,      0x0002 ],
    [ 0,      0,      0x0003,  1,      0,      0x0002 ],
    [ 1,      0,      0x0003,  1,      1,      0x0002 ],
    [ 1,      1,      0x0003,  0,      1,      '?'    ],
    [ 1,      0,      0x0004,  1,      1,      0x0003 ],
    [ 1,      1,      0x0004,  0,      1,      '?'    ],
    [ 0,      0,      0x0004,  1,      1,      0x0004 ],
    [ 0,      1,      0x0004,  0,      1,      '?'    ],

  ]

  # Instantiate and elaborate the model

  model = ValRdyQueue( 16 )
  if test_verilog:
    model = get_verilated( model )
  model.elaborate()

  # Define functions mapping the test vector to ports in model

  def tv_in( model, test_vector ):

    model.enq.val.value = test_vector[0]
    model.enq.msg.value = test_vector[2]
    model.deq.rdy.value = test_vector[4]

  def tv_out( model, test_vector ):

    assert model.enq.rdy.value == test_vector[1]
    assert model.deq.val.value == test_vector[3]
    if not test_vector[5] == '?':
      assert model.deq.msg.value == test_vector[5]

  # Run the test

  sim = TestVectorSimulator( model, test_vectors, tv_in, tv_out )

  #if dump_vcd:
  #  sim.dump_vcd( "valrdy_test.vcd" )

  sim.run_test()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright © 2012-2014 Roberto Alsina and others.

# Permission is hereby granted, free of charge, to any
# person obtaining a copy of this software and associated
# documentation files (the "Software"), to deal in the
# Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the
# Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice
# shall be included in all copies or substantial portions of
# the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

from __future__ import unicode_literals

import os

try:
    import webassets
except ImportError:
    webassets = None  # NOQA

from nikola.plugin_categories import LateTask
from nikola import utils

class BuildBundles(LateTask):
    """Bundle assets using WebAssets."""

    name = "create_bundles"

    def set_site(self, site):
        super(BuildBundles, self).set_site(site)
        if webassets is None and self.site.config['USE_BUNDLES']:
            utils.req_missing(['webassets'], 'USE_BUNDLES', optional=True)
            utils.LOGGER.warn('Setting USE_BUNDLES to False.')
            self.site.config['USE_BUNDLES'] = False

    def gen_tasks(self):
        """Bundle assets using WebAssets."""

        kw = {
            'filters': self.site.config['FILTERS'],
            'output_folder': self.site.config['OUTPUT_FOLDER'],
            'cache_folder': self.site.config['CACHE_FOLDER'],
            'theme_bundles': get_theme_bundles(self.site.THEMES),
            'themes': self.site.THEMES,
            'files_folders': self.site.config['FILES_FOLDERS'],
            'code_color_scheme': self.site.config['CODE_COLOR_SCHEME'],
        }

        def build_bundle(output, inputs):
            out_dir = os.path.join(kw['output_folder'],
                                   os.path.dirname(output))
            inputs = [i for i in inputs if os.path.isfile(
                os.path.join(out_dir, i))]
            cache_dir = os.path.join(kw['cache_folder'], 'webassets')
            utils.makedirs(cache_dir)
            env = webassets.Environment(out_dir, os.path.dirname(output),
                                        cache=cache_dir)
            if inputs:
                bundle = webassets.Bundle(*inputs, output=os.path.basename(output))
                env.register(output, bundle)
                # This generates the file
                env[output].urls()
            else:
                with open(os.path.join(out_dir, os.path.basename(output)), 'wb+'):
                    pass  # Create empty file

        yield self.group_task()
        if (webassets is not None and self.site.config['USE_BUNDLES'] is not
                False):
            for name, _files in kw['theme_bundles'].items():
                output_path = os.path.join(kw['output_folder'], name)
                dname = os.path.dirname(name)
                files = []
                for fname in _files:
                    # paths are relative to dirname
                    files.append(os.path.join(dname, fname))
                file_dep = [os.path.join(kw['output_folder'], fname)
                            for fname in files if
                            utils.get_asset_path(fname, self.site.THEMES, self.site.config['FILES_FOLDERS'])]
                task = {
                    'file_dep': list(file_dep),
                    'task_dep': ['copy_assets'],
                    'basename': str(self.name),
                    'name': str(output_path),
                    'actions': [(build_bundle, (name, files))],
                    'targets': [output_path],
                    'uptodate': [
                        utils.config_changed({
                            1: kw,
                            2: file_dep
                        })],
                    'clean': True,
                }
                yield utils.apply_filters(task, kw['filters'])

def get_theme_bundles(themes):
    """Given a theme chain, return the bundle definitions."""
    bundles = {}
    for theme_name in themes:
        bundles_path = os.path.join(
            utils.get_theme_path(theme_name), 'bundles')
        if os.path.isfile(bundles_path):
            with open(bundles_path) as fd:
                for line in fd:
                    name, files = line.split('=')
                    files = [f.strip() for f in files.split(',')]
                    bundles[name.strip().replace('/', os.sep)] = files
                break
    return bundles
<commit_msg>and fix #1153 fo'real<commit_after># -*- coding: utf-8 -*-

# Copyright © 2012-2014 Roberto Alsina and others.

# Permission is hereby granted, free of charge, to any
# person obtaining a copy of this software and associated
# documentation files (the "Software"), to deal in the
# Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the
# Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice
# shall be included in all copies or substantial portions of
# the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

from __future__ import unicode_literals

import os

try:
    import webassets
except ImportError:
    webassets = None  # NOQA

from nikola.plugin_categories import LateTask
from nikola import utils

class BuildBundles(LateTask):
    """Bundle assets using WebAssets."""

    name = "create_bundles"

    def set_site(self, site):
        super(BuildBundles, self).set_site(site)
        if webassets is None and self.site.config['USE_BUNDLES']:
            utils.req_missing(['webassets'], 'USE_BUNDLES', optional=True)
            utils.LOGGER.warn('Setting USE_BUNDLES to False.')
            self.site.config['USE_BUNDLES'] = False

    def gen_tasks(self):
        """Bundle assets using WebAssets."""

        kw = {
            'filters': self.site.config['FILTERS'],
            'output_folder': self.site.config['OUTPUT_FOLDER'],
            'cache_folder': self.site.config['CACHE_FOLDER'],
            'theme_bundles': get_theme_bundles(self.site.THEMES),
            'themes': self.site.THEMES,
            'files_folders': self.site.config['FILES_FOLDERS'],
            'code_color_scheme': self.site.config['CODE_COLOR_SCHEME'],
        }

        def build_bundle(output, inputs):
            out_dir = os.path.join(kw['output_folder'],
                                   os.path.dirname(output))
            inputs = [os.path.relpath(i, out_dir) for i in inputs if os.path.isfile(i)]
            cache_dir = os.path.join(kw['cache_folder'], 'webassets')
            utils.makedirs(cache_dir)
            env = webassets.Environment(out_dir, os.path.dirname(output),
                                        cache=cache_dir)
            if inputs:
                bundle = webassets.Bundle(*inputs, output=os.path.basename(output))
                env.register(output, bundle)
                # This generates the file
                env[output].urls()
            else:
                with open(os.path.join(out_dir, os.path.basename(output)), 'wb+'):
                    pass  # Create empty file

        yield self.group_task()
        if (webassets is not None and self.site.config['USE_BUNDLES'] is not
                False):
            for name, _files in kw['theme_bundles'].items():
                output_path = os.path.join(kw['output_folder'], name)
                dname = os.path.dirname(name)
                files = []
                for fname in _files:
                    # paths are relative to dirname
                    files.append(os.path.join(dname, fname))
                file_dep = [os.path.join(kw['output_folder'], fname)
                            for fname in files if
                            utils.get_asset_path(fname, self.site.THEMES, self.site.config['FILES_FOLDERS'])]
                task = {
                    'file_dep': list(file_dep),
                    'task_dep': ['copy_assets'],
                    'basename': str(self.name),
                    'name': str(output_path),
                    'actions': [(build_bundle, (name, file_dep))],
                    'targets': [output_path],
                    'uptodate': [
                        utils.config_changed({
                            1: kw,
                            2: file_dep
                        })],
                    'clean': True,
                }
                yield utils.apply_filters(task, kw['filters'])

def get_theme_bundles(themes):
    """Given a theme chain, return the bundle definitions."""
    bundles = {}
    for theme_name in themes:
        bundles_path = os.path.join(
            utils.get_theme_path(theme_name), 'bundles')
        if os.path.isfile(bundles_path):
            with open(bundles_path) as fd:
                for line in fd:
                    name, files = line.split('=')
                    files = [f.strip() for f in files.split(',')]
                    bundles[name.strip().replace('/', os.sep)] = files
                break
    return bundles
<|endoftext|>


<commit_before>import unittest
from ..utils.immutabledict import ImmutableDict
from ..utils.disallowedmodification import DisallowedModification

class ImmutableDictNamedConstructorTest(unittest.TestCase):
  def test_construct_immutable_dict(self):
    my_dict = ImmutableDict.of(foo='bar', baz='quux')
    self.assertEquals('bar', my_dict['foo'])
    self.assertEquals('quux', my_dict['baz'])

class ImmutableDictModificationTest(unittest.TestCase):
  def setUp(self):
    self.test_dict = ImmutableDict({'foo': 'bar'})

  def test_setting_values_should_throw_an_exception(self):
    try:
      self.test_dict['foo'] = 'new_value'
      self.fail()
    except DisallowedModification:
      pass
    self.assertEquals('bar', self.test_dict['foo'])

  def test_adding_new_values_should_throw_an_exception(self):
    try:
      self.test_dict['baz'] = 'new_value'
      self.fail()
    except DisallowedModification:
      pass
    self.assertTrue('baz' not in self.test_dict)

  def test_pop_should_throw_an_exception(self):
    try:
      self.test_dict.pop('foo')
      self.fail()
    except DisallowedModification:
      pass
    self.assertTrue('foo' in self.test_dict)

  def test_popitem_should_throw_an_exception(self):
    try:
      self.test_dict.popitem()
      self.fail()
    except DisallowedModification:
      pass
    self.assertTrue('foo' in self.test_dict)

  def test_setdefault_should_throw_an_exception(self):
    try:
      self.test_dict.setdefault('baz', 'quux')
      self.fail()
    except DisallowedModification:
      pass
    self.assertTrue('baz' not in self.test_dict)

  def test_update_should_throw_an_exception(self):
    try:
      self.test_dict.update(baz='quux')
      self.fail()
    except DisallowedModification:
      pass
    self.assertTrue('baz' not in self.test_dict)

if __name__ == '__main__':
  unittest.main()
<commit_msg>Cleaned up the test a bit<commit_after>import unittest
from ..utils.immutabledict import ImmutableDict
from ..utils.disallowedmodification import DisallowedModification

class ImmutableDictNamedConstructorTest(unittest.TestCase):
  def test_construct_immutable_dict(self):
    my_dict = ImmutableDict.of(foo='bar', baz='quux')
    self.assertEquals('bar', my_dict['foo'])
    self.assertEquals('quux', my_dict['baz'])

class ImmutableDictModificationTest(unittest.TestCase):
  def setUp(self):
    self.test_dict = ImmutableDict({'foo': 'bar'})

  def test_setting_values_should_throw_an_exception(self):
    try:
      self.test_dict['foo'] = 'new_value'
      self.fail()
    except DisallowedModification:
      pass
    self.assertEquals('bar', self.test_dict['foo'])

  def test_adding_new_values_should_throw_an_exception(self):
    try:
      self.test_dict['baz'] = 'new_value'
      self.fail()
    except DisallowedModification:
      pass
    self.assertTrue('baz' not in self.test_dict)

  def test_pop_should_throw_an_exception(self):
    self.assertRaises(DisallowedModification, self.test_dict.pop, 'foo')
    self.assertTrue('foo' in self.test_dict)

  def test_popitem_should_throw_an_exception(self):
    self.assertRaises(DisallowedModification, self.test_dict.popitem)
    self.assertTrue('foo' in self.test_dict)

  def test_setdefault_should_throw_an_exception(self):
    self.assertRaises(DisallowedModification, self.test_dict.setdefault, 'baz',
        'quux')
    self.assertTrue('baz' not in self.test_dict)

  def test_update_should_throw_an_exception(self):
    try:
      self.test_dict.update(baz='quux')
      self.fail()
    except DisallowedModification:
      pass
    self.assertTrue('baz' not in self.test_dict)

if __name__ == '__main__':
  unittest.main()
<|endoftext|>


<commit_before>"""apache_configurator_test - unittests

A series of basic full integration tests to ensure that Letsencrypt is
still running smoothly.

.. note:: This code is not complete
.. note:: Do not document this code... it will change quickly

"""

import re
import os
import shutil
import sys
import unittest

from letsencrypt.client import apache_configurator
from letsencrypt.client import CONFIG
from letsencrypt.client import display
from letsencrypt.client import logger

# Some of these will likely go into a letsencrypt.tests.CONFIG file
TESTING_DIR = os.path.dirname(os.path.realpath(__file__))
UBUNTU_CONFIGS = os.path.join(TESTING_DIR, "debian_apache_2_4/")
TEMP_DIR = os.path.join(TESTING_DIR, "temp")

def setUpModule():
    """Run once before all unittests."""
    logger.setLogger(logger.FileLogger(sys.stdout))
    logger.setLogLevel(logger.INFO)
    display.set_display(display.NcursesDisplay())

    if not os.path.isdir(UBUNTU_CONFIGS):
        print "Please place the configuration directory: %s" % UBUNTU_CONFIGS
        sys.exit(1)

    shutil.copytree(UBUNTU_CONFIGS, TEMP_DIR, symlinks=True)

def tearDownModule():
    """Run once after all unittests."""
    shutil.rmtree(TEMP_DIR)

class TwoVhost80(unittest.TestCase):
    """Standard two http vhosts that are well configured."""

    def setUp(self):
        """Run before each and every tests."""
        # Final slash is currently important
        self.config_path = os.path.join(TEMP_DIR, "two_vhost_80/apache2/")

        # Using a new configurator every time allows the Configurator to clean
        # up after itself
        self.config = apache_configurator.ApacheConfigurator(
            self.config_path,
            {"backup": os.path.join(TESTING_DIR, "backups"),
             "temp": os.path.join(TESTING_DIR, "temp_checkpoint"),
             "progress": os.path.join(TESTING_DIR, "backups", "IN_PROGRESS"),
             "config": os.path.join(TESTING_DIR, "config"),
             "work": os.path.join(TESTING_DIR, "work")},
            (2, 4, 7))

        self.aug_path = "/files" + self.config_path

        prefix = os.path.join(TEMP_DIR, "two_vhost_80/apache2/sites-available/")
        aug_pre = "/files" + prefix
        self.vh_truth = []
        self.vh_truth.append(apache_configurator.VH(
            os.path.join(prefix + "encryption-example.conf"),
            os.path.join(aug_pre, "encryption-example.conf/VirtualHost"),
            ["*:80"], False, True))
        self.vh_truth.append(apache_configurator.VH(
            os.path.join(prefix, "default-ssl.conf"),
            os.path.join(aug_pre, "default-ssl.conf/IfModule/VirtualHost"),
            ["_default_:443"], True, False))
        self.vh_truth.append(apache_configurator.VH(
            os.path.join(prefix, "000-default.conf"),
            os.path.join(aug_pre, "000-default.conf/VirtualHost"),
            ["*:80"], False, True))
        self.vh_truth.append(apache_configurator.VH(
            os.path.join(prefix, "letsencrypt.conf"),
            os.path.join(aug_pre, "letsencrypt.conf/VirtualHost"),
            ["*:80"], False, True))
        self.vh_truth[0].add_name("encryption-example.demo")
        self.vh_truth[2].add_name("ip-172-30-0-17")
        self.vh_truth[3].add_name("letsencrypt.demo")

    def test_parse_file(self):
        """test parse_file.

        letsencrypt.conf is chosen as the test file as it will not be
        included during the normal course of execution.

        """
        file_path = os.path.join(
            self.config_path, "sites-available", "letsencrypt.conf")
        self.config._parse_file(file_path)

        # search for the httpd incl
        matches = self.config.aug.match(
            "/augeas/load/Httpd/incl [. ='%s']" % file_path)

        self.assertTrue(matches)

    def test_get_all_names(self):
        """test get_all_names."""
        names = self.config.get_all_names()
        self.assertTrue(set(names) == set(
            ['letsencrypt.demo', 'encryption-example.demo', 'ip-172-30-0-17']))

    def test_find_directive(self):
        """test find_directive."""
        test = self.config.find_directive(
            apache_configurator.case_i("Listen"), "443")
        # This will only look in enabled hosts
        test2 = self.config.find_directive(
            apache_configurator.case_i("documentroot"))
        self.assertTrue(len(test) == 2)
        self.assertTrue(len(test2) == 3)

    def test_get_virtual_hosts(self):
        """inefficient get_virtual_hosts check."""
        vhs = self.config.get_virtual_hosts()
        self.assertTrue(len(vhs) == 4)
        found = 0
        for vhost in vhs:
            for truth in self.vh_truth:
                if vhost == truth:
                    found += 1
                    break

        self.assertTrue(found == 4)

    def test_is_site_enabled(self):
        """test is_site_enabled"""
        self.assertTrue(self.config.is_site_enabled(self.vh_truth[0].file))
        self.assertTrue(not self.config.is_site_enabled(self.vh_truth[1].file))
        self.assertTrue(self.config.is_site_enabled(self.vh_truth[2].file))
        self.assertTrue(self.config.is_site_enabled(self.vh_truth[3].file))

    def test_add_dir(self):
        """test add_dir."""
        aug_default = "/files" + self.config.location["default"]
        self.config.add_dir(
            aug_default, "AddDirective", "test")

        self.assertTrue(
            self.config.find_directive("AddDirective", "test", aug_default))

    def test_deploy_cert(self):
        """test deploy_cert.

        This test modifies the default-ssl vhost SSL directives.

        """
        self.config.deploy_cert(
            self.vh_truth[1],
            "example/cert.pem", "example/key.pem", "example/cert_chain.pem")

        loc_cert = self.config.find_directive(
            apache_configurator.case_i("sslcertificatefile"),
            re.escape("example/cert.pem"), self.vh_truth[1].path)
        loc_key = self.config.find_directive(
            apache_configurator.case_i("sslcertificateKeyfile"),
            re.escape("example/key.pem"), self.vh_truth[1].path)
        loc_chain = self.config.find_directive(
            apache_configurator.case_i("SSLCertificateChainFile"),
            re.escape("example/cert_chain.pem"), self.vh_truth[1].path)

        # debug_file(self.vh_truth[1].file)

        # Verify one directive was found in the correct file
        self.assertTrue(len(loc_cert) == 1 and
                        apache_configurator.get_file_path(loc_cert[0]) ==
                        self.vh_truth[1].file)

        self.assertTrue(len(loc_key) == 1 and
                        apache_configurator.get_file_path(loc_key[0]) ==
                        self.vh_truth[1].file)

        self.assertTrue(len(loc_chain) == 1 and
                        apache_configurator.get_file_path(loc_chain[0]) ==
                        self.vh_truth[1].file)

    def test_is_name_vhost(self):
        """test is_name_vhost."""
        self.assertTrue(self.config.is_name_vhost("*:80"))
        self.config.version = (2, 2)
        self.assertFalse(self.config.is_name_vhost("*:80"))

    def test_add_name_vhost(self):
        """test add_name_vhost."""
        self.config.add_name_vhost("*:443")
        # self.config.save(temporary=True)
        self.assertTrue(self.config.find_directive(
            "NameVirtualHost", re.escape("*:443")))

    def test_add_dir_to_ifmodssl(self):
        """test _add_dir_to_ifmodssl.

        .. todo:: test what happens when a bad path is given... ie. ports.conf
            doesn't exist

        """
        self.config._add_dir_to_ifmodssl(
            "/files" + self.config.location["default"], "FakeDirective", "123")

        matches = self.config.find_directive("FakeDirective", "123")

        self.assertTrue(len(matches) == 1)
        self.assertTrue("IfModule" in matches[0])

    def test_make_vhost_ssl(self):
        """test make_vhost_ssl."""
        ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0])

        self.assertTrue(
            ssl_vhost.file ==
            os.path.join(self.config_path, "sites-available",
                         "encryption-example-le-ssl.conf"))

        self.assertTrue(ssl_vhost.path ==
                        "/files" + ssl_vhost.file + "/IfModule/VirtualHost")
        self.assertTrue(ssl_vhost.addrs == ["*:443"])
        self.assertTrue(ssl_vhost.names == ["encryption-example.demo"])
        self.assertTrue(ssl_vhost.ssl)
        self.assertFalse(ssl_vhost.enabled)

        self.assertTrue(self.config.find_directive(
            "SSLCertificateFile", None, ssl_vhost.path))
        self.assertTrue(self.config.find_directive(
            "SSLCertificateKeyFile", None, ssl_vhost.path))
        self.assertTrue(self.config.find_directive(
            "Include", CONFIG.OPTIONS_SSL_CONF, ssl_vhost.path))

        self.assertTrue(self.config.is_name_vhost(self.vh_truth[0]) ==
                        self.config.is_name_vhost(ssl_vhost))

        self.assertTrue(len(self.config.vhosts) == 5)

    # def _verify_redirect(self, config_path):
    #     """Verifies that the vhost contains the REWRITE."""
    #     with open(config_path, 'r') as config_fd:
    #         conf = config_fd.read()

    #     return CONFIG.REWRITE_HTTPS_ARGS[1] in conf

def debug_file(filepath):
    """Print out the file."""
    with open(filepath, 'r')as file_d:
        print file_d.read()

if __name__ == '__main__':
    unittest.main()
<commit_msg>Mock out check_ssl_loaded<commit_after>"""apache_configurator_test - unittests

A series of basic full integration tests to ensure that Letsencrypt is
still running smoothly.

.. note:: This code is not complete
.. note:: Do not document this code... it will change quickly

"""

import mock
import re
import os
import shutil
import sys
import unittest

from letsencrypt.client import apache_configurator
from letsencrypt.client import CONFIG
from letsencrypt.client import display
from letsencrypt.client import logger

# Some of these will likely go into a letsencrypt.tests.CONFIG file
TESTING_DIR = os.path.dirname(os.path.realpath(__file__))
UBUNTU_CONFIGS = os.path.join(TESTING_DIR, "debian_apache_2_4/")
TEMP_DIR = os.path.join(TESTING_DIR, "temp")

def setUpModule():
    """Run once before all unittests."""
    logger.setLogger(logger.FileLogger(sys.stdout))
    logger.setLogLevel(logger.INFO)
    display.set_display(display.NcursesDisplay())

    if not os.path.isdir(UBUNTU_CONFIGS):
        print "Please place the configuration directory: %s" % UBUNTU_CONFIGS
        sys.exit(1)

    shutil.copytree(UBUNTU_CONFIGS, TEMP_DIR, symlinks=True)

def tearDownModule():
    """Run once after all unittests."""
    shutil.rmtree(TEMP_DIR)

class TwoVhost80(unittest.TestCase):
    """Standard two http vhosts that are well configured."""

    @mock.patch("letsencrypt.client.apache_configurator."
                "subprocess.Popen.communicate")
    def setUp(self, mock_subprocess):
        """Run before each and every tests."""

        # This just states that the ssl module is already loaded
        mock_subprocess.return_value = ("ssl_module", "")

        # Final slash is currently important
        self.config_path = os.path.join(TEMP_DIR, "two_vhost_80/apache2/")

        self.config = apache_configurator.ApacheConfigurator(
            self.config_path,
            {"backup": os.path.join(TESTING_DIR, "backups"),
             "temp": os.path.join(TESTING_DIR, "temp_checkpoint"),
             "progress": os.path.join(TESTING_DIR, "backups", "IN_PROGRESS"),
             "config": os.path.join(TESTING_DIR, "config"),
             "work": os.path.join(TESTING_DIR, "work")},
            (2, 4, 7))

        self.aug_path = "/files" + self.config_path

        prefix = os.path.join(TEMP_DIR, "two_vhost_80/apache2/sites-available/")
        aug_pre = "/files" + prefix
        self.vh_truth = []
        self.vh_truth.append(apache_configurator.VH(
            os.path.join(prefix + "encryption-example.conf"),
            os.path.join(aug_pre, "encryption-example.conf/VirtualHost"),
            ["*:80"], False, True))
        self.vh_truth.append(apache_configurator.VH(
            os.path.join(prefix, "default-ssl.conf"),
            os.path.join(aug_pre, "default-ssl.conf/IfModule/VirtualHost"),
            ["_default_:443"], True, False))
        self.vh_truth.append(apache_configurator.VH(
            os.path.join(prefix, "000-default.conf"),
            os.path.join(aug_pre, "000-default.conf/VirtualHost"),
            ["*:80"], False, True))
        self.vh_truth.append(apache_configurator.VH(
            os.path.join(prefix, "letsencrypt.conf"),
            os.path.join(aug_pre, "letsencrypt.conf/VirtualHost"),
            ["*:80"], False, True))
        self.vh_truth[0].add_name("encryption-example.demo")
        self.vh_truth[2].add_name("ip-172-30-0-17")
        self.vh_truth[3].add_name("letsencrypt.demo")

    def test_parse_file(self):
        """test parse_file.

        letsencrypt.conf is chosen as the test file as it will not be
        included during the normal course of execution.

        """
        file_path = os.path.join(
            self.config_path, "sites-available", "letsencrypt.conf")
        self.config._parse_file(file_path)

        # search for the httpd incl
        matches = self.config.aug.match(
            "/augeas/load/Httpd/incl [. ='%s']" % file_path)

        self.assertTrue(matches)

    def test_get_all_names(self):
        """test get_all_names."""
        names = self.config.get_all_names()
        self.assertTrue(set(names) == set(
            ['letsencrypt.demo', 'encryption-example.demo', 'ip-172-30-0-17']))

    def test_find_directive(self):
        """test find_directive."""
        test = self.config.find_directive(
            apache_configurator.case_i("Listen"), "443")
        # This will only look in enabled hosts
        test2 = self.config.find_directive(
            apache_configurator.case_i("documentroot"))
        self.assertTrue(len(test) == 2)
        self.assertTrue(len(test2) == 3)

    def test_get_virtual_hosts(self):
        """inefficient get_virtual_hosts check."""
        vhs = self.config.get_virtual_hosts()
        self.assertTrue(len(vhs) == 4)
        found = 0
        for vhost in vhs:
            for truth in self.vh_truth:
                if vhost == truth:
                    found += 1
                    break

        self.assertTrue(found == 4)

    def test_is_site_enabled(self):
        """test is_site_enabled"""
        self.assertTrue(self.config.is_site_enabled(self.vh_truth[0].file))
        self.assertTrue(not self.config.is_site_enabled(self.vh_truth[1].file))
        self.assertTrue(self.config.is_site_enabled(self.vh_truth[2].file))
        self.assertTrue(self.config.is_site_enabled(self.vh_truth[3].file))

    def test_add_dir(self):
        """test add_dir."""
        aug_default = "/files" + self.config.location["default"]
        self.config.add_dir(
            aug_default, "AddDirective", "test")

        self.assertTrue(
            self.config.find_directive("AddDirective", "test", aug_default))

    def test_deploy_cert(self):
        """test deploy_cert.

        This test modifies the default-ssl vhost SSL directives.

        """
        self.config.deploy_cert(
            self.vh_truth[1],
            "example/cert.pem", "example/key.pem", "example/cert_chain.pem")

        loc_cert = self.config.find_directive(
            apache_configurator.case_i("sslcertificatefile"),
            re.escape("example/cert.pem"), self.vh_truth[1].path)
        loc_key = self.config.find_directive(
            apache_configurator.case_i("sslcertificateKeyfile"),
            re.escape("example/key.pem"), self.vh_truth[1].path)
        loc_chain = self.config.find_directive(
            apache_configurator.case_i("SSLCertificateChainFile"),
            re.escape("example/cert_chain.pem"), self.vh_truth[1].path)

        # debug_file(self.vh_truth[1].file)

        # Verify one directive was found in the correct file
        self.assertTrue(len(loc_cert) == 1 and
                        apache_configurator.get_file_path(loc_cert[0]) ==
                        self.vh_truth[1].file)

        self.assertTrue(len(loc_key) == 1 and
                        apache_configurator.get_file_path(loc_key[0]) ==
                        self.vh_truth[1].file)

        self.assertTrue(len(loc_chain) == 1 and
                        apache_configurator.get_file_path(loc_chain[0]) ==
                        self.vh_truth[1].file)

    def test_is_name_vhost(self):
        """test is_name_vhost."""
        self.assertTrue(self.config.is_name_vhost("*:80"))
        self.config.version = (2, 2)
        self.assertFalse(self.config.is_name_vhost("*:80"))

    def test_add_name_vhost(self):
        """test add_name_vhost."""
        self.config.add_name_vhost("*:443")
        # self.config.save(temporary=True)
        self.assertTrue(self.config.find_directive(
            "NameVirtualHost", re.escape("*:443")))

    def test_add_dir_to_ifmodssl(self):
        """test _add_dir_to_ifmodssl.

        .. todo:: test what happens when a bad path is given... ie. ports.conf
            doesn't exist

        """
        self.config._add_dir_to_ifmodssl(
            "/files" + self.config.location["default"], "FakeDirective", "123")

        matches = self.config.find_directive("FakeDirective", "123")

        self.assertTrue(len(matches) == 1)
        self.assertTrue("IfModule" in matches[0])

    def test_make_vhost_ssl(self):
        """test make_vhost_ssl."""
        ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0])

        self.assertTrue(
            ssl_vhost.file ==
            os.path.join(self.config_path, "sites-available",
                         "encryption-example-le-ssl.conf"))

        self.assertTrue(ssl_vhost.path ==
                        "/files" + ssl_vhost.file + "/IfModule/VirtualHost")
        self.assertTrue(ssl_vhost.addrs == ["*:443"])
        self.assertTrue(ssl_vhost.names == ["encryption-example.demo"])
        self.assertTrue(ssl_vhost.ssl)
        self.assertFalse(ssl_vhost.enabled)

        self.assertTrue(self.config.find_directive(
            "SSLCertificateFile", None, ssl_vhost.path))
        self.assertTrue(self.config.find_directive(
            "SSLCertificateKeyFile", None, ssl_vhost.path))
        self.assertTrue(self.config.find_directive(
            "Include", CONFIG.OPTIONS_SSL_CONF, ssl_vhost.path))

        self.assertTrue(self.config.is_name_vhost(self.vh_truth[0]) ==
                        self.config.is_name_vhost(ssl_vhost))

        self.assertTrue(len(self.config.vhosts) == 5)

    # def _verify_redirect(self, config_path):
    #     """Verifies that the vhost contains the REWRITE."""
    #     with open(config_path, 'r') as config_fd:
    #         conf = config_fd.read()

    #     return CONFIG.REWRITE_HTTPS_ARGS[1] in conf

def debug_file(filepath):
    """Print out the file."""
    with open(filepath, 'r')as file_d:
        print file_d.read()

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>import csv
import logging

from datetime import date, datetime
from django.utils.timezone import now
from django.db import models
from django.core.management.base import BaseCommand, CommandError
from reversion import revisions as reversion
from reversion.models import Version
from ....users.models import Editor
from ....applications.models import Application
from ....resources.models import Partner, Stream

logger = logging.getLogger(__name__)

class Command(BaseCommand):

    # Input file really needs to be UTF-8 encoded. We should do some sort of
    # assertion for that.

    def add_arguments(self, parser):
        parser.add_argument('file')

    def handle(self, *args, **options):
        with open(options['file']) as f:
               reader = csv.reader(f)
               # Skip first row, we expect it to be a header.
               next(reader, None)  # skip the headers
               for row in reader:
                   try:
                       # If the collections field is empty, make it a list containing an empty string.
                       if row[4] in (None, ""):
                           specific_stream_ids = ['']
                       # Otherwise try to split it on comma
                       else:
                           specific_stream_ids = row[4].split(',')

                       # If the title field is empty, make it a list containing an empty string.
                       if row[5] in (None, ""):
                           specific_titles = ['']
                       # Otherwise try to split it on semicolon
                       else:
                           specific_titles = row[5].split(';')

                       for specific_stream_id in specific_stream_ids:

                           for specific_title in specific_titles:

                               # Where possible we're fetching the objects attached
                               # to our attributes to verify they already exist.

                               partner_id = row[0]
                               partner = Partner.objects.get(pk=partner_id)

                               # Inconsistent date format on the input files
                               # If the date field is empty, set it to the beginning of (Unix) time.
                               if row[1] in (None, ""):
                                   datetime_created = datetime.strptime('01/01/1970 00:00:00', '%d/%m/%Y %H:%M:%S')

                               # If we have a date field
                               else:
                                   # Try fetching the most precise timestamp
                                   try:
                                       datetime_created = datetime.strptime(row[1], '%d/%m/%Y %H:%M:%S')

                                   except ValueError:
                                       # If that doesn't work, try getting a less precise timestamp
                                       try:
                                           datetime_created = datetime.strptime(row[1], '%d/%m/%Y %H:%M')

                                       except ValueError:
                                           # If that doesn't work, try getting a date
                                           try:
                                               datetime_created = datetime.strptime(row[1], '%d/%m/%Y')
                                           except:
                                               pass

                               date_created = datetime_created

                               wp_username = self.normalize_wp_username(row[2])
                               editor = Editor.objects.get(wp_username=wp_username)
                               editor_id = editor.pk

                               try:
                                   stream = Stream.objects.get(pk=specific_stream_id)
                               except:
                                   specific_stream_id = None
                                   stream = None

                               import_note = 'Imported on ' + str(date.today()) + '.'

                               try:
                                   application = Application.objects.get(
                                       partner_id = partner_id,
                                       date_created = date_created,
                                       date_closed = date_created,
                                       editor_id = editor_id,
                                       specific_stream_id = specific_stream_id,
                                       specific_title = specific_title,
                                       imported = True,
                                       status = 4
                                   )
                               except Application.DoesNotExist:
                                   application = Application(
                                       partner_id = partner_id,
                                       date_created = date_created,
                                       date_closed = date_created,
                                       editor_id = editor_id,
                                       specific_stream_id = specific_stream_id,
                                       specific_title = specific_title,
                                       comments = import_note,
                                       rationale = import_note,
                                       imported = True,
                                       status = 4
                                   )
                                   with reversion.create_revision():
                                       reversion.set_date_created(datetime_created)
                                       application.save()
                                       logger.info("Application created.")
                   except:
                       logger.exception("Unable to create {wp_username}'s application to {partner_id}.".format(wp_username=normalize_wp_username(row[2]),partner_id=row[0]))
                       pass

    # Cribbed from stack overflow
    # https://stackoverflow.com/a/32232764
    # WP Usernames are uppercase and have spaces, not underscores
    def normalize_wp_username(self, wp_username):
        wp_username = wp_username.strip()
        wp_username = wp_username.replace('_', ' ')
        wp_username = wp_username[0].upper() + wp_username[1:]

        return wp_username
<commit_msg>call normalize_wp_username correctly in application import.<commit_after>import csv
import logging

from datetime import date, datetime
from django.utils.timezone import now
from django.db import models
from django.core.management.base import BaseCommand, CommandError
from reversion import revisions as reversion
from reversion.models import Version
from ....users.models import Editor
from ....applications.models import Application
from ....resources.models import Partner, Stream

logger = logging.getLogger(__name__)

class Command(BaseCommand):

    # Input file really needs to be UTF-8 encoded. We should do some sort of
    # assertion for that.

    def add_arguments(self, parser):
        parser.add_argument('file')

    def handle(self, *args, **options):
        with open(options['file']) as f:
               reader = csv.reader(f)
               # Skip first row, we expect it to be a header.
               next(reader, None)  # skip the headers
               for row in reader:
                   try:
                       # If the collections field is empty, make it a list containing an empty string.
                       if row[4] in (None, ""):
                           specific_stream_ids = ['']
                       # Otherwise try to split it on comma
                       else:
                           specific_stream_ids = row[4].split(',')

                       # If the title field is empty, make it a list containing an empty string.
                       if row[5] in (None, ""):
                           specific_titles = ['']
                       # Otherwise try to split it on semicolon
                       else:
                           specific_titles = row[5].split(';')

                       for specific_stream_id in specific_stream_ids:

                           for specific_title in specific_titles:

                               # Where possible we're fetching the objects attached
                               # to our attributes to verify they already exist.

                               partner_id = row[0]
                               partner = Partner.objects.get(pk=partner_id)

                               # Inconsistent date format on the input files
                               # If the date field is empty, set it to the beginning of (Unix) time.
                               if row[1] in (None, ""):
                                   datetime_created = datetime.strptime('01/01/1970 00:00:00', '%d/%m/%Y %H:%M:%S')

                               # If we have a date field
                               else:
                                   # Try fetching the most precise timestamp
                                   try:
                                       datetime_created = datetime.strptime(row[1], '%d/%m/%Y %H:%M:%S')

                                   except ValueError:
                                       # If that doesn't work, try getting a less precise timestamp
                                       try:
                                           datetime_created = datetime.strptime(row[1], '%d/%m/%Y %H:%M')

                                       except ValueError:
                                           # If that doesn't work, try getting a date
                                           try:
                                               datetime_created = datetime.strptime(row[1], '%d/%m/%Y')
                                           except:
                                               pass

                               date_created = datetime_created

                               wp_username = self.normalize_wp_username(row[2])
                               editor = Editor.objects.get(wp_username=wp_username)
                               editor_id = editor.pk

                               try:
                                   stream = Stream.objects.get(pk=specific_stream_id)
                               except:
                                   specific_stream_id = None
                                   stream = None

                               import_note = 'Imported on ' + str(date.today()) + '.'

                               try:
                                   application = Application.objects.get(
                                       partner_id = partner_id,
                                       date_created = date_created,
                                       date_closed = date_created,
                                       editor_id = editor_id,
                                       specific_stream_id = specific_stream_id,
                                       specific_title = specific_title,
                                       imported = True,
                                       status = 4
                                   )
                               except Application.DoesNotExist:
                                   application = Application(
                                       partner_id = partner_id,
                                       date_created = date_created,
                                       date_closed = date_created,
                                       editor_id = editor_id,
                                       specific_stream_id = specific_stream_id,
                                       specific_title = specific_title,
                                       comments = import_note,
                                       rationale = import_note,
                                       imported = True,
                                       status = 4
                                   )
                                   with reversion.create_revision():
                                       reversion.set_date_created(datetime_created)
                                       application.save()
                                       logger.info("Application created.")
                   except:
                       logger.exception("Unable to create {wp_username}'s application to {partner_id}.".format(wp_username=self.normalize_wp_username(row[2]),partner_id=row[0]))
                       pass

    # Cribbed from stack overflow
    # https://stackoverflow.com/a/32232764
    # WP Usernames are uppercase and have spaces, not underscores
    def normalize_wp_username(self, wp_username):
        wp_username = wp_username.strip()
        wp_username = wp_username.replace('_', ' ')
        wp_username = wp_username[0].upper() + wp_username[1:]

        return wp_username
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Gamma Colour Component Transfer Function
========================================

Defines gamma encoding / decoding colour component transfer function related
objects:

- :func:`gamma_function`

See Also
--------
`RGB Colourspaces Jupyter Notebook
<http://nbviewer.jupyter.org/github/colour-science/colour-notebooks/\
blob/master/notebooks/models/rgb.ipynb>`_
"""

from __future__ import division, unicode_literals

import numpy as np

from colour.utilities import as_numeric

__author__ = 'Colour Developers'
__copyright__ = 'Copyright (C) 2013-2016 - Colour Developers'
__license__ = 'New BSD License - http://opensource.org/licenses/BSD-3-Clause'
__maintainer__ = 'Colour Developers'
__email__ = 'colour-science@googlegroups.com'
__status__ = 'Production'

__all__ = ['gamma_function']

def gamma_function(a,
                   exponent=1,
                   negative_number_handling='indeterminate'):
    """
    Defines a typical gamma encoding / decoding function.

    Parameters
    ----------
    a : numeric or array_like
        Array to encode / decode.
    exponent : numeric or array_like, optional
        Encoding / decoding exponent.
    negative_number_handling : unicode, optional
        **{'Indeterminate', 'Mirror', 'Preserve', 'Clamp'}**,
        Defines the behaviour for `a` negative numbers and / or the definition
        return value:

        -   *Indeterminate*: The behaviour will be indeterminate and
            definition return value might contain *nans*.
        -   *Mirror*: The definition return value will be mirrored around
            abscissa and ordinate axis, i.e. Blackmagic Design: Davinci Resolve
            behaviour.
        -   *Preserve*: The definition will preserve any negative number in
            `a`, i.e. The Foundry Nuke behaviour.
        -   *Clamp*: The definition will clamp any negative number in `a` to 0.

    Returns
    -------
    numeric or ndarray
        Encoded / decoded array.

    Examples
    --------
    >>> gamma_function(0.18, 2.2)  # doctest: +ELLIPSIS
    0.0229932...
    >>> gamma_function(-0.18, 2.0)  # doctest: +ELLIPSIS
    0.0323999...
    >>> gamma_function(-0.18, 2.2)
    nan
    >>> gamma_function(-0.18, 2.2, 'Mirror')  # doctest: +ELLIPSIS
    -0.0229932...
    >>> gamma_function(-0.18, 2.2, 'Preserve')  # doctest: +ELLIPSIS
    -0.1...
    >>> gamma_function(-0.18, 2.2, 'Clamp')  # doctest: +ELLIPSIS
    0.0
    """

    a = np.asarray(a)
    exponent = np.asarray(exponent)

    negative_number_handling = negative_number_handling.lower()
    if negative_number_handling == 'indeterminate':
        return as_numeric(a ** exponent)
    elif negative_number_handling == 'mirror':
        return as_numeric(np.sign(a) * np.abs(a) ** exponent)
    elif negative_number_handling == 'preserve':
        return as_numeric(np.where(a < 0, a, a ** exponent))
    elif negative_number_handling == 'clamp':
        return as_numeric(np.where(a < 0, 0, a ** exponent))
<commit_msg>Raises “ValueError” exception if negative number handling method is undefined in “colour.gamma_function” definition.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Gamma Colour Component Transfer Function
========================================

Defines gamma encoding / decoding colour component transfer function related
objects:

- :func:`gamma_function`

See Also
--------
`RGB Colourspaces Jupyter Notebook
<http://nbviewer.jupyter.org/github/colour-science/colour-notebooks/\
blob/master/notebooks/models/rgb.ipynb>`_
"""

from __future__ import division, unicode_literals

import numpy as np

from colour.utilities import as_numeric

__author__ = 'Colour Developers'
__copyright__ = 'Copyright (C) 2013-2016 - Colour Developers'
__license__ = 'New BSD License - http://opensource.org/licenses/BSD-3-Clause'
__maintainer__ = 'Colour Developers'
__email__ = 'colour-science@googlegroups.com'
__status__ = 'Production'

__all__ = ['gamma_function']

def gamma_function(a,
                   exponent=1,
                   negative_number_handling='indeterminate'):
    """
    Defines a typical gamma encoding / decoding function.

    Parameters
    ----------
    a : numeric or array_like
        Array to encode / decode.
    exponent : numeric or array_like, optional
        Encoding / decoding exponent.
    negative_number_handling : unicode, optional
        **{'Indeterminate', 'Mirror', 'Preserve', 'Clamp'}**,
        Defines the behaviour for `a` negative numbers and / or the definition
        return value:

        -   *Indeterminate*: The behaviour will be indeterminate and
            definition return value might contain *nans*.
        -   *Mirror*: The definition return value will be mirrored around
            abscissa and ordinate axis, i.e. Blackmagic Design: Davinci Resolve
            behaviour.
        -   *Preserve*: The definition will preserve any negative number in
            `a`, i.e. The Foundry Nuke behaviour.
        -   *Clamp*: The definition will clamp any negative number in `a` to 0.

    Returns
    -------
    numeric or ndarray
        Encoded / decoded array.

    Raises
    ------
    ValueError
        If the negative number handling method is not defined.

    Examples
    --------
    >>> gamma_function(0.18, 2.2)  # doctest: +ELLIPSIS
    0.0229932...
    >>> gamma_function(-0.18, 2.0)  # doctest: +ELLIPSIS
    0.0323999...
    >>> gamma_function(-0.18, 2.2)
    nan
    >>> gamma_function(-0.18, 2.2, 'Mirror')  # doctest: +ELLIPSIS
    -0.0229932...
    >>> gamma_function(-0.18, 2.2, 'Preserve')  # doctest: +ELLIPSIS
    -0.1...
    >>> gamma_function(-0.18, 2.2, 'Clamp')  # doctest: +ELLIPSIS
    0.0
    """

    a = np.asarray(a)
    exponent = np.asarray(exponent)

    negative_number_handling = negative_number_handling.lower()
    if negative_number_handling == 'indeterminate':
        return as_numeric(a ** exponent)
    elif negative_number_handling == 'mirror':
        return as_numeric(np.sign(a) * np.abs(a) ** exponent)
    elif negative_number_handling == 'preserve':
        return as_numeric(np.where(a < 0, a, a ** exponent))
    elif negative_number_handling == 'clamp':
        return as_numeric(np.where(a < 0, 0, a ** exponent))
    else:
        raise ValueError(
            'Undefined negative number handling method: "{0}".'.format(
                negative_number_handling))
<|endoftext|>


<commit_before>from models import User
import factory
import random

class UserFactory(factory.Factory):
    FACTORY_FOR = User

    graph_id = factory.Sequence(lambda n: n)
    gender = random.choice(['male','female'])<commit_msg>updated factory<commit_after>from models import User
import factory
import random

class UserFactory(factory.DjangoModelFactory):
    FACTORY_FOR = User

    graph_id = factory.Sequence(lambda n: n)
    gender = random.choice(['male','female'])
<|endoftext|>