<commit_before>r"""
Kontsevich graph sums

"""
from sage.kontsevich_graph_series.kontsevich_graph import KontsevichGraph
from sage.structure.element import ModuleElement
from sage.modules.module import Module
from sage.symbolic.ring import is_SymbolicExpressionRing

class KontsevichGraphSum(ModuleElement):
    def __init__(self, parent, terms):
        """
        Kontsevich graph sum.

        A formal sum of Kontsevich graphs, modulo the relation that
        swapping two edges (L and R) originating from an internal vertex
        introduces a minus sign.

        INPUT:

        - ``parent`` -- a ``KontsevichGraphSums`` module.
        - ``terms`` -- list of ``(coefficient, graph)`` tuples,
          where ``coefficient`` is in ``parent.base_ring()``,
          and ``graph`` is an immutable KontsevichGraph.

        OUTPUT:

        A formal sum of Kontsevich graphs.

        EXAMPLES::

            sage: K = KontsevichGraphSums(QQ)
            sage: KG = KontsevichGraph(ground_vertices=(), immutable=True)
            sage: KontsevichGraphSum(K, [(1/2, KG)])
            1/2*(Kontsevich graph with 0 vertices on 0 ground vertices)
        """
        if terms == 0:
            terms = []
        if not isinstance(terms, list):
            raise TypeError('Input must be a list of terms.')
        if not all(isinstance(t, tuple) and len(t) == 2 for t in terms):
            raise TypeError('Terms must be (coefficient, graph) tuples.')
        if not all(c in parent.base_ring() and isinstance(g, KontsevichGraph)
                   and getattr(g, '_immutable', False) for (c,g) in terms):
            raise TypeError('Coefficients must be in base ring, and ' +
                            'graphs must be immutable KontsevichGraphs.')
        self._terms = terms
        ModuleElement.__init__(self, parent=parent)

    def _rmul_(self, c):
        """
        Return the product of ``c`` and ``self``.

        INPUT:

        - ``c`` -- multiplicand.

        OUTPUT:

        Kontsevich graph sum with coefficients ``c`` times those of ``self``.

        EXAMPLES::

            sage: K = KontsevichGraphSums(QQ)
            sage: KG = KontsevichGraph(ground_vertices=(), immutable=True)
            sage: 2*KontsevichGraphSum(K, [(1/2, KG)])
            1*(Kontsevich graph with 0 vertices on 0 ground vertices)
        """
        return self.parent()([(c*d,g) for (d,g) in self._terms])

    def _add_(self, other):
        """
        Return the sum of ``self`` and ``other``.

        INPUT:

        - ``other`` -- a KontsevichGraphSum.

        OUTPUT:

        The sum of ``self`` and ``other``.

        EXAMPLES::

            sage: K = KontsevichGraphSums(QQ)
            sage: KG = KontsevichGraph(ground_vertices=(), immutable=True)
            sage: KontsevichGraphSum(K, [(1/2, KG)]) + \
                  KontsevichGraphSum(K, [(1/2, KG)])
            1*(Kontsevich graph with 0 vertices on 0 ground vertices)
        """
        return self.parent()(self._terms + other._terms)

    def __eq__(self, other):
        """
        Compare ``self`` and ``other`` for equality.

        Checks if the reduced difference is zero.

        INPUT:

        - ``other`` -- a KontsevichGraphSum with the same parent.

        EXAMPLES::

            sage: K = KontsevichGraphSums(QQ)
            sage: KG = KontsevichGraph(ground_vertices=(), immutable=True)
            sage: A = KontsevichGraphSum(K, [(1/2, KG)]) + \
                  KontsevichGraphSum(K, [(1/2, KG)])
            sage: B = KontsevichGraphSum(K, [(1, KG)])
            sage: A == B
            True
        """
        difference = self - other
        difference.reduce()
        return difference._terms == []

    def __ne__(self, other):
        """
        Unequality testing by using :meth:`.__eq__`.
        """
        return not self.__eq__(other)

    def __hash__(self):
        """
        Return the hash value.
        """
        return hash(tuple(self._terms))

    def reduce(self):
        """
        Reduce the sum by collecting terms with proportional graphs.

        Swapping the edge labels (L and R) of two edges originating from an
        internal vertex introduces a minus sign. Hence graphs that differ only
        in their edge labelings are related by a sign, equal to
        (-1)^(the number of edge swaps).

        ALGORITHM:

        First collect terms with exactly the same graph. Call this the list
        of old terms. For the first old term, find all the signed edge
        relabelings of the graph in the old terms, and add up the signed
        coefficients (+ or - according to the number of edge swaps).
        If the result is nonzero, this gives a new term. In any case, remove
        all the old terms involving these relabelings. Rinse, repeat.
        """
        # Collect terms with exactly the same graph.
        all_graphs = [g for (c,g) in self._terms]
        graphs = list(set(all_graphs))
        graphs.sort(key=all_graphs.index) # preserve ordering of terms
        coefficient = lambda g: sum(c for (c,h) in self._terms if h == g)
        self._terms = [(coefficient(g), g) for g in graphs
                        if coefficient(g) != self.base_ring()(0)]

        # Remove terms with graphs which are zero.
        self._terms = filter(lambda (c,g): not g.is_zero(), self._terms)
        # This is necessary for the next step, where we assume that all
        # edge relabelings are distinct.

        # Collect terms with the same graph up to sign.
        old_terms = list(self._terms)
        self._terms = []
        while len(old_terms) > 0:
            _, g = old_terms[0]
            graphs = [h for (d,h) in old_terms]
            signed_relabelings = filter(lambda (h, s): h in graphs, \
                                        g.edge_relabelings(signs=True))
            coefficient = lambda g: sum([d for (d,h) in old_terms \
                                         if h == g])
            total = sum(s*coefficient(h) for (h,s) in signed_relabelings)
            if total != self.base_ring()(0):
                self._terms.append((total, g))
            for (h,s) in signed_relabelings:
                old_terms.remove((coefficient(h), h))

    def _repr_(self):
        """
        EXAMPLES::

            sage: K = KontsevichGraphSums(QQ)
            sage: KG = KontsevichGraph(ground_vertices=(), immutable=True)
            sage: KontsevichGraphSum(K, [(1/2, KG)])
            1/2*(Kontsevich graph with 0 vertices on 0 ground vertices)
        """
        self.reduce()
        if self._terms == []:
            return '0'
        parenthesize = lambda c: str(c)
        if is_SymbolicExpressionRing(self.base_ring()):
            from sage.symbolic.operators import add_vararg
            is_sum = lambda x: self.base_ring()(x).operator() == add_vararg
            parenthesize = lambda c: '(%s)' % c if is_sum(c) else c
        return ' + '.join('%s*(%s)' % (parenthesize(c), g)
                          for (c,g) in self._terms)

    def subs(self, *args):
        """
        Substitute the KontsevichGraphSums ``args`` in place of the ground
        vertices, by multilinearity and the Leibniz rule.

        Here, the Leibniz rule means summing over all possible attachments.
        """
        assert all(s in self.parent() for s in args)
        assert all(len(g.ground_vertices()) == len(args) \
                   for (c,g) in self._terms)
        total_terms = []
        for (c,g) in self._terms:
            from itertools import product
            for terms in product(*[s._terms for s in args]):
                coeffs, graphs = zip(*terms)
                import operator
                coeff = c*reduce(operator.mul, coeffs)
                # Attach in all the possible ways.
                for attachment_points in product(*[h.vertices() \
                                                   for h in graphs]):
                    graph_attachments = zip(graphs, attachment_points)
                    total_terms.append((coeff, g.attach(*graph_attachments)))
        return self.parent()(total_terms)

class KontsevichGraphSums(Module):
    """
    The module of Kontsevich graph sums with coefficients in some ring.
    """
    Element = KontsevichGraphSum
    def _repr_(self):
        """
        EXAMPLES::

            sage: KontsevichGraphSums(QQ)
            Module of Kontsevich graph sums over Rational Field
            sage: KontsevichGraphSums(SR)
            Module of Kontsevich graph sums over Symbolic Ring
        """
        return 'Module of Kontsevich graph sums over %s' % self.base_ring()

    def _element_constructor_(self, terms):
        """
        Make a KontsevichGraphSum in ``self`` from ``terms``.

        INPUT:

        - ``terms`` -- KontsevichGraphSum, list of terms, or 0.
        """
        if isinstance(terms, KontsevichGraphSum): terms = terms._terms
        return self.element_class(self, terms)

    def __cmp__(self, other):
        """
        Compare ``self`` and ``other`` for equality.
        """
        if not isinstance(other, KontsevichGraphSums):
            return cmp(type(other), KontsevichGraphSums)
        return cmp(self.base_ring(), other.base_ring())

    def _an_element_(self):
        """
        EXAMPLES::

            sage: KontsevichGraphSums(QQ).an_element()
            1/2*(Kontsevich graph with 1 vertices on 2 ground vertices)
            sage: KontsevichGraphSums(SR).an_element()
            some_variable*(Kontsevich graph with 1 vertices on 2
            ground vertices)

        """
        KG = KontsevichGraph({'F' : {}, 'G' : {}, 1 : {'F' : 'L', 'G' : 'R'}},
                             ground_vertices=('F', 'G'),
                             immutable=True)
        return self.element_class(self, [(self.base_ring().an_element(), KG)])
<commit_msg>Don't apply Leibniz rule to operator of differential order 0.<commit_after>r"""
Kontsevich graph sums

"""
from sage.kontsevich_graph_series.kontsevich_graph import KontsevichGraph
from sage.structure.element import ModuleElement
from sage.modules.module import Module
from sage.symbolic.ring import is_SymbolicExpressionRing

class KontsevichGraphSum(ModuleElement):
    def __init__(self, parent, terms):
        """
        Kontsevich graph sum.

        A formal sum of Kontsevich graphs, modulo the relation that
        swapping two edges (L and R) originating from an internal vertex
        introduces a minus sign.

        INPUT:

        - ``parent`` -- a ``KontsevichGraphSums`` module.
        - ``terms`` -- list of ``(coefficient, graph)`` tuples,
          where ``coefficient`` is in ``parent.base_ring()``,
          and ``graph`` is an immutable KontsevichGraph.

        OUTPUT:

        A formal sum of Kontsevich graphs.

        EXAMPLES::

            sage: K = KontsevichGraphSums(QQ)
            sage: KG = KontsevichGraph(ground_vertices=(), immutable=True)
            sage: KontsevichGraphSum(K, [(1/2, KG)])
            1/2*(Kontsevich graph with 0 vertices on 0 ground vertices)
        """
        if terms == 0:
            terms = []
        if not isinstance(terms, list):
            raise TypeError('Input must be a list of terms.')
        if not all(isinstance(t, tuple) and len(t) == 2 for t in terms):
            raise TypeError('Terms must be (coefficient, graph) tuples.')
        if not all(c in parent.base_ring() and isinstance(g, KontsevichGraph)
                   and getattr(g, '_immutable', False) for (c,g) in terms):
            raise TypeError('Coefficients must be in base ring, and ' +
                            'graphs must be immutable KontsevichGraphs.')
        self._terms = terms
        ModuleElement.__init__(self, parent=parent)

    def _rmul_(self, c):
        """
        Return the product of ``c`` and ``self``.

        INPUT:

        - ``c`` -- multiplicand.

        OUTPUT:

        Kontsevich graph sum with coefficients ``c`` times those of ``self``.

        EXAMPLES::

            sage: K = KontsevichGraphSums(QQ)
            sage: KG = KontsevichGraph(ground_vertices=(), immutable=True)
            sage: 2*KontsevichGraphSum(K, [(1/2, KG)])
            1*(Kontsevich graph with 0 vertices on 0 ground vertices)
        """
        return self.parent()([(c*d,g) for (d,g) in self._terms])

    def _add_(self, other):
        """
        Return the sum of ``self`` and ``other``.

        INPUT:

        - ``other`` -- a KontsevichGraphSum.

        OUTPUT:

        The sum of ``self`` and ``other``.

        EXAMPLES::

            sage: K = KontsevichGraphSums(QQ)
            sage: KG = KontsevichGraph(ground_vertices=(), immutable=True)
            sage: KontsevichGraphSum(K, [(1/2, KG)]) + \
                  KontsevichGraphSum(K, [(1/2, KG)])
            1*(Kontsevich graph with 0 vertices on 0 ground vertices)
        """
        return self.parent()(self._terms + other._terms)

    def __eq__(self, other):
        """
        Compare ``self`` and ``other`` for equality.

        Checks if the reduced difference is zero.

        INPUT:

        - ``other`` -- a KontsevichGraphSum with the same parent.

        EXAMPLES::

            sage: K = KontsevichGraphSums(QQ)
            sage: KG = KontsevichGraph(ground_vertices=(), immutable=True)
            sage: A = KontsevichGraphSum(K, [(1/2, KG)]) + \
                  KontsevichGraphSum(K, [(1/2, KG)])
            sage: B = KontsevichGraphSum(K, [(1, KG)])
            sage: A == B
            True
        """
        difference = self - other
        difference.reduce()
        return difference._terms == []

    def __ne__(self, other):
        """
        Unequality testing by using :meth:`.__eq__`.
        """
        return not self.__eq__(other)

    def __hash__(self):
        """
        Return the hash value.
        """
        return hash(tuple(self._terms))

    def reduce(self):
        """
        Reduce the sum by collecting terms with proportional graphs.

        Swapping the edge labels (L and R) of two edges originating from an
        internal vertex introduces a minus sign. Hence graphs that differ only
        in their edge labelings are related by a sign, equal to
        (-1)^(the number of edge swaps).

        ALGORITHM:

        First collect terms with exactly the same graph. Call this the list
        of old terms. For the first old term, find all the signed edge
        relabelings of the graph in the old terms, and add up the signed
        coefficients (+ or - according to the number of edge swaps).
        If the result is nonzero, this gives a new term. In any case, remove
        all the old terms involving these relabelings. Rinse, repeat.
        """
        # Collect terms with exactly the same graph.
        all_graphs = [g for (c,g) in self._terms]
        graphs = list(set(all_graphs))
        graphs.sort(key=all_graphs.index) # preserve ordering of terms
        coefficient = lambda g: sum(c for (c,h) in self._terms if h == g)
        self._terms = [(coefficient(g), g) for g in graphs
                        if coefficient(g) != self.base_ring()(0)]

        # Remove terms with graphs which are zero.
        self._terms = filter(lambda (c,g): not g.is_zero(), self._terms)
        # This is necessary for the next step, where we assume that all
        # edge relabelings are distinct.

        # Collect terms with the same graph up to sign.
        old_terms = list(self._terms)
        self._terms = []
        while len(old_terms) > 0:
            _, g = old_terms[0]
            graphs = [h for (d,h) in old_terms]
            signed_relabelings = filter(lambda (h, s): h in graphs, \
                                        g.edge_relabelings(signs=True))
            coefficient = lambda g: sum([d for (d,h) in old_terms \
                                         if h == g])
            total = sum(s*coefficient(h) for (h,s) in signed_relabelings)
            if total != self.base_ring()(0):
                self._terms.append((total, g))
            for (h,s) in signed_relabelings:
                old_terms.remove((coefficient(h), h))

    def _repr_(self):
        """
        EXAMPLES::

            sage: K = KontsevichGraphSums(QQ)
            sage: KG = KontsevichGraph(ground_vertices=(), immutable=True)
            sage: KontsevichGraphSum(K, [(1/2, KG)])
            1/2*(Kontsevich graph with 0 vertices on 0 ground vertices)
        """
        self.reduce()
        if self._terms == []:
            return '0'
        parenthesize = lambda c: str(c)
        if is_SymbolicExpressionRing(self.base_ring()):
            from sage.symbolic.operators import add_vararg
            is_sum = lambda x: self.base_ring()(x).operator() == add_vararg
            parenthesize = lambda c: '(%s)' % c if is_sum(c) else c
        return ' + '.join('%s*(%s)' % (parenthesize(c), g)
                          for (c,g) in self._terms)

    def subs(self, *args):
        """
        Substitute the KontsevichGraphSums ``args`` in place of the ground
        vertices, by multilinearity and the Leibniz rule.

        Here, the Leibniz rule means summing over all possible attachments.
        """
        assert all(s in self.parent() for s in args)
        assert all(len(g.ground_vertices()) == len(args) \
                   for (c,g) in self._terms)
        total_terms = []
        for (c,g) in self._terms:
            from itertools import product
            for terms in product(*[s._terms for s in args]):
                coeffs, graphs = zip(*terms)
                import operator
                coeff = c*reduce(operator.mul, coeffs)
                # Attach in all the possible ways.
                for attachment_points in product(*[graphs[n].vertices() \
                       if g.in_degree(g.ground_vertices()[n]) > 0 \
                       else [graphs[n].random_vertex()] \
                       for n in range(0, len(args))]):
                    graph_attachments = zip(graphs, attachment_points)
                    total_terms.append((coeff, g.attach(*graph_attachments)))
        return self.parent()(total_terms)

class KontsevichGraphSums(Module):
    """
    The module of Kontsevich graph sums with coefficients in some ring.
    """
    Element = KontsevichGraphSum
    def _repr_(self):
        """
        EXAMPLES::

            sage: KontsevichGraphSums(QQ)
            Module of Kontsevich graph sums over Rational Field
            sage: KontsevichGraphSums(SR)
            Module of Kontsevich graph sums over Symbolic Ring
        """
        return 'Module of Kontsevich graph sums over %s' % self.base_ring()

    def _element_constructor_(self, terms):
        """
        Make a KontsevichGraphSum in ``self`` from ``terms``.

        INPUT:

        - ``terms`` -- KontsevichGraphSum, list of terms, or 0.
        """
        if isinstance(terms, KontsevichGraphSum): terms = terms._terms
        return self.element_class(self, terms)

    def __cmp__(self, other):
        """
        Compare ``self`` and ``other`` for equality.
        """
        if not isinstance(other, KontsevichGraphSums):
            return cmp(type(other), KontsevichGraphSums)
        return cmp(self.base_ring(), other.base_ring())

    def _an_element_(self):
        """
        EXAMPLES::

            sage: KontsevichGraphSums(QQ).an_element()
            1/2*(Kontsevich graph with 1 vertices on 2 ground vertices)
            sage: KontsevichGraphSums(SR).an_element()
            some_variable*(Kontsevich graph with 1 vertices on 2
            ground vertices)

        """
        KG = KontsevichGraph({'F' : {}, 'G' : {}, 1 : {'F' : 'L', 'G' : 'R'}},
                             ground_vertices=('F', 'G'),
                             immutable=True)
        return self.element_class(self, [(self.base_ring().an_element(), KG)])
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import random
import string
from typing import List
from checklist.editor import Editor
from interfaces.SentenceOperation import SentenceOperation
from evaluation.evaluation_engine import evaluate, execute_model
from tasks.TaskTypes import TaskType
import json

with open("data.json", "r", encoding="utf-8") as f:
    names = json.load(f)

with open("noun_pairs.json", "r", encoding="utf-8") as fn:
    nouns = json.load(fn)

f = list(names["1"].values())
m = list(names["2"].values())

# Noun Pairs
def replace_punc(text):
    for i in string.punctuation:
        text = text.replace(i, " " + i)
    return text

def restore_punc(text):
    for i in string.punctuation:
        text = text.replace(" " + i, i)
    return text

preceeding_word = {
    "der": "die",
    "Der": "Die",
    "ein": "eine",
    "Ein": "Eine",
    "mein": "meine",
    "Mein": "Meine",
    "dein": "deine",
    "Dein": "Deine",
    "sein": "seine",
    "Sein": "Seine",
    "unser": "unsere",
    "Unser": "Unsere",
    "euer": "eure",
    "Euer": "Eure",
    "euren": "eure",
    "Euren": "Eure",
    "eurem": "eurer",
    "Eurem": "Eurer",
    "ihr": "ihre",
    "Ihr": "Ihre",
    "den": "die",
    "Den": "Die",
    "einen": "eine",
    "Einen": "Eine",
    "dieser": "diese",
    "Dieser": "Diese",
    "diesen": "diese",
    "Diesen": "Diese",
    "meinen": "meine",
    "Meinen": "Meine",
    "deinen": "deine",
    "Deinen": "Deine",
}

def replace_prev_word(ind, text, noundict):
    t2 = text
    for i in ind:
        prev_ind = i - 1
        if t2[prev_ind] in preceeding_word.keys():
            t2[prev_ind] = preceeding_word[t2[prev_ind]]
        else:
            t2 = text
    return t2

def replace_noun_pairs(inp, nouns):
    i = replace_punc(inp)
    text = i.split()
    for name in nouns.keys():
        if name in text:
            ind = get_index(text, name)
            newtext = replace_name_in_list(ind, text, nouns)
            newtext = replace_prev_word(ind, newtext, preceeding_word)

        else:
            newtext = text
    newtext = " ".join(str(x) for x in newtext)
    return newtext

# Personal Pronouns
personalp = {
    "er": "sie",
    "Er": "Sie",
    "ihr": "sie",
    "Ihr": "Sie",
    "ihn": "sie",
    "Ihn": "Sie",
    "Ihm": "Ihr",
    "ihm": "ihr",
}

def get_index(wl, n):
    indices = [i for i, x in enumerate(wl) if x == n]
    return indices

def replace_name_in_list(ind, text, noundict):
    t2 = text
    for i in ind:
        t2[i] = noundict[t2[i]]
    return t2

def replace_personal(inp, personalp):
    i = replace_punc(inp)
    text = i.split()
    for name in personalp.keys():
        if name in text:
            ind = get_index(text, name)
            newtext = replace_name_in_list(ind, text, personalp)
        else:
            newtext = text
    newtext = " ".join(str(x) for x in newtext)
    return newtext

def findname(sent, malenames, femalenames):
    t = sent.split()
    for word in t:
        if word in malenames:
            w = word
            return w
        elif word in femalenames:
            w = word
            return w
        else:
            print(" ")

def swapname(name, names):
    for n in names["2"].values():
        if name in m:
            new = random.choice([i for i in names["1"].values()])
            return new
        else:
            return name

def newnamerep(m, f, inp, names):
    n = findname(inp, m, f)
    sent = inp
    t = sent.split()
    for word in t:
        if word in m and n != "None":
            if n != None:
                fname = swapname(word, names)
                sent = sent.replace(word, str(fname))
            else:
                sent = sent
    return sent

def german_nouns(inp, nouns, names):
    text = replace_punc(inp)
    t1 = replace_noun_pairs(text, nouns)
    t2 = replace_personal(t1, personalp)
    t3 = newnamerep(m, f, t2, names)
    t4 = restore_punc(t3)
    return t4

class GermanGenderSwap(SentenceOperation):

    tasks = [TaskType.TEXT_CLASSIFICATION, TaskType.TEXT_TO_TEXT_GENERATION]
    languages = ["de"]
    keywords = [
        "lexical",
        "rule-based",
        "high-coverage",
    ]

    def __init__(self, seed=0, max_outputs=1):
        super().__init__(seed, max_outputs)
        self.nouns = nouns
        self.names = names

    def generate(self, sentence: str):
        return [german_nouns(sentence, self.nouns, self.names)]

if __name__ == "__main__":
    import json
    from TestRunner import convert_to_snake_case

    tf = GermanGenderSwap(max_outputs=1)
    test_cases = []
    for sentence in [
        "Der liebe Gott hat uns noch ein paar wunderschöne Tage mit ihr geschenkt, in denen sich der größte Teil ihrer Familie und ihrer Freunde bei ihr verabschieden konnte, sagte Thurn und Taxis auch im Namen ihrer Brüder Alexander und Carl-Alban.",
        "Mein Vater und dein Bruder sind hier?",
        "Er ist der Präsident, sie ist die Herrscherin und sie werden jetzt bald Ehemann und Ehefrau sein!",
        "Er ist ein Arzt und mein Vater.",
        "Ich sehe, dass der Dichter und die Schauspielerin jetzt Freunde sind!",
    ]:
        test_cases.append(
            {
                "class": tf.name(),
                "inputs": {"sentence": sentence},
                "outputs": [{"sentence": o} for o in tf.generate(sentence)],
            }
        )
    json_file = {
        "type": convert_to_snake_case(tf.name()),
        "test_cases": test_cases,
    }
    print(json.dumps(json_file, indent=2))
<commit_msg>Update transformation.py<commit_after># -*- coding: utf-8 -*-
import random
import string
from typing import List
from checklist.editor import Editor
from interfaces.SentenceOperation import SentenceOperation
from evaluation.evaluation_engine import evaluate, execute_model
from tasks.TaskTypes import TaskType
import json

with open("data.json", "r", encoding="utf-8") as f:
    names = json.load(f)

with open("noun_pairs.json", "r", encoding="utf-8") as fn:
    nouns = json.load(fn)

f = list(names["1"].values())
m = list(names["2"].values())

# Noun Pairs.
def replace_punc(text):
    for i in string.punctuation:
        text = text.replace(i, " " + i)
    return text

def restore_punc(text):
    for i in string.punctuation:
        text = text.replace(" " + i, i)
    return text

preceeding_word = {
    "der": "die",
    "Der": "Die",
    "ein": "eine",
    "Ein": "Eine",
    "mein": "meine",
    "Mein": "Meine",
    "dein": "deine",
    "Dein": "Deine",
    "sein": "seine",
    "Sein": "Seine",
    "unser": "unsere",
    "Unser": "Unsere",
    "euer": "eure",
    "Euer": "Eure",
    "euren": "eure",
    "Euren": "Eure",
    "eurem": "eurer",
    "Eurem": "Eurer",
    "ihr": "ihre",
    "Ihr": "Ihre",
    "den": "die",
    "Den": "Die",
    "einen": "eine",
    "Einen": "Eine",
    "dieser": "diese",
    "Dieser": "Diese",
    "diesen": "diese",
    "Diesen": "Diese",
    "meinen": "meine",
    "Meinen": "Meine",
    "deinen": "deine",
    "Deinen": "Deine",
}

def replace_prev_word(ind, text, noundict):
    t2 = text
    for i in ind:
        prev_ind = i - 1
        if t2[prev_ind] in preceeding_word.keys():
            t2[prev_ind] = preceeding_word[t2[prev_ind]]
        else:
            t2 = text
    return t2

def replace_noun_pairs(inp, nouns):
    i = replace_punc(inp)
    text = i.split()
    for name in nouns.keys():
        if name in text:
            ind = get_index(text, name)
            newtext = replace_name_in_list(ind, text, nouns)
            newtext = replace_prev_word(ind, newtext, preceeding_word)

        else:
            newtext = text
    newtext = " ".join(str(x) for x in newtext)
    return newtext

# Personal Pronouns.
personalp = {
    "er": "sie",
    "Er": "Sie",
    "ihr": "sie",
    "Ihr": "Sie",
    "ihn": "sie",
    "Ihn": "Sie",
    "Ihm": "Ihr",
    "ihm": "ihr",
}

def get_index(wl, n):
    indices = [i for i, x in enumerate(wl) if x == n]
    return indices

def replace_name_in_list(ind, text, noundict):
    t2 = text
    for i in ind:
        t2[i] = noundict[t2[i]]
    return t2

def replace_personal(inp, personalp):
    i = replace_punc(inp)
    text = i.split()
    for name in personalp.keys():
        if name in text:
            ind = get_index(text, name)
            newtext = replace_name_in_list(ind, text, personalp)
        else:
            newtext = text
    newtext = " ".join(str(x) for x in newtext)
    return newtext

def findname(sent, malenames, femalenames):
    t = sent.split()
    for word in t:
        if word in malenames:
            w = word
            return w
        elif word in femalenames:
            w = word
            return w
        else:
            print(" ")

def swapname(name, names):
    for n in names["2"].values():
        if name in m:
            new = random.choice([i for i in names["1"].values()])
            return new
        else:
            return name

def newnamerep(m, f, inp, names):
    n = findname(inp, m, f)
    sent = inp
    t = sent.split()
    for word in t:
        if word in m and n != "None":
            if n != None:
                fname = swapname(word, names)
                sent = sent.replace(word, str(fname))
            else:
                sent = sent
    return sent

def german_nouns(inp, nouns, names):
    text = replace_punc(inp)
    t1 = replace_noun_pairs(text, nouns)
    t2 = replace_personal(t1, personalp)
    t3 = newnamerep(m, f, t2, names)
    t4 = restore_punc(t3)
    return t4

class GermanGenderSwap(SentenceOperation):

    tasks = [TaskType.TEXT_CLASSIFICATION, TaskType.TEXT_TO_TEXT_GENERATION]
    languages = ["de"]
    keywords = [
        "lexical",
        "rule-based",
        "high-coverage",
    ]

    def __init__(self, seed=0, max_outputs=1):
        super().__init__(seed, max_outputs)
        self.nouns = nouns
        self.names = names

    def generate(self, sentence: str):
        return [german_nouns(sentence, self.nouns, self.names)]
<|endoftext|>


<commit_before># A post is considered to be extracted from an existing file, and always
# consists in a title, followed by a line of '------------', followed
# immediatly by the tags with a leading #. It is a list of strings, where empty
# ones were removed.
# If it is a valid post, it has at minima 4 lines (title, line of -, tags,
# date)
# They should be extracted recursively, and fed to the different routines
# `_from_post`.
# Note that all doctests are commented, and now executed in the py.test suite,
# for compatibility reasons between py2.7 and py3.3
from __future__ import unicode_literals
from datetime import date
from collections import Counter
from collections import OrderedDict as od
import re
import io

def is_valid_post(post):
    """
    Check that it has all the required arguments

    If the post is valid, the function returns True. Otherwise, an
    MarkdownSyntaxError is raised with a description of the problem.

    """
    # remove all blank lines
    temp = [e for e in post if e]
    if len(temp) < 4:
        raise MarkdownSyntaxError("Post contains under four lines", post)
    else:
        # Recover the index of the line of dashes, in case of long titles
        index = 0
        dashes = [e for e in temp if re.match('^-{2,}$', e)]
        try:
            index = temp.index(dashes[0])
        except IndexError:
            raise MarkdownSyntaxError("Post does not contain dashes", post)
        if index:
            if not temp[0]:
                raise MarkdownSyntaxError("Post title is empty", post)
            if not re.match('^#.*$', temp[index+1]):
                raise MarkdownSyntaxError(
                    "Tags were not found after the dashes", post)
            match = re.match(
                r"^\*[0-9]{2}/[0-1][0-9]/[0-9]{4}\*$",
                temp[index+2])
            if not match:
                raise MarkdownSyntaxError(
                    "The date could not be read", temp)
    return True

def extract_tags_from_post(post):
    """
    Recover the tags from an extracted post

    .. note::

        No tests are being done to ensure that the third line exists, as only
        valid posts, determined with :func:`is_valid_post` are sent to this
        routine.

    """
    tag_line = post[2].strip()
    if tag_line and tag_line[0] == '#':
        tags = [elem.strip().lower() for elem in tag_line[1:].split(',')]

    if any(tags):
        return tags, post[:2]+post[3:]
    else:
        raise MarkdownSyntaxError("No tags specified in the post", post)

def extract_title_from_post(post):
    """
    Recover the title from an extracted post

    """
    return post[0]

def extract_date_from_post(post):
    """
    Recover the date from an extracted post, and return the correct post

    """
    match = re.match(
        r"\*([0-9]{2})/([0-1][0-9])/([0-9]{4})\*",
        post[2])
    if match:
        assert len(match.groups()) == 3
        day, month, year = match.groups()
        extracted_date = date(int(year), int(month), int(day))
        return extracted_date, post[:2]+post[3:]
    else:
        raise MarkdownSyntaxError("No date found in the post", post)

def normalize_post(post):
    """
    Perform normalization of the input post

    - If a title has several lines, merge them
    - If there are missing/added blank lines in the headers, remove them
    """
    # Remove trailing \n
    post = [line.rstrip('\n') for line in post]

    # Recover the dashline (title of the post)
    dashes = [e for e in post if re.match('^-{2,}$', e)]
    dashline_index = post.index(dashes[0])

    title = ' '.join([post[index] for index in range(dashline_index)])
    normalized_post = [title]+[post[dashline_index]]

    # Recover the tag line
    tags = [e for e in post if re.match('^\#(.*)$', e)]
    tag_line_index = post.index(tags[0])
    normalized_post.append(post[tag_line_index])

    # Recover the date
    dates = [e for e in post if re.match(
        r"\*([0-9]{2})/([0-1][0-9])/([0-9]{4})\*", e)]
    date_line_index = post.index(dates[0])
    normalized_post.append(post[date_line_index])

    # Append the rest, starting from the first non-empty line
    non_empty = [e for e in post[date_line_index+1:] if e]
    if non_empty:
        non_empty_index = post.index(non_empty[0])
        normalized_post.extend(post[non_empty_index:])

    return normalized_post

def extract_corpus_from_post(post):
    """
    Recover the whole content of a post

    """
    return post[2:]

def extract_title_and_posts_from_text(text):
    """
    From an entire text (array), recover each posts and the file's title

    """
    # Make a first pass to recover the title (first line that is underlined
    # with = signs) and the indices of the dash, that signals a new post.
    post_starting_indices = []
    has_title = False
    for index, line in enumerate(text):
        # Remove white lines at the beginning
        if not line.strip():
            continue
        if re.match('^={2,}$', line) and not has_title:
            title = ' '.join(text[:index]).strip()
            has_title = True
        if re.match('^-{2,}$', line) and line[0] == '-':
            # find the latest non empty line
            for backward_index in range(1, 10):
                if not text[index-backward_index].strip():
                    post_starting_indices.append(index-backward_index+1)
                    break

    if not has_title:
        raise MarkdownSyntaxError(
            "You should specify a title to your file"
            ", underlined with = signs", [])
    number_of_posts = len(post_starting_indices)

    # Create post_indices such that it stores all the post, so the starting and
    # ending index of each post
    post_indices = []
    for index, elem in enumerate(post_starting_indices):
        if index < number_of_posts - 1:
            post_indices.append([elem, post_starting_indices[index+1]])
        else:
            post_indices.append([elem, len(text)])

    posts = []
    for elem in post_indices:
        start, end = elem
        posts.append(text[start:end])

    # Normalize them all
    for index, post in enumerate(posts):
        posts[index] = normalize_post(post)
        assert is_valid_post(posts[index]) is True

    return title, posts

def post_to_markdown(post):
    """
    Write the markdown for a given post

    post : list
        lines constituting the post

    """
    title = post[0]
    text = ["", "
",
            "## %s {.blog-post-title}" % title, ""]

    tags, post = extract_tags_from_post(post)
    edit_date, post = extract_date_from_post(post)
    text.extend(["
",
                 "%s" % edit_date, "
"])
    corpus = extract_corpus_from_post(post)
    text.extend(corpus)
    text.extend(["
", "", ""])

    return text, tags

def from_notes_to_markdown(path, input_tags=None):
    """
    From a file, given tags, produce an output markdown file.

    This will then be interpreted with the pandoc library into html.

    ..note::

        so far the date is ignored.

    Returns
    -------
    markdown : list
        entire markdown text
    tags : list of tuples
        list of tags extracted from the text, with their importance
    """
    # Create the array to return
    text = io.open(path, 'r', encoding='utf-8', errors='replace').readlines()
    title, posts = extract_title_and_posts_from_text(text)
    markdown = ["
",
                "# %s {.blog-title}" % title, "
", "",
                "
", "
"]
    extracted_tags = []
    for post in posts:
        text, tags = post_to_markdown(post)
        if all([tag in tags for tag in input_tags]):
            # Store the recovered tags
            extracted_tags.extend(tags)
            markdown.extend(text)

    markdown.extend(["
", "
"])
    cleaned_tags = sort_tags(extracted_tags)
    return markdown, cleaned_tags

def sort_tags(source):
    """
    return a sorted version of source, with the biggests tags first
    """
    output = od(sorted(Counter([e for e in source]).items(),
                key=lambda t: -t[1]))
    return output

def create_post_from_entry(title, tags, corpus):
    """
    Create a string containing the post given user's input

    """
    text = [title]
    text.append('\n%s\n' % ''.join(['-' for _ in range(len(title))]))
    text.append('# %s\n' % ', '.join(tags))
    text.append('\n*%s*\n\n' % date.today().strftime("%d/%m/%Y"))
    text.append(corpus+'\n')
    return ''.join(text)

class MarkdownSyntaxError(ValueError):

    def __init__(self, message, post):
        ValueError.__init__(self, message+':\n\n' +
                            '\n'.join(['    %s' % e for e in post]))
<commit_msg>Display tags next to the date<commit_after># A post is considered to be extracted from an existing file, and always
# consists in a title, followed by a line of '------------', followed
# immediatly by the tags with a leading #. It is a list of strings, where empty
# ones were removed.
# If it is a valid post, it has at minima 4 lines (title, line of -, tags,
# date)
# They should be extracted recursively, and fed to the different routines
# `_from_post`.
# Note that all doctests are commented, and now executed in the py.test suite,
# for compatibility reasons between py2.7 and py3.3
from __future__ import unicode_literals
from datetime import date
from collections import Counter
from collections import OrderedDict as od
import re
import io

def is_valid_post(post):
    """
    Check that it has all the required arguments

    If the post is valid, the function returns True. Otherwise, an
    MarkdownSyntaxError is raised with a description of the problem.

    """
    # remove all blank lines
    temp = [e for e in post if e]
    if len(temp) < 4:
        raise MarkdownSyntaxError("Post contains under four lines", post)
    else:
        # Recover the index of the line of dashes, in case of long titles
        index = 0
        dashes = [e for e in temp if re.match('^-{2,}$', e)]
        try:
            index = temp.index(dashes[0])
        except IndexError:
            raise MarkdownSyntaxError("Post does not contain dashes", post)
        if index:
            if not temp[0]:
                raise MarkdownSyntaxError("Post title is empty", post)
            if not re.match('^#.*$', temp[index+1]):
                raise MarkdownSyntaxError(
                    "Tags were not found after the dashes", post)
            match = re.match(
                r"^\*[0-9]{2}/[0-1][0-9]/[0-9]{4}\*$",
                temp[index+2])
            if not match:
                raise MarkdownSyntaxError(
                    "The date could not be read", temp)
    return True

def extract_tags_from_post(post):
    """
    Recover the tags from an extracted post

    .. note::

        No tests are being done to ensure that the third line exists, as only
        valid posts, determined with :func:`is_valid_post` are sent to this
        routine.

    """
    tag_line = post[2].strip()
    if tag_line and tag_line[0] == '#':
        tags = [elem.strip().lower() for elem in tag_line[1:].split(',')]

    if any(tags):
        return tags, post[:2]+post[3:]
    else:
        raise MarkdownSyntaxError("No tags specified in the post", post)

def extract_title_from_post(post):
    """
    Recover the title from an extracted post

    """
    return post[0]

def extract_date_from_post(post):
    """
    Recover the date from an extracted post, and return the correct post

    """
    match = re.match(
        r"\*([0-9]{2})/([0-1][0-9])/([0-9]{4})\*",
        post[2])
    if match:
        assert len(match.groups()) == 3
        day, month, year = match.groups()
        extracted_date = date(int(year), int(month), int(day))
        return extracted_date, post[:2]+post[3:]
    else:
        raise MarkdownSyntaxError("No date found in the post", post)

def normalize_post(post):
    """
    Perform normalization of the input post

    - If a title has several lines, merge them
    - If there are missing/added blank lines in the headers, remove them
    """
    # Remove trailing \n
    post = [line.rstrip('\n') for line in post]

    # Recover the dashline (title of the post)
    dashes = [e for e in post if re.match('^-{2,}$', e)]
    dashline_index = post.index(dashes[0])

    title = ' '.join([post[index] for index in range(dashline_index)])
    normalized_post = [title]+[post[dashline_index]]

    # Recover the tag line
    tags = [e for e in post if re.match('^\#(.*)$', e)]
    tag_line_index = post.index(tags[0])
    normalized_post.append(post[tag_line_index])

    # Recover the date
    dates = [e for e in post if re.match(
        r"\*([0-9]{2})/([0-1][0-9])/([0-9]{4})\*", e)]
    date_line_index = post.index(dates[0])
    normalized_post.append(post[date_line_index])

    # Append the rest, starting from the first non-empty line
    non_empty = [e for e in post[date_line_index+1:] if e]
    if non_empty:
        non_empty_index = post.index(non_empty[0])
        normalized_post.extend(post[non_empty_index:])

    return normalized_post

def extract_corpus_from_post(post):
    """
    Recover the whole content of a post

    """
    return post[2:]

def extract_title_and_posts_from_text(text):
    """
    From an entire text (array), recover each posts and the file's title

    """
    # Make a first pass to recover the title (first line that is underlined
    # with = signs) and the indices of the dash, that signals a new post.
    post_starting_indices = []
    has_title = False
    for index, line in enumerate(text):
        # Remove white lines at the beginning
        if not line.strip():
            continue
        if re.match('^={2,}$', line) and not has_title:
            title = ' '.join(text[:index]).strip()
            has_title = True
        if re.match('^-{2,}$', line) and line[0] == '-':
            # find the latest non empty line
            for backward_index in range(1, 10):
                if not text[index-backward_index].strip():
                    post_starting_indices.append(index-backward_index+1)
                    break

    if not has_title:
        raise MarkdownSyntaxError(
            "You should specify a title to your file"
            ", underlined with = signs", [])
    number_of_posts = len(post_starting_indices)

    # Create post_indices such that it stores all the post, so the starting and
    # ending index of each post
    post_indices = []
    for index, elem in enumerate(post_starting_indices):
        if index < number_of_posts - 1:
            post_indices.append([elem, post_starting_indices[index+1]])
        else:
            post_indices.append([elem, len(text)])

    posts = []
    for elem in post_indices:
        start, end = elem
        posts.append(text[start:end])

    # Normalize them all
    for index, post in enumerate(posts):
        posts[index] = normalize_post(post)
        assert is_valid_post(posts[index]) is True

    return title, posts

def post_to_markdown(post):
    """
    Write the markdown for a given post

    post : list
        lines constituting the post

    """
    title = post[0]
    text = ["", "
",
            "## %s {.blog-post-title}" % title, ""]

    tags, post = extract_tags_from_post(post)
    edit_date, post = extract_date_from_post(post)
    text.extend(["
",
                 "%s:" % edit_date,
                 "%s" % ", ".join(["**%s**" % tag for tag in tags]),
                 "
"])
    corpus = extract_corpus_from_post(post)
    text.extend(corpus)
    text.extend(["
", "", ""])

    return text, tags

def from_notes_to_markdown(path, input_tags=None):
    """
    From a file, given tags, produce an output markdown file.

    This will then be interpreted with the pandoc library into html.

    ..note::

        so far the date is ignored.

    Returns
    -------
    markdown : list
        entire markdown text
    tags : list of tuples
        list of tags extracted from the text, with their importance
    """
    # Create the array to return
    text = io.open(path, 'r', encoding='utf-8', errors='replace').readlines()
    title, posts = extract_title_and_posts_from_text(text)
    markdown = ["
",
                "# %s {.blog-title}" % title, "
", "",
                "
", "
"]
    extracted_tags = []
    for post in posts:
        text, tags = post_to_markdown(post)
        if all([tag in tags for tag in input_tags]):
            # Store the recovered tags
            extracted_tags.extend(tags)
            markdown.extend(text)

    markdown.extend(["
", "
"])
    cleaned_tags = sort_tags(extracted_tags)
    return markdown, cleaned_tags

def sort_tags(source):
    """
    return a sorted version of source, with the biggests tags first
    """
    output = od(sorted(Counter([e for e in source]).items(),
                key=lambda t: -t[1]))
    return output

def create_post_from_entry(title, tags, corpus):
    """
    Create a string containing the post given user's input

    """
    text = [title]
    text.append('\n%s\n' % ''.join(['-' for _ in range(len(title))]))
    text.append('# %s\n' % ', '.join(tags))
    text.append('\n*%s*\n\n' % date.today().strftime("%d/%m/%Y"))
    text.append(corpus+'\n')
    return ''.join(text)

class MarkdownSyntaxError(ValueError):

    def __init__(self, message, post):
        ValueError.__init__(self, message+':\n\n' +
                            '\n'.join(['    %s' % e for e in post]))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
Koordinates API Library

:copyright: (c) 2015 by Koordinates .
:license: BSD, see LICENSE for more details.

'''

__title__ = 'koordinates'
__version__ = '0.2.1'
__author__ = 'Richard Shea'
__license__ = 'BSD'
__copyright__ = 'Copyright 2015 Koordinates'

import logging
try:
    from logging import NullHandler
except ImportError:
    class NullHandler(logging.Handler):
        def emit(self, record):
            pass

logging.getLogger(__name__).addHandler(NullHandler())
<commit_msg>Rolling the version to 0.3.0<commit_after># -*- coding: utf-8 -*-
'''
Koordinates API Library

:copyright: (c) 2015 by Koordinates .
:license: BSD, see LICENSE for more details.

'''

__title__ = 'koordinates'
__version__ = '0.3.0'
__author__ = 'Richard Shea'
__license__ = 'BSD'
__copyright__ = 'Copyright 2015 Koordinates'

import logging
try:
    from logging import NullHandler
except ImportError:
    class NullHandler(logging.Handler):
        def emit(self, record):
            pass

logging.getLogger(__name__).addHandler(NullHandler())
<|endoftext|>


<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2011 OpenStack LLC.
# Copyright 2011 Justin Santa Barbara
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import functools
import imp
import inspect
import os
import sys

from lxml import etree
import routes
import webob.dec
import webob.exc

from nova import exception
from nova import flags
from nova import log as logging
from nova import utils
from nova import wsgi as base_wsgi
import nova.api.openstack
from nova.api.openstack import common
from nova.api.openstack import wsgi
from nova.api.openstack import xmlutil

LOG = logging.getLogger('nova.api.openstack.extensions')

FLAGS = flags.FLAGS

class ExtensionDescriptor(object):
    """Base class that defines the contract for extensions.

    Note that you don't have to derive from this class to have a valid
    extension; it is purely a convenience.

    """

    # The name of the extension, e.g., 'Fox In Socks'
    name = None

    # The alias for the extension, e.g., 'FOXNSOX'
    alias = None

    # Description comes from the docstring for the class

    # The XML namespace for the extension, e.g.,
    # 'http://www.fox.in.socks/api/ext/pie/v1.0'
    namespace = None

    # The timestamp when the extension was last updated, e.g.,
    # '2011-01-22T13:25:27-06:00'
    updated = None

    def __init__(self, ext_mgr):
        """Register extension with the extension manager."""

        ext_mgr.register(self)

    def get_resources(self):
        """List of extensions.ResourceExtension extension objects.

        Resources define new nouns, and are accessible through URLs.

        """
        resources = []
        return resources

    def get_actions(self):
        """List of extensions.ActionExtension extension objects.

        Actions are verbs callable from the API.

        """
        actions = []
        return actions

    def get_request_extensions(self):
        """List of extensions.RequestException extension objects.

        Request extensions are used to handle custom request data.

        """
        request_exts = []
        return request_exts

class ActionExtensionController(object):
    def __init__(self, application):
        self.application = application
        self.action_handlers = {}

    def add_action(self, action_name, handler):
        self.action_handlers[action_name] = handler

    def action(self, req, id, body):
        for action_name, handler in self.action_handlers.iteritems():
            if action_name in body:
                return handler(body, req, id)
        # no action handler found (bump to downstream application)
        res = self.application
        return res

class ActionExtensionResource(wsgi.Resource):

    def __init__(self, application):
        controller = ActionExtensionController(application)
        wsgi.Resource.__init__(self, controller)

    def add_action(self, action_name, handler):
        self.controller.add_action(action_name, handler)

class RequestExtensionController(object):

    def __init__(self, application):
        self.application = application
        self.handlers = []

    def add_handler(self, handler):
        self.handlers.append(handler)

    def process(self, req, *args, **kwargs):
        res = req.get_response(self.application)

        # Deserialize the response body, if any
        body = None
        if res.body:
            body = utils.loads(res.body)

        # currently request handlers are un-ordered
        for handler in self.handlers:
            res = handler(req, res, body)

        # Reserialize the response body
        if body is not None:
            res.body = utils.dumps(body)

        return res

class RequestExtensionResource(wsgi.Resource):

    def __init__(self, application):
        controller = RequestExtensionController(application)
        wsgi.Resource.__init__(self, controller)

    def add_handler(self, handler):
        self.controller.add_handler(handler)

class ExtensionsResource(wsgi.Resource):

    def __init__(self, extension_manager):
        self.extension_manager = extension_manager

    def _translate(self, ext):
        ext_data = {}
        ext_data['name'] = ext.name
        ext_data['alias'] = ext.alias
        ext_data['description'] = ext.__doc__
        ext_data['namespace'] = ext.namespace
        ext_data['updated'] = ext.updated
        ext_data['links'] = []  # TODO(dprince): implement extension links
        return ext_data

    def index(self, req):
        extensions = []
        for _alias, ext in self.extension_manager.extensions.iteritems():
            extensions.append(self._translate(ext))
        return dict(extensions=extensions)

    def show(self, req, id):
        try:
            # NOTE(dprince): the extensions alias is used as the 'id' for show
            ext = self.extension_manager.extensions[id]
        except KeyError:
            raise webob.exc.HTTPNotFound()

        return dict(extension=self._translate(ext))

    def delete(self, req, id):
        raise webob.exc.HTTPNotFound()

    def create(self, req):
        raise webob.exc.HTTPNotFound()

class ExtensionMiddleware(base_wsgi.Middleware):
    """Extensions middleware for WSGI."""
    @classmethod
    def factory(cls, global_config, **local_config):
        """Paste factory."""
        def _factory(app):
            return cls(app, **local_config)
        return _factory

    def _action_ext_resources(self, application, ext_mgr, mapper):
        """Return a dict of ActionExtensionResource-s by collection."""
        action_resources = {}
        for action in ext_mgr.get_actions():
            if not action.collection in action_resources.keys():
                resource = ActionExtensionResource(application)
                mapper.connect("/:(project_id)/%s/:(id)/action.:(format)" %
                                action.collection,
                                action='action',
                                controller=resource,
                                conditions=dict(method=['POST']))
                mapper.connect("/:(project_id)/%s/:(id)/action" %
                                action.collection,
                                action='action',
                                controller=resource,
                                conditions=dict(method=['POST']))
                action_resources[action.collection] = resource

        return action_resources

    def _request_ext_resources(self, application, ext_mgr, mapper):
        """Returns a dict of RequestExtensionResource-s by collection."""
        request_ext_resources = {}
        for req_ext in ext_mgr.get_request_extensions():
            if not req_ext.key in request_ext_resources.keys():
                resource = RequestExtensionResource(application)
                mapper.connect(req_ext.url_route + '.:(format)',
                                action='process',
                                controller=resource,
                                conditions=req_ext.conditions)

                mapper.connect(req_ext.url_route,
                                action='process',
                                controller=resource,
                                conditions=req_ext.conditions)
                request_ext_resources[req_ext.key] = resource

        return request_ext_resources

    def __init__(self, application, ext_mgr=None):

        if ext_mgr is None:
            ext_mgr = ExtensionManager()
        self.ext_mgr = ext_mgr

        mapper = nova.api.openstack.ProjectMapper()

        serializer = wsgi.ResponseSerializer(
            {'application/xml': ExtensionsXMLSerializer()})
        # extended resources
        for resource in ext_mgr.get_resources():
            LOG.debug(_('Extended resource: %s'),
                        resource.collection)
            if resource.serializer is None:
                resource.serializer = serializer

            kargs = dict(
                controller=wsgi.Resource(
                    resource.controller, resource.deserializer,
                    resource.serializer),
                collection=resource.collection_actions,
                member=resource.member_actions)

            if resource.parent:
                kargs['parent_resource'] = resource.parent

            mapper.resource(resource.collection, resource.collection, **kargs)

        # extended actions
        action_resources = self._action_ext_resources(application, ext_mgr,
                                                        mapper)
        for action in ext_mgr.get_actions():
            LOG.debug(_('Extended action: %s'), action.action_name)
            resource = action_resources[action.collection]
            resource.add_action(action.action_name, action.handler)

        # extended requests
        req_controllers = self._request_ext_resources(application, ext_mgr,
                                                      mapper)
        for request_ext in ext_mgr.get_request_extensions():
            LOG.debug(_('Extended request: %s'), request_ext.key)
            controller = req_controllers[request_ext.key]
            controller.add_handler(request_ext.handler)

        self._router = routes.middleware.RoutesMiddleware(self._dispatch,
                                                          mapper)

        super(ExtensionMiddleware, self).__init__(application)

    @webob.dec.wsgify(RequestClass=wsgi.Request)
    def __call__(self, req):
        """Route the incoming request with router."""
        req.environ['extended.app'] = self.application
        return self._router

    @staticmethod
    @webob.dec.wsgify(RequestClass=wsgi.Request)
    def _dispatch(req):
        """Dispatch the request.

        Returns the routed WSGI app's response or defers to the extended
        application.

        """
        match = req.environ['wsgiorg.routing_args'][1]
        if not match:
            return req.environ['extended.app']
        app = match['controller']
        return app

class ExtensionManager(object):
    """Load extensions from the configured extension path.

    See nova/tests/api/openstack/extensions/foxinsocks/extension.py for an
    example extension implementation.

    """

    def __init__(self):
        LOG.audit(_('Initializing extension manager.'))

        self.extensions = {}
        self._load_extensions()

    def register(self, ext):
        # Do nothing if the extension doesn't check out
        if not self._check_extension(ext):
            return

        alias = ext.alias
        LOG.audit(_('Loaded extension: %s'), alias)

        if alias in self.extensions:
            raise exception.Error("Found duplicate extension: %s" % alias)
        self.extensions[alias] = ext

    def get_resources(self):
        """Returns a list of ResourceExtension objects."""
        resources = []
        resources.append(ResourceExtension('extensions',
                                            ExtensionsResource(self)))
        for ext in self.extensions.values():
            try:
                resources.extend(ext.get_resources())
            except AttributeError:
                # NOTE(dprince): Extension aren't required to have resource
                # extensions
                pass
        return resources

    def get_actions(self):
        """Returns a list of ActionExtension objects."""
        actions = []
        for ext in self.extensions.values():
            try:
                actions.extend(ext.get_actions())
            except AttributeError:
                # NOTE(dprince): Extension aren't required to have action
                # extensions
                pass
        return actions

    def get_request_extensions(self):
        """Returns a list of RequestExtension objects."""
        request_exts = []
        for ext in self.extensions.values():
            try:
                request_exts.extend(ext.get_request_extensions())
            except AttributeError:
                # NOTE(dprince): Extension aren't required to have request
                # extensions
                pass
        return request_exts

    def _check_extension(self, extension):
        """Checks for required methods in extension objects."""
        try:
            LOG.debug(_('Ext name: %s'), extension.name)
            LOG.debug(_('Ext alias: %s'), extension.alias)
            LOG.debug(_('Ext description: %s'),
                      ' '.join(extension.__doc__.strip().split()))
            LOG.debug(_('Ext namespace: %s'), extension.namespace)
            LOG.debug(_('Ext updated: %s'), extension.updated)
        except AttributeError as ex:
            LOG.exception(_("Exception loading extension: %s"), unicode(ex))
            return False
        return True

    def load_extension(self, ext_factory):
        """Execute an extension factory.

        Loads an extension.  The 'ext_factory' is the name of a
        callable that will be imported and called with one
        argument--the extension manager.  The factory callable is
        expected to call the register() method at least once.
        """

        LOG.debug(_("Loading extension %s"), ext_factory)

        # Load the factory
        factory = utils.import_class(ext_factory)

        # Call it
        LOG.debug(_("Calling extension factory %s"), ext_factory)
        factory(self)

    def _load_extensions(self):
        """Load extensions specified on the command line."""

        for ext_factory in FLAGS.osapi_extension:
            try:
                self.load_extension(ext_factory)
            except Exception as exc:
                LOG.warn(_('Failed to load extension %(ext_factory)s: '
                           '%(exc)s') % locals())

class RequestExtension(object):
    """Extend requests and responses of core nova OpenStack API resources.

    Provide a way to add data to responses and handle custom request data
    that is sent to core nova OpenStack API controllers.

    """
    def __init__(self, method, url_route, handler):
        self.url_route = url_route
        self.handler = handler
        self.conditions = dict(method=[method])
        self.key = "%s-%s" % (method, url_route)

class ActionExtension(object):
    """Add custom actions to core nova OpenStack API resources."""

    def __init__(self, collection, action_name, handler):
        self.collection = collection
        self.action_name = action_name
        self.handler = handler

class ResourceExtension(object):
    """Add top level resources to the OpenStack API in nova."""

    def __init__(self, collection, controller, parent=None,
                 collection_actions=None, member_actions=None,
                 deserializer=None, serializer=None):
        if not collection_actions:
            collection_actions = {}
        if not member_actions:
            member_actions = {}
        self.collection = collection
        self.controller = controller
        self.parent = parent
        self.collection_actions = collection_actions
        self.member_actions = member_actions
        self.deserializer = deserializer
        self.serializer = serializer

class ExtensionsXMLSerializer(wsgi.XMLDictSerializer):

    NSMAP = {None: xmlutil.XMLNS_V11, 'atom': xmlutil.XMLNS_ATOM}

    def show(self, ext_dict):
        ext = etree.Element('extension', nsmap=self.NSMAP)
        self._populate_ext(ext, ext_dict['extension'])
        return self._to_xml(ext)

    def index(self, exts_dict):
        exts = etree.Element('extensions', nsmap=self.NSMAP)
        for ext_dict in exts_dict['extensions']:
            ext = etree.SubElement(exts, 'extension')
            self._populate_ext(ext, ext_dict)
        return self._to_xml(exts)

    def _populate_ext(self, ext_elem, ext_dict):
        """Populate an extension xml element from a dict."""

        ext_elem.set('name', ext_dict['name'])
        ext_elem.set('namespace', ext_dict['namespace'])
        ext_elem.set('alias', ext_dict['alias'])
        ext_elem.set('updated', ext_dict['updated'])
        desc = etree.Element('description')
        desc.text = ext_dict['description']
        ext_elem.append(desc)
        for link in ext_dict.get('links', []):
            elem = etree.SubElement(ext_elem, '{%s}link' % xmlutil.XMLNS_ATOM)
            elem.set('rel', link['rel'])
            elem.set('href', link['href'])
            elem.set('type', link['type'])
        return ext_elem

    def _to_xml(self, root):
        """Convert the xml object to an xml string."""

        return etree.tostring(root, encoding='UTF-8')

def admin_only(fnc):
    @functools.wraps(fnc)
    def _wrapped(self, *args, **kwargs):
        if FLAGS.allow_admin_api:
            return fnc(self, *args, **kwargs)
        raise webob.exc.HTTPNotFound()
    _wrapped.func_name = fnc.func_name
    return _wrapped

def wrap_errors(fn):
    """"Ensure errors are not passed along."""
    def wrapped(*args):
        try:
            return fn(*args)
        except Exception, e:
            raise webob.exc.HTTPInternalServerError()
    return wrapped
<commit_msg>Fix typo in docstring<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2011 OpenStack LLC.
# Copyright 2011 Justin Santa Barbara
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import functools
import imp
import inspect
import os
import sys

from lxml import etree
import routes
import webob.dec
import webob.exc

from nova import exception
from nova import flags
from nova import log as logging
from nova import utils
from nova import wsgi as base_wsgi
import nova.api.openstack
from nova.api.openstack import common
from nova.api.openstack import wsgi
from nova.api.openstack import xmlutil

LOG = logging.getLogger('nova.api.openstack.extensions')

FLAGS = flags.FLAGS

class ExtensionDescriptor(object):
    """Base class that defines the contract for extensions.

    Note that you don't have to derive from this class to have a valid
    extension; it is purely a convenience.

    """

    # The name of the extension, e.g., 'Fox In Socks'
    name = None

    # The alias for the extension, e.g., 'FOXNSOX'
    alias = None

    # Description comes from the docstring for the class

    # The XML namespace for the extension, e.g.,
    # 'http://www.fox.in.socks/api/ext/pie/v1.0'
    namespace = None

    # The timestamp when the extension was last updated, e.g.,
    # '2011-01-22T13:25:27-06:00'
    updated = None

    def __init__(self, ext_mgr):
        """Register extension with the extension manager."""

        ext_mgr.register(self)

    def get_resources(self):
        """List of extensions.ResourceExtension extension objects.

        Resources define new nouns, and are accessible through URLs.

        """
        resources = []
        return resources

    def get_actions(self):
        """List of extensions.ActionExtension extension objects.

        Actions are verbs callable from the API.

        """
        actions = []
        return actions

    def get_request_extensions(self):
        """List of extensions.RequestExtension extension objects.

        Request extensions are used to handle custom request data.

        """
        request_exts = []
        return request_exts

class ActionExtensionController(object):
    def __init__(self, application):
        self.application = application
        self.action_handlers = {}

    def add_action(self, action_name, handler):
        self.action_handlers[action_name] = handler

    def action(self, req, id, body):
        for action_name, handler in self.action_handlers.iteritems():
            if action_name in body:
                return handler(body, req, id)
        # no action handler found (bump to downstream application)
        res = self.application
        return res

class ActionExtensionResource(wsgi.Resource):

    def __init__(self, application):
        controller = ActionExtensionController(application)
        wsgi.Resource.__init__(self, controller)

    def add_action(self, action_name, handler):
        self.controller.add_action(action_name, handler)

class RequestExtensionController(object):

    def __init__(self, application):
        self.application = application
        self.handlers = []

    def add_handler(self, handler):
        self.handlers.append(handler)

    def process(self, req, *args, **kwargs):
        res = req.get_response(self.application)

        # Deserialize the response body, if any
        body = None
        if res.body:
            body = utils.loads(res.body)

        # currently request handlers are un-ordered
        for handler in self.handlers:
            res = handler(req, res, body)

        # Reserialize the response body
        if body is not None:
            res.body = utils.dumps(body)

        return res

class RequestExtensionResource(wsgi.Resource):

    def __init__(self, application):
        controller = RequestExtensionController(application)
        wsgi.Resource.__init__(self, controller)

    def add_handler(self, handler):
        self.controller.add_handler(handler)

class ExtensionsResource(wsgi.Resource):

    def __init__(self, extension_manager):
        self.extension_manager = extension_manager

    def _translate(self, ext):
        ext_data = {}
        ext_data['name'] = ext.name
        ext_data['alias'] = ext.alias
        ext_data['description'] = ext.__doc__
        ext_data['namespace'] = ext.namespace
        ext_data['updated'] = ext.updated
        ext_data['links'] = []  # TODO(dprince): implement extension links
        return ext_data

    def index(self, req):
        extensions = []
        for _alias, ext in self.extension_manager.extensions.iteritems():
            extensions.append(self._translate(ext))
        return dict(extensions=extensions)

    def show(self, req, id):
        try:
            # NOTE(dprince): the extensions alias is used as the 'id' for show
            ext = self.extension_manager.extensions[id]
        except KeyError:
            raise webob.exc.HTTPNotFound()

        return dict(extension=self._translate(ext))

    def delete(self, req, id):
        raise webob.exc.HTTPNotFound()

    def create(self, req):
        raise webob.exc.HTTPNotFound()

class ExtensionMiddleware(base_wsgi.Middleware):
    """Extensions middleware for WSGI."""
    @classmethod
    def factory(cls, global_config, **local_config):
        """Paste factory."""
        def _factory(app):
            return cls(app, **local_config)
        return _factory

    def _action_ext_resources(self, application, ext_mgr, mapper):
        """Return a dict of ActionExtensionResource-s by collection."""
        action_resources = {}
        for action in ext_mgr.get_actions():
            if not action.collection in action_resources.keys():
                resource = ActionExtensionResource(application)
                mapper.connect("/:(project_id)/%s/:(id)/action.:(format)" %
                                action.collection,
                                action='action',
                                controller=resource,
                                conditions=dict(method=['POST']))
                mapper.connect("/:(project_id)/%s/:(id)/action" %
                                action.collection,
                                action='action',
                                controller=resource,
                                conditions=dict(method=['POST']))
                action_resources[action.collection] = resource

        return action_resources

    def _request_ext_resources(self, application, ext_mgr, mapper):
        """Returns a dict of RequestExtensionResource-s by collection."""
        request_ext_resources = {}
        for req_ext in ext_mgr.get_request_extensions():
            if not req_ext.key in request_ext_resources.keys():
                resource = RequestExtensionResource(application)
                mapper.connect(req_ext.url_route + '.:(format)',
                                action='process',
                                controller=resource,
                                conditions=req_ext.conditions)

                mapper.connect(req_ext.url_route,
                                action='process',
                                controller=resource,
                                conditions=req_ext.conditions)
                request_ext_resources[req_ext.key] = resource

        return request_ext_resources

    def __init__(self, application, ext_mgr=None):

        if ext_mgr is None:
            ext_mgr = ExtensionManager()
        self.ext_mgr = ext_mgr

        mapper = nova.api.openstack.ProjectMapper()

        serializer = wsgi.ResponseSerializer(
            {'application/xml': ExtensionsXMLSerializer()})
        # extended resources
        for resource in ext_mgr.get_resources():
            LOG.debug(_('Extended resource: %s'),
                        resource.collection)
            if resource.serializer is None:
                resource.serializer = serializer

            kargs = dict(
                controller=wsgi.Resource(
                    resource.controller, resource.deserializer,
                    resource.serializer),
                collection=resource.collection_actions,
                member=resource.member_actions)

            if resource.parent:
                kargs['parent_resource'] = resource.parent

            mapper.resource(resource.collection, resource.collection, **kargs)

        # extended actions
        action_resources = self._action_ext_resources(application, ext_mgr,
                                                        mapper)
        for action in ext_mgr.get_actions():
            LOG.debug(_('Extended action: %s'), action.action_name)
            resource = action_resources[action.collection]
            resource.add_action(action.action_name, action.handler)

        # extended requests
        req_controllers = self._request_ext_resources(application, ext_mgr,
                                                      mapper)
        for request_ext in ext_mgr.get_request_extensions():
            LOG.debug(_('Extended request: %s'), request_ext.key)
            controller = req_controllers[request_ext.key]
            controller.add_handler(request_ext.handler)

        self._router = routes.middleware.RoutesMiddleware(self._dispatch,
                                                          mapper)

        super(ExtensionMiddleware, self).__init__(application)

    @webob.dec.wsgify(RequestClass=wsgi.Request)
    def __call__(self, req):
        """Route the incoming request with router."""
        req.environ['extended.app'] = self.application
        return self._router

    @staticmethod
    @webob.dec.wsgify(RequestClass=wsgi.Request)
    def _dispatch(req):
        """Dispatch the request.

        Returns the routed WSGI app's response or defers to the extended
        application.

        """
        match = req.environ['wsgiorg.routing_args'][1]
        if not match:
            return req.environ['extended.app']
        app = match['controller']
        return app

class ExtensionManager(object):
    """Load extensions from the configured extension path.

    See nova/tests/api/openstack/extensions/foxinsocks/extension.py for an
    example extension implementation.

    """

    def __init__(self):
        LOG.audit(_('Initializing extension manager.'))

        self.extensions = {}
        self._load_extensions()

    def register(self, ext):
        # Do nothing if the extension doesn't check out
        if not self._check_extension(ext):
            return

        alias = ext.alias
        LOG.audit(_('Loaded extension: %s'), alias)

        if alias in self.extensions:
            raise exception.Error("Found duplicate extension: %s" % alias)
        self.extensions[alias] = ext

    def get_resources(self):
        """Returns a list of ResourceExtension objects."""
        resources = []
        resources.append(ResourceExtension('extensions',
                                            ExtensionsResource(self)))
        for ext in self.extensions.values():
            try:
                resources.extend(ext.get_resources())
            except AttributeError:
                # NOTE(dprince): Extension aren't required to have resource
                # extensions
                pass
        return resources

    def get_actions(self):
        """Returns a list of ActionExtension objects."""
        actions = []
        for ext in self.extensions.values():
            try:
                actions.extend(ext.get_actions())
            except AttributeError:
                # NOTE(dprince): Extension aren't required to have action
                # extensions
                pass
        return actions

    def get_request_extensions(self):
        """Returns a list of RequestExtension objects."""
        request_exts = []
        for ext in self.extensions.values():
            try:
                request_exts.extend(ext.get_request_extensions())
            except AttributeError:
                # NOTE(dprince): Extension aren't required to have request
                # extensions
                pass
        return request_exts

    def _check_extension(self, extension):
        """Checks for required methods in extension objects."""
        try:
            LOG.debug(_('Ext name: %s'), extension.name)
            LOG.debug(_('Ext alias: %s'), extension.alias)
            LOG.debug(_('Ext description: %s'),
                      ' '.join(extension.__doc__.strip().split()))
            LOG.debug(_('Ext namespace: %s'), extension.namespace)
            LOG.debug(_('Ext updated: %s'), extension.updated)
        except AttributeError as ex:
            LOG.exception(_("Exception loading extension: %s"), unicode(ex))
            return False
        return True

    def load_extension(self, ext_factory):
        """Execute an extension factory.

        Loads an extension.  The 'ext_factory' is the name of a
        callable that will be imported and called with one
        argument--the extension manager.  The factory callable is
        expected to call the register() method at least once.
        """

        LOG.debug(_("Loading extension %s"), ext_factory)

        # Load the factory
        factory = utils.import_class(ext_factory)

        # Call it
        LOG.debug(_("Calling extension factory %s"), ext_factory)
        factory(self)

    def _load_extensions(self):
        """Load extensions specified on the command line."""

        for ext_factory in FLAGS.osapi_extension:
            try:
                self.load_extension(ext_factory)
            except Exception as exc:
                LOG.warn(_('Failed to load extension %(ext_factory)s: '
                           '%(exc)s') % locals())

class RequestExtension(object):
    """Extend requests and responses of core nova OpenStack API resources.

    Provide a way to add data to responses and handle custom request data
    that is sent to core nova OpenStack API controllers.

    """
    def __init__(self, method, url_route, handler):
        self.url_route = url_route
        self.handler = handler
        self.conditions = dict(method=[method])
        self.key = "%s-%s" % (method, url_route)

class ActionExtension(object):
    """Add custom actions to core nova OpenStack API resources."""

    def __init__(self, collection, action_name, handler):
        self.collection = collection
        self.action_name = action_name
        self.handler = handler

class ResourceExtension(object):
    """Add top level resources to the OpenStack API in nova."""

    def __init__(self, collection, controller, parent=None,
                 collection_actions=None, member_actions=None,
                 deserializer=None, serializer=None):
        if not collection_actions:
            collection_actions = {}
        if not member_actions:
            member_actions = {}
        self.collection = collection
        self.controller = controller
        self.parent = parent
        self.collection_actions = collection_actions
        self.member_actions = member_actions
        self.deserializer = deserializer
        self.serializer = serializer

class ExtensionsXMLSerializer(wsgi.XMLDictSerializer):

    NSMAP = {None: xmlutil.XMLNS_V11, 'atom': xmlutil.XMLNS_ATOM}

    def show(self, ext_dict):
        ext = etree.Element('extension', nsmap=self.NSMAP)
        self._populate_ext(ext, ext_dict['extension'])
        return self._to_xml(ext)

    def index(self, exts_dict):
        exts = etree.Element('extensions', nsmap=self.NSMAP)
        for ext_dict in exts_dict['extensions']:
            ext = etree.SubElement(exts, 'extension')
            self._populate_ext(ext, ext_dict)
        return self._to_xml(exts)

    def _populate_ext(self, ext_elem, ext_dict):
        """Populate an extension xml element from a dict."""

        ext_elem.set('name', ext_dict['name'])
        ext_elem.set('namespace', ext_dict['namespace'])
        ext_elem.set('alias', ext_dict['alias'])
        ext_elem.set('updated', ext_dict['updated'])
        desc = etree.Element('description')
        desc.text = ext_dict['description']
        ext_elem.append(desc)
        for link in ext_dict.get('links', []):
            elem = etree.SubElement(ext_elem, '{%s}link' % xmlutil.XMLNS_ATOM)
            elem.set('rel', link['rel'])
            elem.set('href', link['href'])
            elem.set('type', link['type'])
        return ext_elem

    def _to_xml(self, root):
        """Convert the xml object to an xml string."""

        return etree.tostring(root, encoding='UTF-8')

def admin_only(fnc):
    @functools.wraps(fnc)
    def _wrapped(self, *args, **kwargs):
        if FLAGS.allow_admin_api:
            return fnc(self, *args, **kwargs)
        raise webob.exc.HTTPNotFound()
    _wrapped.func_name = fnc.func_name
    return _wrapped

def wrap_errors(fn):
    """"Ensure errors are not passed along."""
    def wrapped(*args):
        try:
            return fn(*args)
        except Exception, e:
            raise webob.exc.HTTPInternalServerError()
    return wrapped
<|endoftext|>


<commit_before>"""Utility I/O functions."""

# -----------------------------------------------------------------------------
# Imports
# -----------------------------------------------------------------------------
import json

import numpy as np

# -----------------------------------------------------------------------------
# Data type conversions
# -----------------------------------------------------------------------------
_maxint16 = 2.**12
_maxint16inv = 1./_maxint16
_maxint8 = 127
_maxint8inv = 1./255
_dtype_factors = {
    (np.int16, np.float32): _maxint16inv,
    (np.int16, np.float64): _maxint16inv,
    (np.int32, np.float32): _maxint16inv,
    (np.int32, np.float64): _maxint16inv,
    (np.int64, np.float32): _maxint16inv,
    (np.int64, np.float64): _maxint16inv,
    (np.float32, np.int16): _maxint16,
    (np.float64, np.int16): _maxint16,
    (np.int8, np.float32): _maxint8inv,
    (np.float32, np.int8): _maxint8,
}

def _get_dtype(dtype):
    if isinstance(dtype, np.dtype):
        return dtype.type
    else:
        return dtype

def convert_dtype(data, dtype=None, factor=None):
    if not dtype:
        return data
    dtype_old = data.dtype
    if dtype_old == dtype:
        return data
    key = (_get_dtype(dtype_old), _get_dtype(dtype))
    factor = factor or _dtype_factors.get(key, 1)
    if dtype_old in (np.float32, np.float64):
        factor = factor/data.max()
    # We avoid unnecessary array copy when factor == 1
    if factor != 1:
        return (data * factor).astype(dtype)
    else:
        return data.astype(dtype)

def ensure_vector(data, size=None):
    """Ensure data is a 1D vector."""
    if isinstance(data, (list, tuple)):
        data = np.array(data)
    elif not isinstance(data, np.ndarray):
        data = np.array([data])
    assert data.ndim == 1
    # If size is specified, tile the vector by this size.
    if len(data) == 1 and size is not None:
        data = np.tile(data, size)
    return data

# -----------------------------------------------------------------------------
# JSON functions
# -----------------------------------------------------------------------------
def load_json(path):
    with open(path, 'r') as f:
        return json.load(f)

def save_json(path, d):
    with open(path, 'w') as f:
        json.dump(d, f, indent=4)

<commit_msg>Fixing dtype conversion bug.<commit_after>"""Utility I/O functions."""

# -----------------------------------------------------------------------------
# Imports
# -----------------------------------------------------------------------------
import json

import numpy as np

# -----------------------------------------------------------------------------
# Data type conversions
# -----------------------------------------------------------------------------
_maxint16 = 2.**12
_maxint16inv = 1./_maxint16
_maxint8 = 127
_maxint8inv = 1./255
_dtype_factors = {
    (np.int16, np.float32): _maxint16inv,
    (np.int16, np.float64): _maxint16inv,
    (np.int32, np.float32): _maxint16inv,
    (np.int32, np.float64): _maxint16inv,
    (np.int64, np.float32): _maxint16inv,
    (np.int64, np.float64): _maxint16inv,
    (np.float32, np.int16): _maxint16,
    (np.float64, np.int16): _maxint16,
    (np.int8, np.float32): _maxint8inv,
    (np.float32, np.int8): _maxint8,
}

def _get_dtype(dtype):
    if isinstance(dtype, np.dtype):
        return dtype.type
    else:
        return dtype

def convert_dtype(data, dtype=None, factor=None):
    if not dtype:
        return data
    dtype_old = data.dtype
    if dtype_old == dtype:
        return data
    key = (_get_dtype(dtype_old), _get_dtype(dtype))
    factor = factor or _dtype_factors.get(key, 1)
    if dtype_old in (np.float32, np.float64):
        factor = factor/np.abs(data).max()
    # We avoid unnecessary array copy when factor == 1
    if factor != 1:
        return (data * factor).astype(dtype)
    else:
        return data.astype(dtype)

def ensure_vector(data, size=None):
    """Ensure data is a 1D vector."""
    if isinstance(data, (list, tuple)):
        data = np.array(data)
    elif not isinstance(data, np.ndarray):
        data = np.array([data])
    assert data.ndim == 1
    # If size is specified, tile the vector by this size.
    if len(data) == 1 and size is not None:
        data = np.tile(data, size)
    return data

# -----------------------------------------------------------------------------
# JSON functions
# -----------------------------------------------------------------------------
def load_json(path):
    with open(path, 'r') as f:
        return json.load(f)

def save_json(path, d):
    with open(path, 'w') as f:
        json.dump(d, f, indent=4)

<|endoftext|>


<commit_before>from . import base
from . import mixins

from datetime import date

class TransformedRecord(
    mixins.GenericCompensationMixin,
    mixins.GenericDepartmentMixin, mixins.GenericIdentifierMixin,
    mixins.GenericJobTitleMixin, mixins.GenericPersonMixin,
    mixins.MembershipMixin, mixins.OrganizationMixin, mixins.PostMixin,
        mixins.RaceMixin, mixins.LinkMixin, base.BaseTransformedRecord):

    MAP = {
        'last_name': 'Last',
        'first_name': 'First Name',
        'department': 'Dept',
        'job_title': 'Job Title',
        'hire_date': 'Start Date',
        'compensation': 'Annual Rt',
        'gender': 'Sex',
        'given_race': 'Race',
        'employee_type': 'Full/Part',
    }

    ORGANIZATION_NAME = 'University of Texas of the Permian Basin'

    NAME_FIELDS = ('first_name', 'last_name', )

    ORGANIZATION_CLASSIFICATION = 'Universtiy'

    DATE_PROVIDED = date(2016, 2, 29)

    URL = ('http://raw.texastribune.org.s3.amazonaws.com/ut_permian_basin'
           '/salaries/2016-02/ut_permbasin.xls')

    @property
    def is_valid(self):
        # Adjust to return False on invalid fields.  For example:
        return self.last_name.strip() != ''

    @property
    def compensation_type(self):
        emptype = self.get_mapped_value('employee_type')

        if 'Full' in emptype:
            return 'FT'
        else:
            return 'PT'

    @property
    def description(self):
        emptype = self.get_mapped_value('employee_type')

        if 'Full' in emptype:
            return 'Annual rate'
        else:
            return 'Part-time annual rate'

    @property
    def person(self):
        name = self.get_name()
        r = {
            'family_name': name.last,
            'given_name': name.first,
            'additional_name': name.middle,
            'name': unicode(name),
            'gender': self.gender.strip()
        }

        return r

    @property
    def race(self):
        race = self.given_race.strip()
        if race == '':
            race = 'Not given'
        return {'name': race}

transform = base.transform_factory(TransformedRecord)
<commit_msg>fixed first name in all caps to normal<commit_after>from . import base
from . import mixins

from datetime import date

class TransformedRecord(
    mixins.GenericCompensationMixin,
    mixins.GenericDepartmentMixin, mixins.GenericIdentifierMixin,
    mixins.GenericJobTitleMixin, mixins.GenericPersonMixin,
    mixins.MembershipMixin, mixins.OrganizationMixin, mixins.PostMixin,
        mixins.RaceMixin, mixins.LinkMixin, base.BaseTransformedRecord):

    MAP = {
        'last_name': 'Last',
        'first_name': 'First Name',
        'department': 'Dept',
        'job_title': 'Job Title',
        'hire_date': 'Start Date',
        'compensation': 'Annual Rt',
        'gender': 'Sex',
        'given_race': 'Race',
        'employee_type': 'Full/Part',
    }

    ORGANIZATION_NAME = 'University of Texas of the Permian Basin'

    ORGANIZATION_CLASSIFICATION = 'Universtiy'

    DATE_PROVIDED = date(2016, 2, 29)

    URL = ('http://raw.texastribune.org.s3.amazonaws.com/ut_permian_basin'
           '/salaries/2016-02/ut_permbasin.xls')

    @property
    def is_valid(self):
        # Adjust to return False on invalid fields.  For example:
        return self.last_name.strip() != ''

    @property
    def compensation_type(self):
        emptype = self.get_mapped_value('employee_type')

        if 'Full' in emptype:
            return 'FT'
        else:
            return 'PT'

    @property
    def description(self):
        emptype = self.get_mapped_value('employee_type')

        if 'Full' in emptype:
            return 'Annual rate'
        else:
            return 'Part-time annual rate'

    @property
    def person(self):
        formatted_name = self.get_name()

        r = {
            'family_name': formatted_name['last_name'],
            'given_name': formatted_name['first_name'],
            'name': " ".join([n for n in formatted_name.values()]),
            'gender': self.gender.strip()
        }

        return r

    def get_name(self):
        last_name = self.get_mapped_value('last_name')
        first_name = self.get_mapped_value('first_name').title()
        return {
            'first_name': first_name,
            'last_name': last_name
        }

    @property
    def race(self):
        race = self.given_race.strip()
        if race == '':
            race = 'Not given'
        return {'name': race}

transform = base.transform_factory(TransformedRecord)
<|endoftext|>