<commit_before>import itertools
import os
import struct
import marisa_trie
from utils import numerics, words
from dictionaries import dictionary
from config import lmconfig

class DictDict():
    def __init__(self, known_words):
        self._cost = {}
        self._updated_count = {}
        self._count = 0
        dictionary = {}
        for word in known_words:
            s, p = words.surface_pronoun(word)
            dictionary[p] = word.encode('utf-8')
        self._dict = marisa_trie.BytesTrie(dictionary.iteritems())

    def _cost_get(self, key):
        if key in self._cost:
            return self._cost[key]
        else:
            return 0

    def _dict_get(self, pronoun):
        return [word.decode('utf-8') for word in self._dict[pronoun]]

    def pronoun_prefixes(self, pronoun):
        return self._dict.prefixes(pronoun)

    def get_from_pronoun(self, pronoun):
        words = self._dict_get(pronoun)
        return [(word, self._cost_get(word)) for word in words]

    def get_unknownword_cost(self, unknown):
        return self._cost_get(unknown)

    def get_bigram_cost(self, word1, word2):
        k = words.compose_bigram_key(word1, word2)
        return self._cost_get(k)

    def fobos_update(self, g):
        self._count += 1
        for key in g.dict.keys():
            w = self._cost_get(key) + lmconfig.eta * g.get(key)
            if key in self._updated_count:
                d = self._updated_count[key]
            else:
                d = 0
            w = numerics.clip(w, lmconfig.normalization_factor * (self._count - d))
            if w == 0 and key in self._cost:
                del(self._cost[key])
            else:
                self._cost[key] = w
            self._updated_count[key] = self._count

    def fobos_regularize(self):
        for key in self._cost.keys():
            w = numerics.clip(self._cost[key], lmconfig.normalization_factor * (self._count - self._updated_count[key]))
            if w == 0 and key in self._cost:
                del(self._cost[key])
            else:
                self._cost[key] = w

    def save(self, path):
        dictionary_with_cost = [(p, (word, self._cost[word])) for p, word in self._dict.items()]
        dict_trie = marisa_trie.RecordTrie('<sf', dictionary_with_cost)
        dict_filename = os.path.join(path, 'dictionary')
        dict_trie.save(dict_filename)

        costs = ((key, cost) for (key, cost) in self._cost.iteritems() if not words.is_word(key))
        bytesitems = ((key, struct.pack('<f', cost)) for key, cost in costs)
        cost_trie = marisa_trie.BytesTrie(bytesitems)
        costs_filename = os.path.join(path, 'costs')
        cost_trie.save(costs_filename)
<commit_msg>CRF model learning (completed)<commit_after>import itertools
import os
import struct
import marisa_trie
from utils import numerics, words
from dictionaries import dictionary
from config import lmconfig

class DictDict():
    def __init__(self, known_words):
        self._cost = {}
        self._updated_count = {}
        self._count = 0
        dictionary = {}
        for word in known_words:
            s, p = words.surface_pronoun(word)
            dictionary[p] = word.encode('utf-8')
        self._dict = marisa_trie.BytesTrie(dictionary.iteritems())

    def _get_cost(self, key):
        if key in self._cost:
            return self._cost[key]
        else:
            return 0

    def _dict_get(self, pronoun):
        return [word.decode('utf-8') for word in self._dict[pronoun]]

    def pronoun_prefixes(self, pronoun):
        return self._dict.prefixes(pronoun)

    def get_from_pronoun(self, pronoun):
        words = self._dict_get(pronoun)
        return [(word, self._get_cost(word)) for word in words]

    def get_unknownword_cost(self, unknown):
        return self._get_cost(unknown)

    def get_bigram_cost(self, word1, word2):
        k = words.compose_bigram_key(word1, word2)
        return self._get_cost(k)

    def fobos_update(self, g):
        self._count += 1
        for key in g.dict.keys():
            w = self._get_cost(key) + lmconfig.eta * g.get(key)
            if key in self._updated_count:
                d = self._updated_count[key]
            else:
                d = 0
            w = numerics.clip(w, lmconfig.normalization_factor * (self._count - d))
            if w == 0 and key in self._cost:
                del(self._cost[key])
            else:
                self._cost[key] = w
            self._updated_count[key] = self._count

    def fobos_regularize(self):
        for key in self._cost.keys():
            w = numerics.clip(self._cost[key], lmconfig.normalization_factor * (self._count - self._updated_count[key]))
            if w == 0 and key in self._cost:
                del(self._cost[key])
            else:
                self._cost[key] = w

    def save(self, path):
        dictionary_with_cost = [(p, (word, self._get_cost(word))) for p, word in self._dict.items()]
        dict_trie = marisa_trie.RecordTrie('<sf', dictionary_with_cost)
        dict_filename = os.path.join(path, 'dictionary')
        dict_trie.save(dict_filename)

        costs = ((key, cost) for (key, cost) in self._cost.iteritems() if not words.is_word(key))
        bytesitems = ((key, struct.pack('<f', cost)) for key, cost in costs)
        cost_trie = marisa_trie.BytesTrie(bytesitems)
        costs_filename = os.path.join(path, 'costs')
        cost_trie.save(costs_filename)
<|endoftext|>


<commit_before># Natural Language Toolkit: Confusion Matrices
#
# Copyright (C) 2001-2022 NLTK Project
# Author: Edward Loper <edloper@gmail.com>
#         Steven Bird <stevenbird1@gmail.com>
#         Tom Aarsen <>
# URL: <https://www.nltk.org/>
# For license information, see LICENSE.TXT

from nltk.probability import FreqDist

class ConfusionMatrix:
    """
    The confusion matrix between a list of reference values and a
    corresponding list of test values.  Entry *[r,t]* of this
    matrix is a count of the number of times that the reference value
    *r* corresponds to the test value *t*.  E.g.:

        >>> from nltk.metrics import ConfusionMatrix
        >>> ref  = 'DET NN VB DET JJ NN NN IN DET NN'.split()
        >>> test = 'DET VB VB DET NN NN NN IN DET NN'.split()
        >>> cm = ConfusionMatrix(ref, test)
        >>> print(cm['NN', 'NN'])
        3

    Note that the diagonal entries *Ri=Tj* of this matrix
    corresponds to correct values; and the off-diagonal entries
    correspond to incorrect values.
    """

    def __init__(self, reference, test, sort_by_count=False):
        """
        Construct a new confusion matrix from a list of reference
        values and a corresponding list of test values.

        :type reference: list
        :param reference: An ordered list of reference values.
        :type test: list
        :param test: A list of values to compare against the
            corresponding reference values.
        :raise ValueError: If ``reference`` and ``length`` do not have
            the same length.
        """
        if len(reference) != len(test):
            raise ValueError("Lists must have the same length.")

        # Get a list of all values.
        if sort_by_count:
            ref_fdist = FreqDist(reference)
            test_fdist = FreqDist(test)

            def key(v):
                return -(ref_fdist[v] + test_fdist[v])

            values = sorted(set(reference + test), key=key)
        else:
            values = sorted(set(reference + test))

        # Construct a value->index dictionary
        indices = {val: i for (i, val) in enumerate(values)}

        # Make a confusion matrix table.
        confusion = [[0 for val in values] for val in values]
        max_conf = 0  # Maximum confusion
        for w, g in zip(reference, test):
            confusion[indices[w]][indices[g]] += 1
            max_conf = max(max_conf, confusion[indices[w]][indices[g]])

        #: A list of all values in ``reference`` or ``test``.
        self._values = values
        #: A dictionary mapping values in ``self._values`` to their indices.
        self._indices = indices
        #: The confusion matrix itself (as a list of lists of counts).
        self._confusion = confusion
        #: The greatest count in ``self._confusion`` (used for printing).
        self._max_conf = max_conf
        #: The total number of values in the confusion matrix.
        self._total = len(reference)
        #: The number of correct (on-diagonal) values in the matrix.
        self._correct = sum(confusion[i][i] for i in range(len(values)))

    def __getitem__(self, li_lj_tuple):
        """
        :return: The number of times that value ``li`` was expected and
        value ``lj`` was given.
        :rtype: int
        """
        (li, lj) = li_lj_tuple
        i = self._indices[li]
        j = self._indices[lj]
        return self._confusion[i][j]

    def __repr__(self):
        return f"<ConfusionMatrix: {self._correct}/{self._total} correct>"

    def __str__(self):
        return self.pretty_format()

    def pretty_format(
        self,
        show_percents=False,
        values_in_chart=True,
        truncate=None,
        sort_by_count=False,
    ):
        """
        :return: A multi-line string representation of this confusion matrix.
        :type truncate: int
        :param truncate: If specified, then only show the specified
            number of values.  Any sorting (e.g., sort_by_count)
            will be performed before truncation.
        :param sort_by_count: If true, then sort by the count of each
            label in the reference data.  I.e., labels that occur more
            frequently in the reference label will be towards the left
            edge of the matrix, and labels that occur less frequently
            will be towards the right edge.

        @todo: add marginals?
        """
        confusion = self._confusion

        values = self._values
        if sort_by_count:
            values = sorted(
                values, key=lambda v: -sum(self._confusion[self._indices[v]])
            )

        if truncate:
            values = values[:truncate]

        if values_in_chart:
            value_strings = ["%s" % val for val in values]
        else:
            value_strings = [str(n + 1) for n in range(len(values))]

        # Construct a format string for row values
        valuelen = max(len(val) for val in value_strings)
        value_format = "%" + repr(valuelen) + "s | "
        # Construct a format string for matrix entries
        if show_percents:
            entrylen = 6
            entry_format = "%5.1f%%"
            zerostr = "     ."
        else:
            entrylen = len(repr(self._max_conf))
            entry_format = "%" + repr(entrylen) + "d"
            zerostr = " " * (entrylen - 1) + "."

        # Write the column values.
        s = ""
        for i in range(valuelen):
            s += (" " * valuelen) + " |"
            for val in value_strings:
                if i >= valuelen - len(val):
                    s += val[i - valuelen + len(val)].rjust(entrylen + 1)
                else:
                    s += " " * (entrylen + 1)
            s += " |\n"

        # Write a dividing line
        s += "{}-+-{}+\n".format("-" * valuelen, "-" * ((entrylen + 1) * len(values)))

        # Write the entries.
        for val, li in zip(value_strings, values):
            i = self._indices[li]
            s += value_format % val
            for lj in values:
                j = self._indices[lj]
                if confusion[i][j] == 0:
                    s += zerostr
                elif show_percents:
                    s += entry_format % (100.0 * confusion[i][j] / self._total)
                else:
                    s += entry_format % confusion[i][j]
                if i == j:
                    prevspace = s.rfind(" ")
                    s = s[:prevspace] + "<" + s[prevspace + 1 :] + ">"
                else:
                    s += " "
            s += "|\n"

        # Write a dividing line
        s += "{}-+-{}+\n".format("-" * valuelen, "-" * ((entrylen + 1) * len(values)))

        # Write a key
        s += "(row = reference; col = test)\n"
        if not values_in_chart:
            s += "Value key:\n"
            for i, value in enumerate(values):
                s += "%6d: %s\n" % (i + 1, value)

        return s

    def key(self):
        values = self._values
        str = "Value key:\n"
        indexlen = len(repr(len(values) - 1))
        key_format = "  %" + repr(indexlen) + "d: %s\n"
        for i in range(len(values)):
            str += key_format % (i, values[i])

        return str

    def recall(self, value):
        """Given a value in the confusion matrix, return the recall
        that corresponds to this value. The recall is defined as:

        - *r* = true positive / (true positive + false positive)

        and can loosely be considered the ratio of how often ``value``
        was predicted correctly relative to how often ``value`` was
        the true result.

        :param value: value used in the ConfusionMatrix
        :return: the recall corresponding to ``value``.
        :rtype: float
        """
        # Number of times `value` was correct, and also predicted
        TP = self[value, value]
        # Number of times `value` was correct
        TP_FN = sum(self[value, pred_value] for pred_value in self._values)
        if TP_FN == 0:
            return 0.0
        return TP / TP_FN

    def precision(self, value):
        """Given a value in the confusion matrix, return the precision
        that corresponds to this value. The precision is defined as:

        - *p* = true positive / (true positive + false negative)

        and can loosely be considered the ratio of how often ``value``
        was predicted correctly relative to the number of predictions
        for ``value``.

        :param value: value used in the ConfusionMatrix
        :return: the precision corresponding to ``value``.
        :rtype: float
        """
        # Number of times `value` was correct, and also predicted
        TP = self[value, value]
        # Number of times `value` was predicted
        TP_FP = sum(self[real_value, value] for real_value in self._values)
        if TP_FP == 0:
            return 0.0
        return TP / TP_FP

    def f_measure(self, value, alpha=0.5):
        """
        Given a value used in the confusion matrix, return the f-measure
        that corresponds to this value. The f-measure is the harmonic mean
        of the ``precision`` and ``recall``, weighted by ``alpha``.
        In particular, given the precision *p* and recall *r* defined by:

        - *p* = true positive / (true positive + false negative)
        - *r* = true positive / (true positive + false positive)

        The f-measure is:

        - *1/(alpha/p + (1-alpha)/r)*

        With ``alpha = 0.5``, this reduces to:

        - *2pr / (p + r)*

        :param value: value used in the ConfusionMatrix
        :param alpha: Ratio of the cost of false negative compared to false
            positives. Defaults to 0.5, where the costs are equal.
        :type alpha: float
        :return: the F-measure corresponding to ``value``.
        :rtype: float
        """
        p = self.precision(value)
        r = self.recall(value)
        if p == 0.0 or r == 0.0:
            return 0.0
        return 1.0 / (alpha / p + (1 - alpha) / r)

    def evaluate(self, alpha=0.5, truncate=None, sort_by_count=False):
        """
        Tabulate the **recall**, **precision** and **f-measure**
        for each value in this confusion matrix.

        >>> reference = "DET NN VB DET JJ NN NN IN DET NN".split()
        >>> test = "DET VB VB DET NN NN NN IN DET NN".split()
        >>> cm = ConfusionMatrix(reference, test)
        >>> print(cm.evaluate())
        Tag | Prec.  | Recall | F-measure
        ----+--------+--------+-----------
        DET | 1.0000 | 1.0000 | 1.0000
         IN | 1.0000 | 1.0000 | 1.0000
         JJ | 0.0000 | 0.0000 | 0.0000
         NN | 0.7500 | 0.7500 | 0.7500
         VB | 0.5000 | 1.0000 | 0.6667
        <BLANKLINE>

        :param alpha: Ratio of the cost of false negative compared to false
            positives, as used in the f-measure computation. Defaults to 0.5,
            where the costs are equal.
        :type alpha: float
        :param truncate: If specified, then only show the specified
            number of values. Any sorting (e.g., sort_by_count)
            will be performed before truncation. Defaults to None
        :type truncate: int, optional
        :param sort_by_count: Whether to sort the outputs on frequency
            in the reference label. Defaults to False.
        :type sort_by_count: bool, optional
        :return: A tabulated recall, precision and f-measure string
        :rtype: str
        """
        tags = self._values

        # Apply keyword parameters
        if sort_by_count:
            tags = sorted(tags, key=lambda v: -sum(self._confusion[self._indices[v]]))
        if truncate:
            tags = tags[:truncate]

        tag_column_len = max(max(len(tag) for tag in tags), 3)

        # Construct the header
        s = (
            f"{' ' * (tag_column_len - 3)}Tag | Prec.  | Recall | F-measure\n"
            f"{'-' * tag_column_len}-+--------+--------+-----------\n"
        )

        # Construct the body
        for tag in tags:
            s += (
                f"{tag:>{tag_column_len}} | "
                f"{self.precision(tag):<6.4f} | "
                f"{self.recall(tag):<6.4f} | "
                f"{self.f_measure(tag, alpha=alpha):.4f}\n"
            )

        return s

def demo():
    reference = "DET NN VB DET JJ NN NN IN DET NN".split()
    test = "DET VB VB DET NN NN NN IN DET NN".split()
    print("Reference =", reference)
    print("Test    =", test)
    print("Confusion matrix:")
    print(ConfusionMatrix(reference, test))
    print(ConfusionMatrix(reference, test).pretty_format(sort_by_count=True))

    print(ConfusionMatrix(reference, test).recall("VB"))

if __name__ == "__main__":
    demo()
<commit_msg>Rename unused variable to _ in list comprehension<commit_after># Natural Language Toolkit: Confusion Matrices
#
# Copyright (C) 2001-2022 NLTK Project
# Author: Edward Loper <edloper@gmail.com>
#         Steven Bird <stevenbird1@gmail.com>
#         Tom Aarsen <>
# URL: <https://www.nltk.org/>
# For license information, see LICENSE.TXT

from nltk.probability import FreqDist

class ConfusionMatrix:
    """
    The confusion matrix between a list of reference values and a
    corresponding list of test values.  Entry *[r,t]* of this
    matrix is a count of the number of times that the reference value
    *r* corresponds to the test value *t*.  E.g.:

        >>> from nltk.metrics import ConfusionMatrix
        >>> ref  = 'DET NN VB DET JJ NN NN IN DET NN'.split()
        >>> test = 'DET VB VB DET NN NN NN IN DET NN'.split()
        >>> cm = ConfusionMatrix(ref, test)
        >>> print(cm['NN', 'NN'])
        3

    Note that the diagonal entries *Ri=Tj* of this matrix
    corresponds to correct values; and the off-diagonal entries
    correspond to incorrect values.
    """

    def __init__(self, reference, test, sort_by_count=False):
        """
        Construct a new confusion matrix from a list of reference
        values and a corresponding list of test values.

        :type reference: list
        :param reference: An ordered list of reference values.
        :type test: list
        :param test: A list of values to compare against the
            corresponding reference values.
        :raise ValueError: If ``reference`` and ``length`` do not have
            the same length.
        """
        if len(reference) != len(test):
            raise ValueError("Lists must have the same length.")

        # Get a list of all values.
        if sort_by_count:
            ref_fdist = FreqDist(reference)
            test_fdist = FreqDist(test)

            def key(v):
                return -(ref_fdist[v] + test_fdist[v])

            values = sorted(set(reference + test), key=key)
        else:
            values = sorted(set(reference + test))

        # Construct a value->index dictionary
        indices = {val: i for (i, val) in enumerate(values)}

        # Make a confusion matrix table.
        confusion = [[0 for _ in values] for _ in values]
        max_conf = 0  # Maximum confusion
        for w, g in zip(reference, test):
            confusion[indices[w]][indices[g]] += 1
            max_conf = max(max_conf, confusion[indices[w]][indices[g]])

        #: A list of all values in ``reference`` or ``test``.
        self._values = values
        #: A dictionary mapping values in ``self._values`` to their indices.
        self._indices = indices
        #: The confusion matrix itself (as a list of lists of counts).
        self._confusion = confusion
        #: The greatest count in ``self._confusion`` (used for printing).
        self._max_conf = max_conf
        #: The total number of values in the confusion matrix.
        self._total = len(reference)
        #: The number of correct (on-diagonal) values in the matrix.
        self._correct = sum(confusion[i][i] for i in range(len(values)))

    def __getitem__(self, li_lj_tuple):
        """
        :return: The number of times that value ``li`` was expected and
        value ``lj`` was given.
        :rtype: int
        """
        (li, lj) = li_lj_tuple
        i = self._indices[li]
        j = self._indices[lj]
        return self._confusion[i][j]

    def __repr__(self):
        return f"<ConfusionMatrix: {self._correct}/{self._total} correct>"

    def __str__(self):
        return self.pretty_format()

    def pretty_format(
        self,
        show_percents=False,
        values_in_chart=True,
        truncate=None,
        sort_by_count=False,
    ):
        """
        :return: A multi-line string representation of this confusion matrix.
        :type truncate: int
        :param truncate: If specified, then only show the specified
            number of values.  Any sorting (e.g., sort_by_count)
            will be performed before truncation.
        :param sort_by_count: If true, then sort by the count of each
            label in the reference data.  I.e., labels that occur more
            frequently in the reference label will be towards the left
            edge of the matrix, and labels that occur less frequently
            will be towards the right edge.

        @todo: add marginals?
        """
        confusion = self._confusion

        values = self._values
        if sort_by_count:
            values = sorted(
                values, key=lambda v: -sum(self._confusion[self._indices[v]])
            )

        if truncate:
            values = values[:truncate]

        if values_in_chart:
            value_strings = ["%s" % val for val in values]
        else:
            value_strings = [str(n + 1) for n in range(len(values))]

        # Construct a format string for row values
        valuelen = max(len(val) for val in value_strings)
        value_format = "%" + repr(valuelen) + "s | "
        # Construct a format string for matrix entries
        if show_percents:
            entrylen = 6
            entry_format = "%5.1f%%"
            zerostr = "     ."
        else:
            entrylen = len(repr(self._max_conf))
            entry_format = "%" + repr(entrylen) + "d"
            zerostr = " " * (entrylen - 1) + "."

        # Write the column values.
        s = ""
        for i in range(valuelen):
            s += (" " * valuelen) + " |"
            for val in value_strings:
                if i >= valuelen - len(val):
                    s += val[i - valuelen + len(val)].rjust(entrylen + 1)
                else:
                    s += " " * (entrylen + 1)
            s += " |\n"

        # Write a dividing line
        s += "{}-+-{}+\n".format("-" * valuelen, "-" * ((entrylen + 1) * len(values)))

        # Write the entries.
        for val, li in zip(value_strings, values):
            i = self._indices[li]
            s += value_format % val
            for lj in values:
                j = self._indices[lj]
                if confusion[i][j] == 0:
                    s += zerostr
                elif show_percents:
                    s += entry_format % (100.0 * confusion[i][j] / self._total)
                else:
                    s += entry_format % confusion[i][j]
                if i == j:
                    prevspace = s.rfind(" ")
                    s = s[:prevspace] + "<" + s[prevspace + 1 :] + ">"
                else:
                    s += " "
            s += "|\n"

        # Write a dividing line
        s += "{}-+-{}+\n".format("-" * valuelen, "-" * ((entrylen + 1) * len(values)))

        # Write a key
        s += "(row = reference; col = test)\n"
        if not values_in_chart:
            s += "Value key:\n"
            for i, value in enumerate(values):
                s += "%6d: %s\n" % (i + 1, value)

        return s

    def key(self):
        values = self._values
        str = "Value key:\n"
        indexlen = len(repr(len(values) - 1))
        key_format = "  %" + repr(indexlen) + "d: %s\n"
        for i in range(len(values)):
            str += key_format % (i, values[i])

        return str

    def recall(self, value):
        """Given a value in the confusion matrix, return the recall
        that corresponds to this value. The recall is defined as:

        - *r* = true positive / (true positive + false positive)

        and can loosely be considered the ratio of how often ``value``
        was predicted correctly relative to how often ``value`` was
        the true result.

        :param value: value used in the ConfusionMatrix
        :return: the recall corresponding to ``value``.
        :rtype: float
        """
        # Number of times `value` was correct, and also predicted
        TP = self[value, value]
        # Number of times `value` was correct
        TP_FN = sum(self[value, pred_value] for pred_value in self._values)
        if TP_FN == 0:
            return 0.0
        return TP / TP_FN

    def precision(self, value):
        """Given a value in the confusion matrix, return the precision
        that corresponds to this value. The precision is defined as:

        - *p* = true positive / (true positive + false negative)

        and can loosely be considered the ratio of how often ``value``
        was predicted correctly relative to the number of predictions
        for ``value``.

        :param value: value used in the ConfusionMatrix
        :return: the precision corresponding to ``value``.
        :rtype: float
        """
        # Number of times `value` was correct, and also predicted
        TP = self[value, value]
        # Number of times `value` was predicted
        TP_FP = sum(self[real_value, value] for real_value in self._values)
        if TP_FP == 0:
            return 0.0
        return TP / TP_FP

    def f_measure(self, value, alpha=0.5):
        """
        Given a value used in the confusion matrix, return the f-measure
        that corresponds to this value. The f-measure is the harmonic mean
        of the ``precision`` and ``recall``, weighted by ``alpha``.
        In particular, given the precision *p* and recall *r* defined by:

        - *p* = true positive / (true positive + false negative)
        - *r* = true positive / (true positive + false positive)

        The f-measure is:

        - *1/(alpha/p + (1-alpha)/r)*

        With ``alpha = 0.5``, this reduces to:

        - *2pr / (p + r)*

        :param value: value used in the ConfusionMatrix
        :param alpha: Ratio of the cost of false negative compared to false
            positives. Defaults to 0.5, where the costs are equal.
        :type alpha: float
        :return: the F-measure corresponding to ``value``.
        :rtype: float
        """
        p = self.precision(value)
        r = self.recall(value)
        if p == 0.0 or r == 0.0:
            return 0.0
        return 1.0 / (alpha / p + (1 - alpha) / r)

    def evaluate(self, alpha=0.5, truncate=None, sort_by_count=False):
        """
        Tabulate the **recall**, **precision** and **f-measure**
        for each value in this confusion matrix.

        >>> reference = "DET NN VB DET JJ NN NN IN DET NN".split()
        >>> test = "DET VB VB DET NN NN NN IN DET NN".split()
        >>> cm = ConfusionMatrix(reference, test)
        >>> print(cm.evaluate())
        Tag | Prec.  | Recall | F-measure
        ----+--------+--------+-----------
        DET | 1.0000 | 1.0000 | 1.0000
         IN | 1.0000 | 1.0000 | 1.0000
         JJ | 0.0000 | 0.0000 | 0.0000
         NN | 0.7500 | 0.7500 | 0.7500
         VB | 0.5000 | 1.0000 | 0.6667
        <BLANKLINE>

        :param alpha: Ratio of the cost of false negative compared to false
            positives, as used in the f-measure computation. Defaults to 0.5,
            where the costs are equal.
        :type alpha: float
        :param truncate: If specified, then only show the specified
            number of values. Any sorting (e.g., sort_by_count)
            will be performed before truncation. Defaults to None
        :type truncate: int, optional
        :param sort_by_count: Whether to sort the outputs on frequency
            in the reference label. Defaults to False.
        :type sort_by_count: bool, optional
        :return: A tabulated recall, precision and f-measure string
        :rtype: str
        """
        tags = self._values

        # Apply keyword parameters
        if sort_by_count:
            tags = sorted(tags, key=lambda v: -sum(self._confusion[self._indices[v]]))
        if truncate:
            tags = tags[:truncate]

        tag_column_len = max(max(len(tag) for tag in tags), 3)

        # Construct the header
        s = (
            f"{' ' * (tag_column_len - 3)}Tag | Prec.  | Recall | F-measure\n"
            f"{'-' * tag_column_len}-+--------+--------+-----------\n"
        )

        # Construct the body
        for tag in tags:
            s += (
                f"{tag:>{tag_column_len}} | "
                f"{self.precision(tag):<6.4f} | "
                f"{self.recall(tag):<6.4f} | "
                f"{self.f_measure(tag, alpha=alpha):.4f}\n"
            )

        return s

def demo():
    reference = "DET NN VB DET JJ NN NN IN DET NN".split()
    test = "DET VB VB DET NN NN NN IN DET NN".split()
    print("Reference =", reference)
    print("Test    =", test)
    print("Confusion matrix:")
    print(ConfusionMatrix(reference, test))
    print(ConfusionMatrix(reference, test).pretty_format(sort_by_count=True))

    print(ConfusionMatrix(reference, test).recall("VB"))

if __name__ == "__main__":
    demo()
<|endoftext|>


<commit_before>from flask.blueprints import Blueprint
import logging
from flask_login import login_required, current_user
import flask
from flask.globals import request
from waitlist.data.perm import perm_management
from waitlist.storage.database import CrestFleet, Waitlist, WaitlistGroup,\
    Character, WaitlistEntry, HistoryEntry, HistoryExtInvite
from waitlist.utility.notifications import send_notification
from waitlist.utility.history_utils import create_history_object
from waitlist.utility.fleet import spawn_invite_check, invite, member_info
from flask.json import jsonify
from waitlist.base import db
from datetime import datetime
bp = Blueprint('api_fleet', __name__)
logger = logging.getLogger(__name__)

@bp.route("/<int:fleetID>/", methods=["DELETE"])
@login_required
@perm_management.require(http_exception=401)
def removeFleet(fleetID):
    logger.info("%s deletes crest fleet %i", current_user.username, fleetID)
    db.session.query(CrestFleet).filter(CrestFleet.fleetID == fleetID).delete()
    db.session.commit()
    return flask.jsonify(status_code=200, message="Fleet Deleted")

@bp.route("/fleet/members/", methods=['POST'])
@login_required
@perm_management.require(http_exception=401)
def invite_to_fleet():
    characterID = int(request.form.get('charID'))
    waitlistID = int(request.form.get('waitlistID'))
    groupID = int(request.form.get('groupID'))

    send_notification(characterID, waitlistID)

    waitlist = db.session.query(Waitlist).filter(Waitlist.id == waitlistID).first();
    squad_type = waitlist.name
        # lets check that the given wl exists
    if waitlist is None:
        logger.error("Given waitlist id %s is not valid.", str(waitlistID))
        flask.abort(400)

    group = db.session.query(WaitlistGroup).get(groupID)

    character = db.session.query(Character).get(characterID)
    logger.info("Invited %s by %s into %s", character.eve_name, current_user.username, squad_type)
    if current_user.fleet is None:
        resp = jsonify(status_code=409, message="You are not currently Boss of a Fleet")
        resp.status_code = 409
        return resp
    fleet = current_user.fleet

    # generate a list in which order squads should be preferred in case the main squad is full
    squadIDList = []
    if squad_type == "logi":
        squadIDList = [(fleet.logiWingID, fleet.logiSquadID), (fleet.otherWingID, fleet.otherSquadID), (fleet.sniperWingID, fleet.sniperSquadID), (fleet.dpsWingID, fleet.dpsSquadID)]
    elif squad_type == "dps":
        squadIDList = [(fleet.dpsWingID, fleet.dpsSquadID), (fleet.otherWingID, fleet.otherSquadID), (fleet.sniperWingID, fleet.sniperSquadID), (fleet.logiWingID, fleet.logiSquadID)]
    elif squad_type == "sniper":
        squadIDList = [(fleet.sniperWingID, fleet.sniperSquadID), (fleet.otherWingID, fleet.otherSquadID), (fleet.dpsWingID, fleet.dpsSquadID), (fleet.logiWingID, fleet.logiSquadID)]
    else:
        return Response(flask.jsonify({'message': 'Unknown Squad Type'}), 415)

    # invite over crest and get back the status
    status = invite(characterID, squadIDList)

    resp = flask.jsonify({'status': status['status_code'], 'message': status['text']})
    resp.status_code = status['status_code']

    character = db.session.query(Character).filter(Character.id == characterID).first()
    hEntry = create_history_object(character.get_eve_id(), HistoryEntry.EVENT_COMP_INV_PL, current_user.id)
    hEntry.exref = waitlist.group.groupID

    # create a invite history extension
    # get wl entry for creation time
    wlEntry = db.session.query(WaitlistEntry).filter((WaitlistEntry.waitlist_id == waitlistID) & (WaitlistEntry.user == characterID)).first()
    if wlEntry == None:
        return resp

    db.session.add(hEntry)
    db.session.flush()
    db.session.refresh(hEntry)

    historyExt = HistoryExtInvite()
    historyExt.historyID = hEntry.historyID
    historyExt.waitlistID = waitlistID
    historyExt.timeCreated = wlEntry.creation
    historyExt.timeInvited = datetime.utcnow()
    db.session.add(historyExt)

    db.session.commit()
    logger.info("%s invited %s to fleet from %s.", current_user.username, character.eve_name, waitlist.group.groupName)

    db.session.commit()

    # set a timer for 1min and 6s that checks if the person accepted the invite
    if resp.status_code == 201:
        spawn_invite_check(characterID, groupID, fleet.fleetID)
    return resp

def check_invited(fleetID):
    member_info.get_fleet_members(fleetID, current_user)
<commit_msg>fixed forgotten import<commit_after>from flask.blueprints import Blueprint
import logging
from flask_login import login_required, current_user
import flask
from flask.globals import request
from waitlist.data.perm import perm_management
from waitlist.storage.database import CrestFleet, Waitlist, WaitlistGroup,\
    Character, WaitlistEntry, HistoryEntry, HistoryExtInvite
from waitlist.utility.notifications import send_notification
from waitlist.utility.history_utils import create_history_object
from waitlist.utility.fleet import spawn_invite_check, invite, member_info
from flask.json import jsonify
from waitlist.base import db
from datetime import datetime
from flask.wrappers import Response
bp = Blueprint('api_fleet', __name__)
logger = logging.getLogger(__name__)

@bp.route("/<int:fleetID>/", methods=["DELETE"])
@login_required
@perm_management.require(http_exception=401)
def removeFleet(fleetID):
    logger.info("%s deletes crest fleet %i", current_user.username, fleetID)
    db.session.query(CrestFleet).filter(CrestFleet.fleetID == fleetID).delete()
    db.session.commit()
    return flask.jsonify(status_code=200, message="Fleet Deleted")

@bp.route("/fleet/members/", methods=['POST'])
@login_required
@perm_management.require(http_exception=401)
def invite_to_fleet():
    characterID = int(request.form.get('charID'))
    waitlistID = int(request.form.get('waitlistID'))
    groupID = int(request.form.get('groupID'))

    send_notification(characterID, waitlistID)

    waitlist = db.session.query(Waitlist).filter(Waitlist.id == waitlistID).first();
    squad_type = waitlist.name
        # lets check that the given wl exists
    if waitlist is None:
        logger.error("Given waitlist id %s is not valid.", str(waitlistID))
        flask.abort(400)

    group = db.session.query(WaitlistGroup).get(groupID)

    character = db.session.query(Character).get(characterID)
    logger.info("Invited %s by %s into %s", character.eve_name, current_user.username, squad_type)
    if current_user.fleet is None:
        resp = jsonify(status_code=409, message="You are not currently Boss of a Fleet")
        resp.status_code = 409
        return resp
    fleet = current_user.fleet

    # generate a list in which order squads should be preferred in case the main squad is full
    squadIDList = []
    if squad_type == "logi":
        squadIDList = [(fleet.logiWingID, fleet.logiSquadID), (fleet.otherWingID, fleet.otherSquadID), (fleet.sniperWingID, fleet.sniperSquadID), (fleet.dpsWingID, fleet.dpsSquadID)]
    elif squad_type == "dps":
        squadIDList = [(fleet.dpsWingID, fleet.dpsSquadID), (fleet.otherWingID, fleet.otherSquadID), (fleet.sniperWingID, fleet.sniperSquadID), (fleet.logiWingID, fleet.logiSquadID)]
    elif squad_type == "sniper":
        squadIDList = [(fleet.sniperWingID, fleet.sniperSquadID), (fleet.otherWingID, fleet.otherSquadID), (fleet.dpsWingID, fleet.dpsSquadID), (fleet.logiWingID, fleet.logiSquadID)]
    else:
        return Response(flask.jsonify({'message': 'Unknown Squad Type'}), 415)

    # invite over crest and get back the status
    status = invite(characterID, squadIDList)

    resp = flask.jsonify({'status': status['status_code'], 'message': status['text']})
    resp.status_code = status['status_code']

    character = db.session.query(Character).filter(Character.id == characterID).first()
    hEntry = create_history_object(character.get_eve_id(), HistoryEntry.EVENT_COMP_INV_PL, current_user.id)
    hEntry.exref = waitlist.group.groupID

    # create a invite history extension
    # get wl entry for creation time
    wlEntry = db.session.query(WaitlistEntry).filter((WaitlistEntry.waitlist_id == waitlistID) & (WaitlistEntry.user == characterID)).first()
    if wlEntry == None:
        return resp

    db.session.add(hEntry)
    db.session.flush()
    db.session.refresh(hEntry)

    historyExt = HistoryExtInvite()
    historyExt.historyID = hEntry.historyID
    historyExt.waitlistID = waitlistID
    historyExt.timeCreated = wlEntry.creation
    historyExt.timeInvited = datetime.utcnow()
    db.session.add(historyExt)

    db.session.commit()
    logger.info("%s invited %s to fleet from %s.", current_user.username, character.eve_name, waitlist.group.groupName)

    db.session.commit()

    # set a timer for 1min and 6s that checks if the person accepted the invite
    if resp.status_code == 201:
        spawn_invite_check(characterID, groupID, fleet.fleetID)
    return resp

def check_invited(fleetID):
    member_info.get_fleet_members(fleetID, current_user)
<|endoftext|>


<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""Command-line flag library.

Wraps gflags.

Package-level global flags are defined here, the rest are defined
where they're used.

"""

import getopt
import os
import socket
import string
import sys

import gflags

class FlagValues(gflags.FlagValues):
    """Extension of gflags.FlagValues that allows undefined and runtime flags.

    Unknown flags will be ignored when parsing the command line, but the
    command line will be kept so that it can be replayed if new flags are
    defined after the initial parsing.

    """

    def __init__(self, extra_context=None):
        gflags.FlagValues.__init__(self)
        self.__dict__['__dirty'] = []
        self.__dict__['__was_already_parsed'] = False
        self.__dict__['__stored_argv'] = []
        self.__dict__['__extra_context'] = extra_context

    def __call__(self, argv):
        # We're doing some hacky stuff here so that we don't have to copy
        # out all the code of the original verbatim and then tweak a few lines.
        # We're hijacking the output of getopt so we can still return the
        # leftover args at the end
        sneaky_unparsed_args = {"value": None}
        original_argv = list(argv)

        if self.IsGnuGetOpt():
            orig_getopt = getattr(getopt, 'gnu_getopt')
            orig_name = 'gnu_getopt'
        else:
            orig_getopt = getattr(getopt, 'getopt')
            orig_name = 'getopt'

        def _sneaky(*args, **kw):
            optlist, unparsed_args = orig_getopt(*args, **kw)
            sneaky_unparsed_args['value'] = unparsed_args
            return optlist, unparsed_args

        try:
            setattr(getopt, orig_name, _sneaky)
            args = gflags.FlagValues.__call__(self, argv)
        except gflags.UnrecognizedFlagError:
            # Undefined args were found, for now we don't care so just
            # act like everything went well
            # (these three lines are copied pretty much verbatim from the end
            # of the __call__ function we are wrapping)
            unparsed_args = sneaky_unparsed_args['value']
            if unparsed_args:
                if self.IsGnuGetOpt():
                    args = argv[:1] + unparsed_args
                else:
                    args = argv[:1] + original_argv[-len(unparsed_args):]
            else:
                args = argv[:1]
        finally:
            setattr(getopt, orig_name, orig_getopt)

        # Store the arguments for later, we'll need them for new flags
        # added at runtime
        self.__dict__['__stored_argv'] = original_argv
        self.__dict__['__was_already_parsed'] = True
        self.ClearDirty()
        return args

    def Reset(self):
        gflags.FlagValues.Reset(self)
        self.__dict__['__dirty'] = []
        self.__dict__['__was_already_parsed'] = False
        self.__dict__['__stored_argv'] = []

    def SetDirty(self, name):
        """Mark a flag as dirty so that accessing it will case a reparse."""
        self.__dict__['__dirty'].append(name)

    def IsDirty(self, name):
        return name in self.__dict__['__dirty']

    def ClearDirty(self):
        self.__dict__['__is_dirty'] = []

    def WasAlreadyParsed(self):
        return self.__dict__['__was_already_parsed']

    def ParseNewFlags(self):
        if '__stored_argv' not in self.__dict__:
            return
        new_flags = FlagValues(self)
        for k in self.__dict__['__dirty']:
            new_flags[k] = gflags.FlagValues.__getitem__(self, k)

        new_flags(self.__dict__['__stored_argv'])
        for k in self.__dict__['__dirty']:
            setattr(self, k, getattr(new_flags, k))
        self.ClearDirty()

    def __setitem__(self, name, flag):
        gflags.FlagValues.__setitem__(self, name, flag)
        if self.WasAlreadyParsed():
            self.SetDirty(name)

    def __getitem__(self, name):
        if self.IsDirty(name):
            self.ParseNewFlags()
        return gflags.FlagValues.__getitem__(self, name)

    def __getattr__(self, name):
        if self.IsDirty(name):
            self.ParseNewFlags()
        val = gflags.FlagValues.__getattr__(self, name)
        if type(val) is str:
            tmpl = string.Template(val)
            context = [self, self.__dict__['__extra_context']]
            return tmpl.substitute(StrWrapper(context))
        return val

class StrWrapper(object):
    """Wrapper around FlagValues objects.

    Wraps FlagValues objects for string.Template so that we're
    sure to return strings.

    """
    def __init__(self, context_objs):
        self.context_objs = context_objs

    def __getitem__(self, name):
        for context in self.context_objs:
            val = getattr(context, name, False)
            if val:
                return str(val)
        raise KeyError(name)

# Copied from gflags with small mods to get the naming correct.
# Originally gflags checks for the first module that is not gflags that is
# in the call chain, we want to check for the first module that is not gflags
# and not this module.
def _GetCallingModule():
    """Returns the name of the module that's calling into this module.

    We generally use this function to get the name of the module calling a
    DEFINE_foo... function.

    """
    # Walk down the stack to find the first globals dict that's not ours.
    for depth in range(1, sys.getrecursionlimit()):
        if not sys._getframe(depth).f_globals is globals():
            module_name = __GetModuleName(sys._getframe(depth).f_globals)
            if module_name == 'gflags':
                continue
            if module_name is not None:
                return module_name
    raise AssertionError("No module was found")

# Copied from gflags because it is a private function
def __GetModuleName(globals_dict):
    """Given a globals dict, returns the name of the module that defines it.

    Args:
    globals_dict: A dictionary that should correspond to an environment
      providing the values of the globals.

    Returns:
    A string (the name of the module) or None (if the module could not
    be identified.

    """
    for name, module in sys.modules.iteritems():
        if getattr(module, '__dict__', None) is globals_dict:
            if name == '__main__':
                return sys.argv[0]
            return name
    return None

def _wrapper(func):
    def _wrapped(*args, **kw):
        kw.setdefault('flag_values', FLAGS)
        func(*args, **kw)
    _wrapped.func_name = func.func_name
    return _wrapped

FLAGS = FlagValues()
gflags.FLAGS = FLAGS
gflags._GetCallingModule = _GetCallingModule

DEFINE = _wrapper(gflags.DEFINE)
DEFINE_string = _wrapper(gflags.DEFINE_string)
DEFINE_integer = _wrapper(gflags.DEFINE_integer)
DEFINE_bool = _wrapper(gflags.DEFINE_bool)
DEFINE_boolean = _wrapper(gflags.DEFINE_boolean)
DEFINE_float = _wrapper(gflags.DEFINE_float)
DEFINE_enum = _wrapper(gflags.DEFINE_enum)
DEFINE_list = _wrapper(gflags.DEFINE_list)
DEFINE_spaceseplist = _wrapper(gflags.DEFINE_spaceseplist)
DEFINE_multistring = _wrapper(gflags.DEFINE_multistring)
DEFINE_multi_int = _wrapper(gflags.DEFINE_multi_int)
DEFINE_flag = _wrapper(gflags.DEFINE_flag)
HelpFlag = gflags.HelpFlag
HelpshortFlag = gflags.HelpshortFlag
HelpXMLFlag = gflags.HelpXMLFlag

def DECLARE(name, module_string, flag_values=FLAGS):
    if module_string not in sys.modules:
        __import__(module_string, globals(), locals())
    if name not in flag_values:
        raise gflags.UnrecognizedFlag(
                "%s not defined by %s" % (name, module_string))

def _get_my_ip():
    """Returns the actual ip of the local machine."""
    try:
        csock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        csock.connect(('8.8.8.8', 80))
        (addr, port) = csock.getsockname()
        csock.close()
        return addr
    except socket.error as ex:
        return "127.0.0.1"

# __GLOBAL FLAGS ONLY__
# Define any app-specific flags in their own files, docs at:
# http://code.google.com/p/python-gflags/source/browse/trunk/gflags.py#a9
DEFINE_string('my_ip', _get_my_ip(), 'host ip address')
DEFINE_list('region_list',
            [],
            'list of region=fqdn pairs separated by commas')
DEFINE_string('connection_type', 'libvirt', 'libvirt, xenapi or fake')
DEFINE_string('aws_access_key_id', 'admin', 'AWS Access ID')
DEFINE_string('aws_secret_access_key', 'admin', 'AWS Access Key')
DEFINE_integer('glance_port', 9292, 'glance port')
DEFINE_string('glance_host', '$my_ip', 'glance host')
DEFINE_integer('s3_port', 3333, 's3 port')
DEFINE_string('s3_host', '$my_ip', 's3 host (for infrastructure)')
DEFINE_string('s3_dmz', '$my_ip', 's3 dmz ip (for instances)')
DEFINE_string('compute_topic', 'compute', 'the topic compute nodes listen on')
DEFINE_string('console_topic', 'console',
              'the topic console proxy nodes listen on')
DEFINE_string('scheduler_topic', 'scheduler',
              'the topic scheduler nodes listen on')
DEFINE_string('volume_topic', 'volume', 'the topic volume nodes listen on')
DEFINE_string('network_topic', 'network', 'the topic network nodes listen on')
DEFINE_string('ajax_console_proxy_topic', 'ajax_proxy',
              'the topic ajax proxy nodes listen on')
DEFINE_string('ajax_console_proxy_url',
              'http://127.0.0.1:8000',
              'location of ajax console proxy, \
               in the form "http://127.0.0.1:8000"')
DEFINE_string('ajax_console_proxy_port',
               8000, 'port that ajax_console_proxy binds')
DEFINE_bool('verbose', False, 'show debug output')
DEFINE_boolean('fake_rabbit', False, 'use a fake rabbit')
DEFINE_bool('fake_network', False,
            'should we use fake network devices and addresses')
DEFINE_string('rabbit_host', 'localhost', 'rabbit host')
DEFINE_integer('rabbit_port', 5672, 'rabbit port')
DEFINE_string('rabbit_userid', 'guest', 'rabbit userid')
DEFINE_string('rabbit_password', 'guest', 'rabbit password')
DEFINE_string('rabbit_virtual_host', '/', 'rabbit virtual host')
DEFINE_integer('rabbit_retry_interval', 10, 'rabbit connection retry interval')
DEFINE_integer('rabbit_max_retries', 12, 'rabbit connection attempts')
DEFINE_string('control_exchange', 'nova', 'the main exchange to connect to')
DEFINE_string('ec2_host', '$my_ip', 'ip of api server')
DEFINE_string('ec2_dmz_host', '$my_ip', 'internal ip of api server')
DEFINE_integer('ec2_port', 8773, 'cloud controller port')
DEFINE_string('ec2_scheme', 'http', 'prefix for ec2')
DEFINE_string('ec2_path', '/services/Cloud', 'suffix for ec2')
DEFINE_string('osapi_extensions_path', '/var/lib/nova/extensions',
               'default directory for nova extensions')
DEFINE_string('osapi_host', '$my_ip', 'ip of api server')
DEFINE_string('osapi_scheme', 'http', 'prefix for openstack')
DEFINE_integer('osapi_port', 8774, 'OpenStack API port')
DEFINE_string('osapi_path', '/v1.0/', 'suffix for openstack')
DEFINE_integer('osapi_max_limit', 1000,
               'max number of items returned in a collection response')

DEFINE_string('default_project', 'openstack', 'default project for openstack')
DEFINE_string('default_image', 'ami-11111',
              'default image to use, testing only')
DEFINE_string('default_instance_type', 'm1.small',
              'default instance type to use, testing only')
DEFINE_string('null_kernel', 'nokernel',
              'kernel image that indicates not to use a kernel,'
              ' but to use a raw disk image instead')

DEFINE_integer('vpn_image_id', 0, 'integer id for cloudpipe vpn server')
DEFINE_string('vpn_key_suffix',
              '-vpn',
              'Suffix to add to project name for vpn key and secgroups')

DEFINE_integer('auth_token_ttl', 3600, 'Seconds for auth tokens to linger')

DEFINE_string('state_path', os.path.join(os.path.dirname(__file__), '../'),
              "Top-level directory for maintaining nova's state")
DEFINE_string('lock_path', os.path.join(os.path.dirname(__file__), '../'),
              'Directory for lock files')
DEFINE_string('logdir', None, 'output to a per-service log file in named '
                              'directory')

DEFINE_string('sqlite_db', 'nova.sqlite', 'file name for sqlite')
DEFINE_string('sql_connection',
              'sqlite:///$state_path/$sqlite_db',
              'connection string for sql database')
DEFINE_integer('sql_idle_timeout',
              3600,
              'timeout for idle sql database connections')
DEFINE_integer('sql_max_retries', 12, 'sql connection attempts')
DEFINE_integer('sql_retry_interval', 10, 'sql connection retry interval')

DEFINE_string('compute_manager', 'nova.compute.manager.ComputeManager',
              'Manager for compute')
DEFINE_string('console_manager', 'nova.console.manager.ConsoleProxyManager',
              'Manager for console proxy')
DEFINE_string('network_manager', 'nova.network.manager.VlanManager',
              'Manager for network')
DEFINE_string('volume_manager', 'nova.volume.manager.VolumeManager',
              'Manager for volume')
DEFINE_string('scheduler_manager', 'nova.scheduler.manager.SchedulerManager',
              'Manager for scheduler')

# The service to use for image search and retrieval
DEFINE_string('image_service', 'nova.image.local.LocalImageService',
              'The service to use for retrieving and searching for images.')

DEFINE_string('host', socket.gethostname(),
              'name of this node')

DEFINE_string('node_availability_zone', 'nova',
              'availability zone of this node')

DEFINE_list('memcached_servers', None,
            'Memcached servers or None for in process cache.')

DEFINE_string('zone_name', 'nova', 'name of this zone')
DEFINE_list('zone_capabilities',
                ['hypervisor=xenserver;kvm', 'os=linux;windows'],
                 'Key/Multi-value list representng capabilities of this zone')
<commit_msg>get integrated server_tests passing<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""Command-line flag library.

Wraps gflags.

Package-level global flags are defined here, the rest are defined
where they're used.

"""

import getopt
import os
import socket
import string
import sys

import gflags

class FlagValues(gflags.FlagValues):
    """Extension of gflags.FlagValues that allows undefined and runtime flags.

    Unknown flags will be ignored when parsing the command line, but the
    command line will be kept so that it can be replayed if new flags are
    defined after the initial parsing.

    """

    def __init__(self, extra_context=None):
        gflags.FlagValues.__init__(self)
        self.__dict__['__dirty'] = []
        self.__dict__['__was_already_parsed'] = False
        self.__dict__['__stored_argv'] = []
        self.__dict__['__extra_context'] = extra_context

    def __call__(self, argv):
        # We're doing some hacky stuff here so that we don't have to copy
        # out all the code of the original verbatim and then tweak a few lines.
        # We're hijacking the output of getopt so we can still return the
        # leftover args at the end
        sneaky_unparsed_args = {"value": None}
        original_argv = list(argv)

        if self.IsGnuGetOpt():
            orig_getopt = getattr(getopt, 'gnu_getopt')
            orig_name = 'gnu_getopt'
        else:
            orig_getopt = getattr(getopt, 'getopt')
            orig_name = 'getopt'

        def _sneaky(*args, **kw):
            optlist, unparsed_args = orig_getopt(*args, **kw)
            sneaky_unparsed_args['value'] = unparsed_args
            return optlist, unparsed_args

        try:
            setattr(getopt, orig_name, _sneaky)
            args = gflags.FlagValues.__call__(self, argv)
        except gflags.UnrecognizedFlagError:
            # Undefined args were found, for now we don't care so just
            # act like everything went well
            # (these three lines are copied pretty much verbatim from the end
            # of the __call__ function we are wrapping)
            unparsed_args = sneaky_unparsed_args['value']
            if unparsed_args:
                if self.IsGnuGetOpt():
                    args = argv[:1] + unparsed_args
                else:
                    args = argv[:1] + original_argv[-len(unparsed_args):]
            else:
                args = argv[:1]
        finally:
            setattr(getopt, orig_name, orig_getopt)

        # Store the arguments for later, we'll need them for new flags
        # added at runtime
        self.__dict__['__stored_argv'] = original_argv
        self.__dict__['__was_already_parsed'] = True
        self.ClearDirty()
        return args

    def Reset(self):
        gflags.FlagValues.Reset(self)
        self.__dict__['__dirty'] = []
        self.__dict__['__was_already_parsed'] = False
        self.__dict__['__stored_argv'] = []

    def SetDirty(self, name):
        """Mark a flag as dirty so that accessing it will case a reparse."""
        self.__dict__['__dirty'].append(name)

    def IsDirty(self, name):
        return name in self.__dict__['__dirty']

    def ClearDirty(self):
        self.__dict__['__is_dirty'] = []

    def WasAlreadyParsed(self):
        return self.__dict__['__was_already_parsed']

    def ParseNewFlags(self):
        if '__stored_argv' not in self.__dict__:
            return
        new_flags = FlagValues(self)
        for k in self.__dict__['__dirty']:
            new_flags[k] = gflags.FlagValues.__getitem__(self, k)

        new_flags(self.__dict__['__stored_argv'])
        for k in self.__dict__['__dirty']:
            setattr(self, k, getattr(new_flags, k))
        self.ClearDirty()

    def __setitem__(self, name, flag):
        gflags.FlagValues.__setitem__(self, name, flag)
        if self.WasAlreadyParsed():
            self.SetDirty(name)

    def __getitem__(self, name):
        if self.IsDirty(name):
            self.ParseNewFlags()
        return gflags.FlagValues.__getitem__(self, name)

    def __getattr__(self, name):
        if self.IsDirty(name):
            self.ParseNewFlags()
        val = gflags.FlagValues.__getattr__(self, name)
        if type(val) is str:
            tmpl = string.Template(val)
            context = [self, self.__dict__['__extra_context']]
            return tmpl.substitute(StrWrapper(context))
        return val

class StrWrapper(object):
    """Wrapper around FlagValues objects.

    Wraps FlagValues objects for string.Template so that we're
    sure to return strings.

    """
    def __init__(self, context_objs):
        self.context_objs = context_objs

    def __getitem__(self, name):
        for context in self.context_objs:
            val = getattr(context, name, False)
            if val:
                return str(val)
        raise KeyError(name)

# Copied from gflags with small mods to get the naming correct.
# Originally gflags checks for the first module that is not gflags that is
# in the call chain, we want to check for the first module that is not gflags
# and not this module.
def _GetCallingModule():
    """Returns the name of the module that's calling into this module.

    We generally use this function to get the name of the module calling a
    DEFINE_foo... function.

    """
    # Walk down the stack to find the first globals dict that's not ours.
    for depth in range(1, sys.getrecursionlimit()):
        if not sys._getframe(depth).f_globals is globals():
            module_name = __GetModuleName(sys._getframe(depth).f_globals)
            if module_name == 'gflags':
                continue
            if module_name is not None:
                return module_name
    raise AssertionError("No module was found")

# Copied from gflags because it is a private function
def __GetModuleName(globals_dict):
    """Given a globals dict, returns the name of the module that defines it.

    Args:
    globals_dict: A dictionary that should correspond to an environment
      providing the values of the globals.

    Returns:
    A string (the name of the module) or None (if the module could not
    be identified.

    """
    for name, module in sys.modules.iteritems():
        if getattr(module, '__dict__', None) is globals_dict:
            if name == '__main__':
                return sys.argv[0]
            return name
    return None

def _wrapper(func):
    def _wrapped(*args, **kw):
        kw.setdefault('flag_values', FLAGS)
        func(*args, **kw)
    _wrapped.func_name = func.func_name
    return _wrapped

FLAGS = FlagValues()
gflags.FLAGS = FLAGS
gflags._GetCallingModule = _GetCallingModule

DEFINE = _wrapper(gflags.DEFINE)
DEFINE_string = _wrapper(gflags.DEFINE_string)
DEFINE_integer = _wrapper(gflags.DEFINE_integer)
DEFINE_bool = _wrapper(gflags.DEFINE_bool)
DEFINE_boolean = _wrapper(gflags.DEFINE_boolean)
DEFINE_float = _wrapper(gflags.DEFINE_float)
DEFINE_enum = _wrapper(gflags.DEFINE_enum)
DEFINE_list = _wrapper(gflags.DEFINE_list)
DEFINE_spaceseplist = _wrapper(gflags.DEFINE_spaceseplist)
DEFINE_multistring = _wrapper(gflags.DEFINE_multistring)
DEFINE_multi_int = _wrapper(gflags.DEFINE_multi_int)
DEFINE_flag = _wrapper(gflags.DEFINE_flag)
HelpFlag = gflags.HelpFlag
HelpshortFlag = gflags.HelpshortFlag
HelpXMLFlag = gflags.HelpXMLFlag

def DECLARE(name, module_string, flag_values=FLAGS):
    if module_string not in sys.modules:
        __import__(module_string, globals(), locals())
    if name not in flag_values:
        raise gflags.UnrecognizedFlag(
                "%s not defined by %s" % (name, module_string))

def _get_my_ip():
    """Returns the actual ip of the local machine."""
    try:
        csock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        csock.connect(('8.8.8.8', 80))
        (addr, port) = csock.getsockname()
        csock.close()
        return addr
    except socket.error as ex:
        return "127.0.0.1"

# __GLOBAL FLAGS ONLY__
# Define any app-specific flags in their own files, docs at:
# http://code.google.com/p/python-gflags/source/browse/trunk/gflags.py#a9
DEFINE_string('my_ip', _get_my_ip(), 'host ip address')
DEFINE_list('region_list',
            [],
            'list of region=fqdn pairs separated by commas')
DEFINE_string('connection_type', 'libvirt', 'libvirt, xenapi or fake')
DEFINE_string('aws_access_key_id', 'admin', 'AWS Access ID')
DEFINE_string('aws_secret_access_key', 'admin', 'AWS Access Key')
DEFINE_integer('glance_port', 9292, 'glance port')
DEFINE_string('glance_host', '$my_ip', 'glance host')
DEFINE_integer('s3_port', 3333, 's3 port')
DEFINE_string('s3_host', '$my_ip', 's3 host (for infrastructure)')
DEFINE_string('s3_dmz', '$my_ip', 's3 dmz ip (for instances)')
DEFINE_string('compute_topic', 'compute', 'the topic compute nodes listen on')
DEFINE_string('console_topic', 'console',
              'the topic console proxy nodes listen on')
DEFINE_string('scheduler_topic', 'scheduler',
              'the topic scheduler nodes listen on')
DEFINE_string('volume_topic', 'volume', 'the topic volume nodes listen on')
DEFINE_string('network_topic', 'network', 'the topic network nodes listen on')
DEFINE_string('ajax_console_proxy_topic', 'ajax_proxy',
              'the topic ajax proxy nodes listen on')
DEFINE_string('ajax_console_proxy_url',
              'http://127.0.0.1:8000',
              'location of ajax console proxy, \
               in the form "http://127.0.0.1:8000"')
DEFINE_string('ajax_console_proxy_port',
               8000, 'port that ajax_console_proxy binds')
DEFINE_bool('verbose', False, 'show debug output')
DEFINE_boolean('fake_rabbit', False, 'use a fake rabbit')
DEFINE_bool('fake_network', False,
            'should we use fake network devices and addresses')
DEFINE_string('rabbit_host', 'localhost', 'rabbit host')
DEFINE_integer('rabbit_port', 5672, 'rabbit port')
DEFINE_string('rabbit_userid', 'guest', 'rabbit userid')
DEFINE_string('rabbit_password', 'guest', 'rabbit password')
DEFINE_string('rabbit_virtual_host', '/', 'rabbit virtual host')
DEFINE_integer('rabbit_retry_interval', 10, 'rabbit connection retry interval')
DEFINE_integer('rabbit_max_retries', 12, 'rabbit connection attempts')
DEFINE_string('control_exchange', 'nova', 'the main exchange to connect to')
DEFINE_string('ec2_host', '$my_ip', 'ip of api server')
DEFINE_string('ec2_dmz_host', '$my_ip', 'internal ip of api server')
DEFINE_integer('ec2_port', 8773, 'cloud controller port')
DEFINE_string('ec2_scheme', 'http', 'prefix for ec2')
DEFINE_string('ec2_path', '/services/Cloud', 'suffix for ec2')
DEFINE_string('osapi_extensions_path', '/var/lib/nova/extensions',
               'default directory for nova extensions')
DEFINE_string('osapi_host', '$my_ip', 'ip of api server')
DEFINE_string('osapi_scheme', 'http', 'prefix for openstack')
DEFINE_integer('osapi_port', 8774, 'OpenStack API port')
DEFINE_string('osapi_path', '/v1.0/', 'suffix for openstack')
DEFINE_integer('osapi_max_limit', 1000,
               'max number of items returned in a collection response')

DEFINE_string('default_project', 'openstack', 'default project for openstack')
DEFINE_string('default_image', 'ami-11111',
              'default image to use, testing only')
DEFINE_string('default_instance_type', 'm1.small',
              'default instance type to use, testing only')
DEFINE_string('null_kernel', 'nokernel',
              'kernel image that indicates not to use a kernel,'
              ' but to use a raw disk image instead')

DEFINE_integer('vpn_image_id', 0, 'integer id for cloudpipe vpn server')
DEFINE_string('vpn_key_suffix',
              '-vpn',
              'Suffix to add to project name for vpn key and secgroups')

DEFINE_integer('auth_token_ttl', 3600, 'Seconds for auth tokens to linger')

DEFINE_string('state_path', os.path.join(os.path.dirname(__file__), '../'),
              "Top-level directory for maintaining nova's state")
DEFINE_string('lock_path', os.path.join(os.path.dirname(__file__), '../'),
              'Directory for lock files')
DEFINE_string('logdir', None, 'output to a per-service log file in named '
                              'directory')

DEFINE_string('sqlite_db', 'nova.sqlite', 'file name for sqlite')
DEFINE_string('sql_connection',
              'sqlite:///$state_path/$sqlite_db',
              'connection string for sql database')
DEFINE_integer('sql_idle_timeout',
              3600,
              'timeout for idle sql database connections')
DEFINE_integer('sql_max_retries', 12, 'sql connection attempts')
DEFINE_integer('sql_retry_interval', 10, 'sql connection retry interval')

DEFINE_string('compute_manager', 'nova.compute.manager.ComputeManager',
              'Manager for compute')
DEFINE_string('console_manager', 'nova.console.manager.ConsoleProxyManager',
              'Manager for console proxy')
DEFINE_string('network_manager', 'nova.network.manager.VlanManager',
              'Manager for network')
DEFINE_string('volume_manager', 'nova.volume.manager.VolumeManager',
              'Manager for volume')
DEFINE_string('scheduler_manager', 'nova.scheduler.manager.SchedulerManager',
              'Manager for scheduler')

# The service to use for image search and retrieval
DEFINE_string('image_service', 'nova.image.local.LocalImageService',
              'The service to use for retrieving and searching for images.')
DEFINE_string('glance_image_service', 'nova.image.local.LocalImageService',
              'The service to use for retrieving and searching for ' +
              'glance images.')

DEFINE_string('host', socket.gethostname(),
              'name of this node')

DEFINE_string('node_availability_zone', 'nova',
              'availability zone of this node')

DEFINE_list('memcached_servers', None,
            'Memcached servers or None for in process cache.')

DEFINE_string('zone_name', 'nova', 'name of this zone')
DEFINE_list('zone_capabilities',
                ['hypervisor=xenserver;kvm', 'os=linux;windows'],
                 'Key/Multi-value list representng capabilities of this zone')
<|endoftext|>


<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
System-level utilities and helper functions.
"""

import datetime
import inspect
import os
import random
import subprocess
import socket
import struct
import sys
import time
from xml.sax import saxutils
import re
import netaddr

from eventlet import event
from eventlet import greenthread

from nova import exception
from nova.exception import ProcessExecutionError
from nova import log as logging

LOG = logging.getLogger("nova.utils")
TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ"

def import_class(import_str):
    """Returns a class from a string including module and class"""
    mod_str, _sep, class_str = import_str.rpartition('.')
    try:
        __import__(mod_str)
        return getattr(sys.modules[mod_str], class_str)
    except (ImportError, ValueError, AttributeError), exc:
        logging.debug(_('Inner Exception: %s'), exc)
        raise exception.NotFound(_('Class %s cannot be found') % class_str)

def import_object(import_str):
    """Returns an object including a module or module and class"""
    try:
        __import__(import_str)
        return sys.modules[import_str]
    except ImportError:
        cls = import_class(import_str)
        return cls()

def vpn_ping(address, port, timeout=0.05, session_id=None):
    """Sends a vpn negotiation packet and returns the server session.

    Returns False on a failure. Basic packet structure is below.

    Client packet (14 bytes)::
     0 1      8 9  13
    +-+--------+-----+
    |x| cli_id |?????|
    +-+--------+-----+
    x = packet identifier 0x38
    cli_id = 64 bit identifier
    ? = unknown, probably flags/padding

    Server packet (26 bytes)::
     0 1      8 9  13 14    21 2225
    +-+--------+-----+--------+----+
    |x| srv_id |?????| cli_id |????|
    +-+--------+-----+--------+----+
    x = packet identifier 0x40
    cli_id = 64 bit identifier
    ? = unknown, probably flags/padding
    bit 9 was 1 and the rest were 0 in testing
    """
    if session_id is None:
        session_id = random.randint(0, 0xffffffffffffffff)
    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    data = struct.pack("!BQxxxxxx", 0x38, session_id)
    sock.sendto(data, (address, port))
    sock.settimeout(timeout)
    try:
        received = sock.recv(2048)
    except socket.timeout:
        return False
    finally:
        sock.close()
    fmt = "!BQxxxxxQxxxx"
    if len(received) != struct.calcsize(fmt):
        print struct.calcsize(fmt)
        return False
    (identifier, server_sess, client_sess) = struct.unpack(fmt, received)
    if identifier == 0x40 and client_sess == session_id:
        return server_sess

def fetchfile(url, target):
    LOG.debug(_("Fetching %s") % url)
#    c = pycurl.Curl()
#    fp = open(target, "wb")
#    c.setopt(c.URL, url)
#    c.setopt(c.WRITEDATA, fp)
#    c.perform()
#    c.close()
#    fp.close()
    execute("curl --fail %s -o %s" % (url, target))

def execute(cmd, process_input=None, addl_env=None, check_exit_code=True):
    LOG.debug(_("Running cmd (subprocess): %s"), cmd)
    env = os.environ.copy()
    if addl_env:
        env.update(addl_env)
    obj = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE,
        stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
    result = None
    if process_input != None:
        result = obj.communicate(process_input)
    else:
        result = obj.communicate()
    obj.stdin.close()
    if obj.returncode:
        LOG.debug(_("Result was %s") % (obj.returncode))
        if check_exit_code and obj.returncode != 0:
            (stdout, stderr) = result
            raise ProcessExecutionError(exit_code=obj.returncode,
                                        stdout=stdout,
                                        stderr=stderr,
                                        cmd=cmd)
    # NOTE(termie): this appears to be necessary to let the subprocess call
    #               clean something up in between calls, without it two
    #               execute calls in a row hangs the second one
    greenthread.sleep(0)
    return result

def abspath(s):
    return os.path.join(os.path.dirname(__file__), s)

def novadir():
    import nova
    return os.path.abspath(nova.__file__).split('nova/__init__.pyc')[0]

def default_flagfile(filename='nova.conf'):
    for arg in sys.argv:
        if arg.find('flagfile') != -1:
            break
    else:
        if not os.path.isabs(filename):
            # turn relative filename into an absolute path
            script_dir = os.path.dirname(inspect.stack()[-1][1])
            filename = os.path.abspath(os.path.join(script_dir, filename))
        if os.path.exists(filename):
            flagfile = ['--flagfile=%s' % filename]
            sys.argv = sys.argv[:1] + flagfile + sys.argv[1:]

def debug(arg):
    LOG.debug(_('debug in callback: %s'), arg)
    return arg

def runthis(prompt, cmd, check_exit_code=True):
    LOG.debug(_("Running %s"), (cmd))
    rv, err = execute(cmd, check_exit_code=check_exit_code)

def generate_uid(topic, size=8):
    characters = '01234567890abcdefghijklmnopqrstuvwxyz'
    choices = [random.choice(characters) for x in xrange(size)]
    return '%s-%s' % (topic, ''.join(choices))

def generate_mac():
    mac = [0x02, 0x16, 0x3e,
           random.randint(0x00, 0x7f),
           random.randint(0x00, 0xff),
           random.randint(0x00, 0xff)]
    return ':'.join(map(lambda x: "%02x" % x, mac))

def last_octet(address):
    return int(address.split(".")[-1])

def get_my_ip():
    """Returns the actual ip of the local machine."""
    try:
        csock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        csock.connect(('8.8.8.8', 80))
        (addr, port) = csock.getsockname()
        csock.close()
        return addr
    except socket.gaierror as ex:
        logging.warn(_("Couldn't get IP, using 127.0.0.1 %s"), ex)
        return "127.0.0.1"

def  get_my_linklocal(interface):
    try:
        if_str = execute("ip -f inet6 -o addr show %s" % interface)
        condition = "\s+inet6\s+([0-9a-f:]+/\d+)\s+scope\s+link"
        links = [re.search(condition, x) for x in if_str[0].split('\n')]
        address = [w.group(1) for w in links if w is not None]
        if address[0] is not None:
            return address[0]
        else:
            return 'fe00::'
    except IndexError as ex:
        logging.warn(_("Couldn't get Link Local IP of %s :%s"), interface, ex)
    except ProcessExecutionError as ex:
        logging.warn(_("Couldn't get Link Local IP of %s :%s"), interface, ex)
    except:
        return 'fe00::'

def to_global_ipv6(prefix, mac):
    mac64 = netaddr.EUI(mac).eui64().words
    int_addr = int(''.join(['%02x' % i for i in mac64]), 16)
    mac64_addr = netaddr.IPAddress(int_addr)
    maskIP = netaddr.IPNetwork(prefix).ip
    return (mac64_addr ^ netaddr.IPAddress('::0200:0:0:0') | maskIP).format()

def to_mac(ipv6_address):
    address = netaddr.IPAddress(ipv6_address)
    mask1 = netaddr.IPAddress("::ffff:ffff:ffff:ffff")
    mask2 = netaddr.IPAddress("::0200:0:0:0")
    mac64 = netaddr.EUI(int(address & mask1 ^ mask2)).words
    return ":".join(["%02x" % i for i in mac64[0:3] + mac64[5:8]])

def utcnow():
    """Overridable version of datetime.datetime.utcnow."""
    if utcnow.override_time:
        return utcnow.override_time
    return datetime.datetime.utcnow()

utcnow.override_time = None

def utcnow_ts():
    """Timestamp version of our utcnow function."""
    return time.mktime(utcnow().timetuple())

def set_time_override(override_time=datetime.datetime.utcnow()):
    """Override utils.utcnow to return a constant time."""
    utcnow.override_time = override_time

def advance_time_delta(timedelta):
    """Advance overriden time using a datetime.timedelta."""
    assert(not utcnow.override_time is None)
    utcnow.override_time += timedelta

def advance_time_seconds(seconds):
    """Advance overriden time by seconds."""
    advance_time_delta(datetime.timedelta(0, seconds))

def clear_time_override():
    """Remove the overridden time."""
    utcnow.override_time = None

def isotime(at=None):
    """Returns iso formatted utcnow."""
    if not at:
        at = utcnow()
    return at.strftime(TIME_FORMAT)

def parse_isotime(timestr):
    """Turn an iso formatted time back into a datetime"""
    return datetime.datetime.strptime(timestr, TIME_FORMAT)

def parse_mailmap(mailmap='.mailmap'):
    mapping = {}
    if os.path.exists(mailmap):
        fp = open(mailmap, 'r')
        for l in fp:
            l = l.strip()
            if not l.startswith('#') and ' ' in l:
                canonical_email, alias = l.split(' ')
                mapping[alias] = canonical_email
    return mapping

def str_dict_replace(s, mapping):
    for s1, s2 in mapping.iteritems():
        s = s.replace(s1, s2)
    return s

class LazyPluggable(object):
    """A pluggable backend loaded lazily based on some value."""

    def __init__(self, pivot, **backends):
        self.__backends = backends
        self.__pivot = pivot
        self.__backend = None

    def __get_backend(self):
        if not self.__backend:
            backend_name = self.__pivot.value
            if backend_name not in self.__backends:
                raise exception.Error(_('Invalid backend: %s') % backend_name)

            backend = self.__backends[backend_name]
            if type(backend) == type(tuple()):
                name = backend[0]
                fromlist = backend[1]
            else:
                name = backend
                fromlist = backend

            self.__backend = __import__(name, None, None, fromlist)
            LOG.debug(_('backend %s'), self.__backend)
        return self.__backend

    def __getattr__(self, key):
        backend = self.__get_backend()
        return getattr(backend, key)

class LoopingCall(object):
    def __init__(self, f=None, *args, **kw):
        self.args = args
        self.kw = kw
        self.f = f
        self._running = False

    def start(self, interval, now=True):
        self._running = True
        done = event.Event()

        def _inner():
            if not now:
                greenthread.sleep(interval)
            try:
                while self._running:
                    self.f(*self.args, **self.kw)
                    greenthread.sleep(interval)
            except Exception:
                logging.exception('in looping call')
                done.send_exception(*sys.exc_info())
                return

            done.send(True)

        self.done = done

        greenthread.spawn(_inner)
        return self.done

    def stop(self):
        self._running = False

    def wait(self):
        return self.done.wait()

def xhtml_escape(value):
    """Escapes a string so it is valid within XML or XHTML.

    Code is directly from the utf8 function in
    http://github.com/facebook/tornado/blob/master/tornado/escape.py

    """
    return saxutils.escape(value, {'"': """})

def utf8(value):
    """Try to turn a string into utf-8 if possible.

    Code is directly from the utf8 function in
    http://github.com/facebook/tornado/blob/master/tornado/escape.py

    """
    if isinstance(value, unicode):
        return value.encode("utf-8")
    assert isinstance(value, str)
    return value
<commit_msg>Fixed missing _(). Fixed to follow logging to LOG changes. Fixed merge miss (get_fixed_ip was moved away). Update some missing comments.<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
System-level utilities and helper functions.
"""

import datetime
import inspect
import os
import random
import subprocess
import socket
import struct
import sys
import time
from xml.sax import saxutils
import re
import netaddr

from eventlet import event
from eventlet import greenthread

from nova import exception
from nova.exception import ProcessExecutionError
from nova import log as logging

LOG = logging.getLogger("nova.utils")
TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ"

def import_class(import_str):
    """Returns a class from a string including module and class"""
    mod_str, _sep, class_str = import_str.rpartition('.')
    try:
        __import__(mod_str)
        return getattr(sys.modules[mod_str], class_str)
    except (ImportError, ValueError, AttributeError), exc:
        logging.debug(_('Inner Exception: %s'), exc)
        raise exception.NotFound(_('Class %s cannot be found') % class_str)

def import_object(import_str):
    """Returns an object including a module or module and class"""
    try:
        __import__(import_str)
        return sys.modules[import_str]
    except ImportError:
        cls = import_class(import_str)
        return cls()

def vpn_ping(address, port, timeout=0.05, session_id=None):
    """Sends a vpn negotiation packet and returns the server session.

    Returns False on a failure. Basic packet structure is below.

    Client packet (14 bytes)::
     0 1      8 9  13
    +-+--------+-----+
    |x| cli_id |?????|
    +-+--------+-----+
    x = packet identifier 0x38
    cli_id = 64 bit identifier
    ? = unknown, probably flags/padding

    Server packet (26 bytes)::
     0 1      8 9  13 14    21 2225
    +-+--------+-----+--------+----+
    |x| srv_id |?????| cli_id |????|
    +-+--------+-----+--------+----+
    x = packet identifier 0x40
    cli_id = 64 bit identifier
    ? = unknown, probably flags/padding
    bit 9 was 1 and the rest were 0 in testing
    """
    if session_id is None:
        session_id = random.randint(0, 0xffffffffffffffff)
    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    data = struct.pack("!BQxxxxxx", 0x38, session_id)
    sock.sendto(data, (address, port))
    sock.settimeout(timeout)
    try:
        received = sock.recv(2048)
    except socket.timeout:
        return False
    finally:
        sock.close()
    fmt = "!BQxxxxxQxxxx"
    if len(received) != struct.calcsize(fmt):
        print struct.calcsize(fmt)
        return False
    (identifier, server_sess, client_sess) = struct.unpack(fmt, received)
    if identifier == 0x40 and client_sess == session_id:
        return server_sess

def fetchfile(url, target):
    LOG.debug(_("Fetching %s") % url)
#    c = pycurl.Curl()
#    fp = open(target, "wb")
#    c.setopt(c.URL, url)
#    c.setopt(c.WRITEDATA, fp)
#    c.perform()
#    c.close()
#    fp.close()
    execute("curl --fail %s -o %s" % (url, target))

def execute(cmd, process_input=None, addl_env=None, check_exit_code=True):
    LOG.debug(_("Running cmd (subprocess): %s"), cmd)
    env = os.environ.copy()
    if addl_env:
        env.update(addl_env)
    obj = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE,
        stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
    result = None
    if process_input != None:
        result = obj.communicate(process_input)
    else:
        result = obj.communicate()
    obj.stdin.close()
    if obj.returncode:
        LOG.debug(_("Result was %s") % (obj.returncode))
        if check_exit_code and obj.returncode != 0:
            (stdout, stderr) = result
            raise ProcessExecutionError(exit_code=obj.returncode,
                                        stdout=stdout,
                                        stderr=stderr,
                                        cmd=cmd)
    # NOTE(termie): this appears to be necessary to let the subprocess call
    #               clean something up in between calls, without it two
    #               execute calls in a row hangs the second one
    greenthread.sleep(0)
    return result

def abspath(s):
    return os.path.join(os.path.dirname(__file__), s)

def novadir():
    import nova
    return os.path.abspath(nova.__file__).split('nova/__init__.pyc')[0]

def default_flagfile(filename='nova.conf'):
    for arg in sys.argv:
        if arg.find('flagfile') != -1:
            break
    else:
        if not os.path.isabs(filename):
            # turn relative filename into an absolute path
            script_dir = os.path.dirname(inspect.stack()[-1][1])
            filename = os.path.abspath(os.path.join(script_dir, filename))
        if os.path.exists(filename):
            flagfile = ['--flagfile=%s' % filename]
            sys.argv = sys.argv[:1] + flagfile + sys.argv[1:]

def debug(arg):
    LOG.debug(_('debug in callback: %s'), arg)
    return arg

def runthis(prompt, cmd, check_exit_code=True):
    LOG.debug(_("Running %s"), (cmd))
    rv, err = execute(cmd, check_exit_code=check_exit_code)

def generate_uid(topic, size=8):
    characters = '01234567890abcdefghijklmnopqrstuvwxyz'
    choices = [random.choice(characters) for x in xrange(size)]
    return '%s-%s' % (topic, ''.join(choices))

def generate_mac():
    mac = [0x02, 0x16, 0x3e,
           random.randint(0x00, 0x7f),
           random.randint(0x00, 0xff),
           random.randint(0x00, 0xff)]
    return ':'.join(map(lambda x: "%02x" % x, mac))

def last_octet(address):
    return int(address.split(".")[-1])

def  get_my_linklocal(interface):
    try:
        if_str = execute("ip -f inet6 -o addr show %s" % interface)
        condition = "\s+inet6\s+([0-9a-f:]+/\d+)\s+scope\s+link"
        links = [re.search(condition, x) for x in if_str[0].split('\n')]
        address = [w.group(1) for w in links if w is not None]
        if address[0] is not None:
            return address[0]
        else:
            return 'fe00::'
    except IndexError as ex:
        LOG.warn(_("Couldn't get Link Local IP of %s :%s"), interface, ex)
    except ProcessExecutionError as ex:
        LOG.warn(_("Couldn't get Link Local IP of %s :%s"), interface, ex)
    except:
        return 'fe00::'

def to_global_ipv6(prefix, mac):
    mac64 = netaddr.EUI(mac).eui64().words
    int_addr = int(''.join(['%02x' % i for i in mac64]), 16)
    mac64_addr = netaddr.IPAddress(int_addr)
    maskIP = netaddr.IPNetwork(prefix).ip
    return (mac64_addr ^ netaddr.IPAddress('::0200:0:0:0') | maskIP).format()

def to_mac(ipv6_address):
    address = netaddr.IPAddress(ipv6_address)
    mask1 = netaddr.IPAddress("::ffff:ffff:ffff:ffff")
    mask2 = netaddr.IPAddress("::0200:0:0:0")
    mac64 = netaddr.EUI(int(address & mask1 ^ mask2)).words
    return ":".join(["%02x" % i for i in mac64[0:3] + mac64[5:8]])

def utcnow():
    """Overridable version of datetime.datetime.utcnow."""
    if utcnow.override_time:
        return utcnow.override_time
    return datetime.datetime.utcnow()

utcnow.override_time = None

def utcnow_ts():
    """Timestamp version of our utcnow function."""
    return time.mktime(utcnow().timetuple())

def set_time_override(override_time=datetime.datetime.utcnow()):
    """Override utils.utcnow to return a constant time."""
    utcnow.override_time = override_time

def advance_time_delta(timedelta):
    """Advance overriden time using a datetime.timedelta."""
    assert(not utcnow.override_time is None)
    utcnow.override_time += timedelta

def advance_time_seconds(seconds):
    """Advance overriden time by seconds."""
    advance_time_delta(datetime.timedelta(0, seconds))

def clear_time_override():
    """Remove the overridden time."""
    utcnow.override_time = None

def isotime(at=None):
    """Returns iso formatted utcnow."""
    if not at:
        at = utcnow()
    return at.strftime(TIME_FORMAT)

def parse_isotime(timestr):
    """Turn an iso formatted time back into a datetime"""
    return datetime.datetime.strptime(timestr, TIME_FORMAT)

def parse_mailmap(mailmap='.mailmap'):
    mapping = {}
    if os.path.exists(mailmap):
        fp = open(mailmap, 'r')
        for l in fp:
            l = l.strip()
            if not l.startswith('#') and ' ' in l:
                canonical_email, alias = l.split(' ')
                mapping[alias] = canonical_email
    return mapping

def str_dict_replace(s, mapping):
    for s1, s2 in mapping.iteritems():
        s = s.replace(s1, s2)
    return s

class LazyPluggable(object):
    """A pluggable backend loaded lazily based on some value."""

    def __init__(self, pivot, **backends):
        self.__backends = backends
        self.__pivot = pivot
        self.__backend = None

    def __get_backend(self):
        if not self.__backend:
            backend_name = self.__pivot.value
            if backend_name not in self.__backends:
                raise exception.Error(_('Invalid backend: %s') % backend_name)

            backend = self.__backends[backend_name]
            if type(backend) == type(tuple()):
                name = backend[0]
                fromlist = backend[1]
            else:
                name = backend
                fromlist = backend

            self.__backend = __import__(name, None, None, fromlist)
            LOG.debug(_('backend %s'), self.__backend)
        return self.__backend

    def __getattr__(self, key):
        backend = self.__get_backend()
        return getattr(backend, key)

class LoopingCall(object):
    def __init__(self, f=None, *args, **kw):
        self.args = args
        self.kw = kw
        self.f = f
        self._running = False

    def start(self, interval, now=True):
        self._running = True
        done = event.Event()

        def _inner():
            if not now:
                greenthread.sleep(interval)
            try:
                while self._running:
                    self.f(*self.args, **self.kw)
                    greenthread.sleep(interval)
            except Exception:
                logging.exception('in looping call')
                done.send_exception(*sys.exc_info())
                return

            done.send(True)

        self.done = done

        greenthread.spawn(_inner)
        return self.done

    def stop(self):
        self._running = False

    def wait(self):
        return self.done.wait()

def xhtml_escape(value):
    """Escapes a string so it is valid within XML or XHTML.

    Code is directly from the utf8 function in
    http://github.com/facebook/tornado/blob/master/tornado/escape.py

    """
    return saxutils.escape(value, {'"': """})

def utf8(value):
    """Try to turn a string into utf-8 if possible.

    Code is directly from the utf8 function in
    http://github.com/facebook/tornado/blob/master/tornado/escape.py

    """
    if isinstance(value, unicode):
        return value.encode("utf-8")
    assert isinstance(value, str)
    return value
<|endoftext|>