<commit_before>"""
    wispy.grammar
    ~~~~~~~~~~~~~

    The Powershell grammar implementation of the *wispy* engine.
    The version used is PowerShell Language Specification Version 3.0.

"""
# pylint: disable=missing-docstring, no-init, too-few-public-methods
# pylint: disable=anomalous-unicode-escape-in-string

from modgrammar import (
    Grammar, OR, WORD, REPEAT, ANY_EXCEPT,
    OPTIONAL, WHITESPACE, ANY, REF, EXCEPT
)

class NewLineCharacter(Grammar):
    grammar = OR("\r\n", "\n", "\r")

class Hashes(Grammar):
    grammar = REPEAT("#")

class NotGreaterThanOrHash(Grammar):
    grammar = ANY_EXCEPT("#>")

class InputCharacter(Grammar):
    grammar = ANY_EXCEPT("\r\n", max=1)

class InputCharacters(Grammar):
    grammar = REPEAT(InputCharacter)

class SingleLineComment(Grammar):
    grammar = ("#", OPTIONAL(WHITESPACE), OPTIONAL(InputCharacters))

class DelimitedCommentSection(Grammar):
    grammar = OR(">", (OPTIONAL(Hashes), NotGreaterThanOrHash))

class DelimitedCommentText(Grammar):
    grammar = OR(DelimitedCommentSection,
                 (REF('DelimitedCommentText', DelimitedCommentSection)))

class DelimitedComment(Grammar):
    grammar = ("<#", OPTIONAL(DelimitedCommentText), Hashes, ">")

class Comment(Grammar):
    grammar = OR(SingleLineComment, DelimitedComment)

class Keyword(Grammar):
    grammar = OR("begin", "break", "catch", "class",
                 "continue", "data", "define", "do",
                 "dynamicparam", "else", "elseif", "end",
                 "exit", "filter", "finally", "for",
                 "foreach", "from", "function", "if",
                 "in", "param", "process", "return",
                 "switch", "throw", "trap", "try",
                 "until", "using", "var", "while")

# Literals

class NumericMultiplier(Grammar):
    grammar = REPEAT(OR("kb", "mb", "tb", "pb", "gb"), max=1)

class LongTypeSuffix(Grammar):
    grammar = OR("l", "L")

class DecimalTypeSuffix(Grammar):
    grammar = OR("d", "D")

class NumericTypeSuffix(Grammar):
    grammar = OR(DecimalTypeSuffix, LongTypeSuffix)

class DecimalDigit(Grammar):
    grammar = WORD('0-9', max=1)

class DecimalDigits(Grammar):
    grammar = REPEAT(DecimalDigit)

class DecimalIntegerLiteral(Grammar):
    grammar = (DecimalDigits,
               OPTIONAL(DecimalTypeSuffix),
               OPTIONAL(NumericMultiplier))

class HexadecimalDigit(Grammar):
    grammar = WORD('0-9a-fA-F', max=1)

class HexadecimalDigits(Grammar):
    grammar = OR(REPEAT(HexadecimalDigit),
                 (HexadecimalDigit, DecimalDigits))

class HexadecimalIntegerLiteral(Grammar):
    grammar = ("0x", HexadecimalDigits,
               OPTIONAL(LongTypeSuffix),
               OPTIONAL(NumericMultiplier))

class IntegerLiteral(Grammar):
    grammar = OR(HexadecimalIntegerLiteral, DecimalIntegerLiteral)

class Dash(Grammar):
    grammar = OR("-", "\u2013", "\u2014", "\u2015")

class Sign(Grammar):
    grammar = OR("+", Dash, max=1)

class ExponentPart(Grammar):
    grammar = (OR("e", "E"), OPTIONAL(Sign), DecimalDigits)

class RealLiteral(Grammar):
    grammar = OR(
        (DecimalDigits, ".", DecimalDigits, OPTIONAL(ExponentPart),
         OPTIONAL(NumericMultiplier)),

        (".", DecimalDigits, OPTIONAL(ExponentPart),
         OPTIONAL(DecimalTypeSuffix), OPTIONAL(NumericMultiplier)),

        (DecimalDigits, ExponentPart, OPTIONAL(DecimalTypeSuffix),
         OPTIONAL(NumericMultiplier)))

class AssignmentOperator(Grammar):
    grammar = OR(
        "=", (Dash, "="), "+=", "*=", "/=", "%="
    )

class FileRedirectionOperator(Grammar):
    grammar = OR(
        ">>", ">", "<", "2>>", "2>"
    )

class ComparisonOperator(Grammar):
    grammar = OR(
        (Dash, "as"), (Dash, "ccontains"), (Dash, "ceq"),
        (Dash, "cge"), (Dash, "cgt"), (Dash, "cle"),
        (Dash, "clike"), (Dash, "clt"), (Dash, "cmatch"),
        (Dash, "cne"), (Dash, "cnotcontains"), (Dash, "cnotlike"),
        (Dash, "cnotmatch"), (Dash, "contains"), (Dash, "creplace"),
        (Dash, "csplit"), (Dash, "eq"), (Dash, "ge"),
        (Dash, "gt"), (Dash, "icontains"), (Dash, "ieq"),
        (Dash, "ige"), (Dash, "igt"), (Dash, "ile"),
        (Dash, "ilike"), (Dash, "ilt"), (Dash, "imatch"),
        (Dash, "ine"), (Dash, "inotcontains"), (Dash, "inotlike"),
        (Dash, "inotmatch"), (Dash, "ireplace"), (Dash, "is"),
        (Dash, "isnot"), (Dash, "isplit"), (Dash, "join"),
        (Dash, "le"), (Dash, "like"), (Dash, "lt"),
        (Dash, "match"), (Dash, "ne"), (Dash, "notcontains"),
        (Dash, "notlike"), (Dash, "notmatch"), (Dash, "replace"),
        (Dash, "split")
    )

class FormatOperator(Grammar):
    grammar = (Dash, "f")

class OperatorOrPunctuator(Grammar):
    grammar = OR(
        "{", "}", "[", "]", "(", ")", "@(", "@{", "$(", ";",
        "&&", "||", "&", "|", ",", "++", "..", "::", ".",
        "!", "*", "/", "%", "+", "2>&1", "1>&2",
        Dash, (Dash, Dash),
        (Dash, "and"), (Dash, "band"), (Dash, "bnot"),
        (Dash, "bor"), (Dash, "bxor"), (Dash, "not"),
        (Dash, "or"), (Dash, "xor"),
        AssignmentOperator,
        FileRedirectionOperator,
        ComparisonOperator,
        FormatOperator
    )

# Variables
class VariableCharacter(Grammar):
    grammar = OR(
        WORD("A-Z", max=1),  # Letter, Uppercase
        WORD("a-z", max=1),  # Letter, Lowercase

        # Letter, Titlecase
        WORD("\u01C5"), WORD("\u01C8"), WORD("\u01CB"), WORD("\u01F2"),
        WORD("\u1F88"), WORD("\u1F89"), WORD("\u1F8A"), WORD("\u1F8B"),
        WORD("\u1F8C"), WORD("\u1F8D"), WORD("\u1F8E"), WORD("\u1F8F"),
        WORD("\u1F98"), WORD("\u1F99"), WORD("\u1F9A"), WORD("\u1F9B"),
        WORD("\u1F9C"), WORD("\u1F9D"), WORD("\u1F9E"), WORD("\u1F9F"),
        WORD("\u1FA8"), WORD("\u1FA9"), WORD("\u1FAA"), WORD("\u1FAB"),
        WORD("\u1FAC"), WORD("\u1FAD"), WORD("\u1FAE"), WORD("\u1FAF"),
        WORD("\u1FBC"), WORD("\u1FCC"), WORD("\u1FFC"),

        # Letter, Modifier
        WORD("\u02B0-\u02EE", max=1),
        WORD("\u0374"), WORD("\u037A"), WORD("\u0559"), WORD("\u0640"),
        WORD("\u06E5"), WORD("\u06E6"), WORD("\u07F4"), WORD("\u07F5"),
        WORD("\u07FA"), WORD("\u081A"), WORD("\u0824"), WORD("\u0828"),
        # TODO: Add more characters from the 'Letter, Modifier` Category
        # TODO: Add characters from the 'Letter, Other' Category

        # Number, Decimal Digit
        WORD("0-9", max=1),
        WORD("\u0660-\u0669", max=1),  # ARABIC-INDIC DIGIT
        WORD("\u06F0-\u06F9", max=1),  # EXTENDED ARABIC-INDIC DIGIT
        # TODO: Add more character from the 'Number, Decimal Digit' Category

        WORD("\u005F"),                # The underscore character
        WORD("?")
    )

class VariableCharacters(Grammar):
    grammar = REPEAT(VariableCharacter)

class VariableNamespace(Grammar):
    grammar = (VariableCharacters, ":")

class VariableScope(Grammar):
    grammar = OR("global:", "local:", "private:", "script:",
                 VariableNamespace)

class EscapedCharacter(Grammar):
    grammar = ("\u0060", ANY)

class BracedVariableCharacter(Grammar):
    grammar = OR(ANY_EXCEPT("\u007D\u0060", max=1), EscapedCharacter)

class BracedVariableCharacters(Grammar):
    grammar = REPEAT(BracedVariableCharacter)

class BracedVariable(Grammar):
    grammar = ("$", "{", OPTIONAL(VariableScope),
               BracedVariableCharacters, "}")

# String Literals

class DoubleQuoteCharacter(Grammar):
    grammar = OR("\u0022", "\u201C", "\u201D", "\u201E")

class ExpandableStringPart(Grammar):
    grammar = OR(
        ANY_EXCEPT("$\u0022\u201C\u201D\u201E\u0060"),
        BracedVariable,
        ("$", ANY_EXCEPT("({\u0022\u201C\u201D\u201E\u0060")),
        ("$", EscapedCharacter),
        EscapedCharacter,
        (DoubleQuoteCharacter, DoubleQuoteCharacter)
    )

class Dollars(Grammar):
    grammar = REPEAT("$")

class ExpandableStringCharacters(Grammar):
    grammar = OR(
        ExpandableStringPart,
        (ExpandableStringCharacters, ExpandableStringPart)
    )

class ExpandableHereStringPart(Grammar):
    grammar = OR(
        EXCEPT(ANY_EXCEPT("$", max=1), NewLineCharacter),
        BracedVariable,
        ("$", EXCEPT(ANY_EXCEPT("(", max=1), NewLineCharacter)),
        ("$", NewLineCharacter, EXCEPT(ANY, DoubleQuoteCharacter)),
        ("$", NewLineCharacter, DoubleQuoteCharacter, ANY_EXCEPT("@")),
        (NewLineCharacter, EXCEPT(ANY, DoubleQuoteCharacter)),
        (NewLineCharacter, DoubleQuoteCharacter, ANY_EXCEPT("@"))
    )

class ExpandableStringWithSubexprStart(Grammar):
    grammar = (DoubleQuoteCharacter, OPTIONAL(ExpandableStringCharacters),
               "$", "(")

class ExpandableStringWithSubexprEnd(Grammar):
    grammar = DoubleQuoteCharacter

class ExpandableHereStringCharacters(Grammar):
    grammar = OR(
        ExpandableHereStringPart,
        (ExpandableHereStringCharacters, ExpandableHereStringPart)
    )

class ExpandableHereStringWithSubexprStart(Grammar):
    grammar = (
        "@", DoubleQuoteCharacter, OPTIONAL(WHITESPACE),
        NewLineCharacter, OPTIONAL(ExpandableHereStringCharacters),
        "$", "("
    )

class ExpandableHereStringWithSubexprEnd(Grammar):
    grammar = (NewLineCharacter, DoubleQuoteCharacter, "@")

class ExpandableStringLiteral(Grammar):
    grammar = (DoubleQuoteCharacter, OPTIONAL(ExpandableStringCharacters),
               OPTIONAL(Dollars), DoubleQuoteCharacter)

class ExpandableHereStringLiteral(Grammar):
    grammar = ("@", DoubleQuoteCharacter, OPTIONAL(WHITESPACE),
               NewLineCharacter, OPTIONAL(ExpandableHereStringCharacters),
               NewLineCharacter, DoubleQuoteCharacter, "@")

class SingleQuoteCharacter(Grammar):
    grammar = OR("\u0027", "\u2018", "\u2019", "\u201A", "\u201B")

class VerbatimStringPart(Grammar):
    grammar = OR(
        EXCEPT(ANY, SingleQuoteCharacter),
        (SingleQuoteCharacter, SingleQuoteCharacter)
    )

class VerbatimStringCharacters(Grammar):
    grammar = OR(
        VerbatimStringPart,
        (VerbatimStringCharacters, VerbatimStringPart)
    )

class VerbatimStringLiteral(Grammar):
    grammar = (SingleQuoteCharacter, OPTIONAL(VerbatimStringCharacters),
               SingleQuoteCharacter)

class VerbatimHereStringPart(Grammar):
    grammar = OR(
        EXCEPT(ANY, NewLineCharacter),
        (NewLineCharacter, EXCEPT(ANY, SingleQuoteCharacter)),
        (NewLineCharacter, SingleQuoteCharacter, ANY_EXCEPT("@"))
    )

class VerbatimHereStringCharacters(Grammar):
    grammar = OR(
        VerbatimHereStringPart,
        (VerbatimHereStringCharacters, VerbatimHereStringPart)
    )

class VerbatimHereStringLiteral(Grammar):
    grammar = ("@", SingleQuoteCharacter, OPTIONAL(WHITESPACE),
               NewLineCharacter, OPTIONAL(VerbatimHereStringCharacters),
               NewLineCharacter, SingleQuoteCharacter, "@")

class StringLiteral(Grammar):
    grammar = OR(ExpandableStringLiteral, ExpandableHereStringLiteral,
                 VerbatimStringLiteral, VerbatimHereStringLiteral)

# Type names.

class TypeCharacter(Grammar):
    grammar = OR(
        WORD("A-Z", max=1),  # Letter, Uppercase
        WORD("a-z", max=1),  # Letter, Lowercase,

        # Letter, Titlecase
        WORD("\u01C5"), WORD("\u01C8"), WORD("\u01CB"), WORD("\u01F2"),
        WORD("\u1F88"), WORD("\u1F89"), WORD("\u1F8A"), WORD("\u1F8B"),
        WORD("\u1F8C"), WORD("\u1F8D"), WORD("\u1F8E"), WORD("\u1F8F"),
        WORD("\u1F98"), WORD("\u1F99"), WORD("\u1F9A"), WORD("\u1F9B"),
        WORD("\u1F9C"), WORD("\u1F9D"), WORD("\u1F9E"), WORD("\u1F9F"),
        WORD("\u1FA8"), WORD("\u1FA9"), WORD("\u1FAA"), WORD("\u1FAB"),
        WORD("\u1FAC"), WORD("\u1FAD"), WORD("\u1FAE"), WORD("\u1FAF"),
        WORD("\u1FBC"), WORD("\u1FCC"), WORD("\u1FFC"),

        # Letter, Modifier
        WORD("\u02B0-\u02EE", max=1),
        WORD("\u0374"), WORD("\u037A"), WORD("\u0559"), WORD("\u0640"),
        WORD("\u06E5"), WORD("\u06E6"), WORD("\u07F4"), WORD("\u07F5"),
        WORD("\u07FA"), WORD("\u081A"), WORD("\u0824"), WORD("\u0828"),
        # TODO: Add more characters from the 'Letter, Modifier` Category
        # TODO: Add characters from the 'Letter, Other' Category

        WORD("\u005F"),
    )

class TypeCharacters(Grammar):
    grammar = REPEAT(TypeCharacter)

class TypeIdentifier(Grammar):
    grammar = TypeCharacters

class TypeName(Grammar):
    grammar = OR(TypeIdentifier, (REF('TypeName'), '.', TypeIdentifier))

class ArrayTypeName(Grammar):
    grammar = (TypeName, "[")

class GenericTypeName(Grammar):
    grammar = ArrayTypeName
<commit_msg>Adding docstrings for String Literals.<commit_after>"""
    wispy.grammar
    ~~~~~~~~~~~~~

    The Powershell grammar implementation of the *wispy* engine.
    The version used is PowerShell Language Specification Version 3.0.

"""
# pylint: disable=missing-docstring, no-init, too-few-public-methods
# pylint: disable=anomalous-unicode-escape-in-string

from modgrammar import (
    Grammar, OR, WORD, REPEAT, ANY_EXCEPT,
    OPTIONAL, WHITESPACE, ANY, REF, EXCEPT
)

class NewLineCharacter(Grammar):
    grammar = OR("\r\n", "\n", "\r")

class Hashes(Grammar):
    grammar = REPEAT("#")

class NotGreaterThanOrHash(Grammar):
    grammar = ANY_EXCEPT("#>")

class InputCharacter(Grammar):
    grammar = ANY_EXCEPT("\r\n", max=1)

class InputCharacters(Grammar):
    grammar = REPEAT(InputCharacter)

class SingleLineComment(Grammar):
    grammar = ("#", OPTIONAL(WHITESPACE), OPTIONAL(InputCharacters))

class DelimitedCommentSection(Grammar):
    grammar = OR(">", (OPTIONAL(Hashes), NotGreaterThanOrHash))

class DelimitedCommentText(Grammar):
    grammar = OR(DelimitedCommentSection,
                 (REF('DelimitedCommentText', DelimitedCommentSection)))

class DelimitedComment(Grammar):
    grammar = ("<#", OPTIONAL(DelimitedCommentText), Hashes, ">")

class Comment(Grammar):
    grammar = OR(SingleLineComment, DelimitedComment)

class Keyword(Grammar):
    grammar = OR("begin", "break", "catch", "class",
                 "continue", "data", "define", "do",
                 "dynamicparam", "else", "elseif", "end",
                 "exit", "filter", "finally", "for",
                 "foreach", "from", "function", "if",
                 "in", "param", "process", "return",
                 "switch", "throw", "trap", "try",
                 "until", "using", "var", "while")

# Literals

class NumericMultiplier(Grammar):
    grammar = REPEAT(OR("kb", "mb", "tb", "pb", "gb"), max=1)

class LongTypeSuffix(Grammar):
    grammar = OR("l", "L")

class DecimalTypeSuffix(Grammar):
    grammar = OR("d", "D")

class NumericTypeSuffix(Grammar):
    grammar = OR(DecimalTypeSuffix, LongTypeSuffix)

class DecimalDigit(Grammar):
    grammar = WORD('0-9', max=1)

class DecimalDigits(Grammar):
    grammar = REPEAT(DecimalDigit)

class DecimalIntegerLiteral(Grammar):
    grammar = (DecimalDigits,
               OPTIONAL(DecimalTypeSuffix),
               OPTIONAL(NumericMultiplier))

class HexadecimalDigit(Grammar):
    grammar = WORD('0-9a-fA-F', max=1)

class HexadecimalDigits(Grammar):
    grammar = OR(REPEAT(HexadecimalDigit),
                 (HexadecimalDigit, DecimalDigits))

class HexadecimalIntegerLiteral(Grammar):
    grammar = ("0x", HexadecimalDigits,
               OPTIONAL(LongTypeSuffix),
               OPTIONAL(NumericMultiplier))

class IntegerLiteral(Grammar):
    grammar = OR(HexadecimalIntegerLiteral, DecimalIntegerLiteral)

class Dash(Grammar):
    grammar = OR("-", "\u2013", "\u2014", "\u2015")

class Sign(Grammar):
    grammar = OR("+", Dash, max=1)

class ExponentPart(Grammar):
    grammar = (OR("e", "E"), OPTIONAL(Sign), DecimalDigits)

class RealLiteral(Grammar):
    grammar = OR(
        (DecimalDigits, ".", DecimalDigits, OPTIONAL(ExponentPart),
         OPTIONAL(NumericMultiplier)),

        (".", DecimalDigits, OPTIONAL(ExponentPart),
         OPTIONAL(DecimalTypeSuffix), OPTIONAL(NumericMultiplier)),

        (DecimalDigits, ExponentPart, OPTIONAL(DecimalTypeSuffix),
         OPTIONAL(NumericMultiplier)))

class AssignmentOperator(Grammar):
    grammar = OR(
        "=", (Dash, "="), "+=", "*=", "/=", "%="
    )

class FileRedirectionOperator(Grammar):
    grammar = OR(
        ">>", ">", "<", "2>>", "2>"
    )

class ComparisonOperator(Grammar):
    grammar = OR(
        (Dash, "as"), (Dash, "ccontains"), (Dash, "ceq"),
        (Dash, "cge"), (Dash, "cgt"), (Dash, "cle"),
        (Dash, "clike"), (Dash, "clt"), (Dash, "cmatch"),
        (Dash, "cne"), (Dash, "cnotcontains"), (Dash, "cnotlike"),
        (Dash, "cnotmatch"), (Dash, "contains"), (Dash, "creplace"),
        (Dash, "csplit"), (Dash, "eq"), (Dash, "ge"),
        (Dash, "gt"), (Dash, "icontains"), (Dash, "ieq"),
        (Dash, "ige"), (Dash, "igt"), (Dash, "ile"),
        (Dash, "ilike"), (Dash, "ilt"), (Dash, "imatch"),
        (Dash, "ine"), (Dash, "inotcontains"), (Dash, "inotlike"),
        (Dash, "inotmatch"), (Dash, "ireplace"), (Dash, "is"),
        (Dash, "isnot"), (Dash, "isplit"), (Dash, "join"),
        (Dash, "le"), (Dash, "like"), (Dash, "lt"),
        (Dash, "match"), (Dash, "ne"), (Dash, "notcontains"),
        (Dash, "notlike"), (Dash, "notmatch"), (Dash, "replace"),
        (Dash, "split")
    )

class FormatOperator(Grammar):
    grammar = (Dash, "f")

class OperatorOrPunctuator(Grammar):
    grammar = OR(
        "{", "}", "[", "]", "(", ")", "@(", "@{", "$(", ";",
        "&&", "||", "&", "|", ",", "++", "..", "::", ".",
        "!", "*", "/", "%", "+", "2>&1", "1>&2",
        Dash, (Dash, Dash),
        (Dash, "and"), (Dash, "band"), (Dash, "bnot"),
        (Dash, "bor"), (Dash, "bxor"), (Dash, "not"),
        (Dash, "or"), (Dash, "xor"),
        AssignmentOperator,
        FileRedirectionOperator,
        ComparisonOperator,
        FormatOperator
    )

# Variables
class VariableCharacter(Grammar):
    grammar = OR(
        WORD("A-Z", max=1),  # Letter, Uppercase
        WORD("a-z", max=1),  # Letter, Lowercase

        # Letter, Titlecase
        WORD("\u01C5"), WORD("\u01C8"), WORD("\u01CB"), WORD("\u01F2"),
        WORD("\u1F88"), WORD("\u1F89"), WORD("\u1F8A"), WORD("\u1F8B"),
        WORD("\u1F8C"), WORD("\u1F8D"), WORD("\u1F8E"), WORD("\u1F8F"),
        WORD("\u1F98"), WORD("\u1F99"), WORD("\u1F9A"), WORD("\u1F9B"),
        WORD("\u1F9C"), WORD("\u1F9D"), WORD("\u1F9E"), WORD("\u1F9F"),
        WORD("\u1FA8"), WORD("\u1FA9"), WORD("\u1FAA"), WORD("\u1FAB"),
        WORD("\u1FAC"), WORD("\u1FAD"), WORD("\u1FAE"), WORD("\u1FAF"),
        WORD("\u1FBC"), WORD("\u1FCC"), WORD("\u1FFC"),

        # Letter, Modifier
        WORD("\u02B0-\u02EE", max=1),
        WORD("\u0374"), WORD("\u037A"), WORD("\u0559"), WORD("\u0640"),
        WORD("\u06E5"), WORD("\u06E6"), WORD("\u07F4"), WORD("\u07F5"),
        WORD("\u07FA"), WORD("\u081A"), WORD("\u0824"), WORD("\u0828"),
        # TODO: Add more characters from the 'Letter, Modifier` Category
        # TODO: Add characters from the 'Letter, Other' Category

        # Number, Decimal Digit
        WORD("0-9", max=1),
        WORD("\u0660-\u0669", max=1),  # ARABIC-INDIC DIGIT
        WORD("\u06F0-\u06F9", max=1),  # EXTENDED ARABIC-INDIC DIGIT
        # TODO: Add more character from the 'Number, Decimal Digit' Category

        WORD("\u005F"),                # The underscore character
        WORD("?")
    )

class VariableCharacters(Grammar):
    grammar = REPEAT(VariableCharacter)

class VariableNamespace(Grammar):
    grammar = (VariableCharacters, ":")

class VariableScope(Grammar):
    grammar = OR("global:", "local:", "private:", "script:",
                 VariableNamespace)

class EscapedCharacter(Grammar):
    grammar = ("\u0060", ANY)

class BracedVariableCharacter(Grammar):
    grammar = OR(ANY_EXCEPT("\u007D\u0060", max=1), EscapedCharacter)

class BracedVariableCharacters(Grammar):
    grammar = REPEAT(BracedVariableCharacter)

class BracedVariable(Grammar):
    grammar = ("$", "{", OPTIONAL(VariableScope),
               BracedVariableCharacters, "}")

# String Literals

class DoubleQuoteCharacter(Grammar):
    grammar = OR("\u0022", "\u201C", "\u201D", "\u201E")

class ExpandableStringPart(Grammar):
    grammar = OR(
        ANY_EXCEPT("$\u0022\u201C\u201D\u201E\u0060"),
        BracedVariable,
        ("$", ANY_EXCEPT("({\u0022\u201C\u201D\u201E\u0060")),
        ("$", EscapedCharacter),
        EscapedCharacter,
        (DoubleQuoteCharacter, DoubleQuoteCharacter)
    )

class Dollars(Grammar):
    grammar = REPEAT("$")

class ExpandableStringCharacters(Grammar):
    grammar = OR(
        ExpandableStringPart,
        (ExpandableStringCharacters, ExpandableStringPart)
    )

class ExpandableHereStringPart(Grammar):
    grammar = OR(
        EXCEPT(ANY_EXCEPT("$", max=1), NewLineCharacter),
        BracedVariable,
        ("$", EXCEPT(ANY_EXCEPT("(", max=1), NewLineCharacter)),
        ("$", NewLineCharacter, EXCEPT(ANY, DoubleQuoteCharacter)),
        ("$", NewLineCharacter, DoubleQuoteCharacter, ANY_EXCEPT("@")),
        (NewLineCharacter, EXCEPT(ANY, DoubleQuoteCharacter)),
        (NewLineCharacter, DoubleQuoteCharacter, ANY_EXCEPT("@"))
    )

class ExpandableStringWithSubexprStart(Grammar):
    grammar = (DoubleQuoteCharacter, OPTIONAL(ExpandableStringCharacters),
               "$", "(")

class ExpandableStringWithSubexprEnd(Grammar):
    grammar = DoubleQuoteCharacter

class ExpandableHereStringCharacters(Grammar):
    grammar = OR(
        ExpandableHereStringPart,
        (ExpandableHereStringCharacters, ExpandableHereStringPart)
    )

class ExpandableHereStringWithSubexprStart(Grammar):
    grammar = (
        "@", DoubleQuoteCharacter, OPTIONAL(WHITESPACE),
        NewLineCharacter, OPTIONAL(ExpandableHereStringCharacters),
        "$", "("
    )

class ExpandableHereStringWithSubexprEnd(Grammar):
    grammar = (NewLineCharacter, DoubleQuoteCharacter, "@")

class SingleQuoteCharacter(Grammar):
    grammar = OR("\u0027", "\u2018", "\u2019", "\u201A", "\u201B")

class VerbatimStringPart(Grammar):
    grammar = OR(
        EXCEPT(ANY, SingleQuoteCharacter),
        (SingleQuoteCharacter, SingleQuoteCharacter)
    )

class VerbatimStringCharacters(Grammar):
    grammar = OR(
        VerbatimStringPart,
        (VerbatimStringCharacters, VerbatimStringPart)
    )

class VerbatimHereStringPart(Grammar):
    grammar = OR(
        EXCEPT(ANY, NewLineCharacter),
        (NewLineCharacter, EXCEPT(ANY, SingleQuoteCharacter)),
        (NewLineCharacter, SingleQuoteCharacter, ANY_EXCEPT("@"))
    )

class VerbatimHereStringCharacters(Grammar):
    grammar = OR(
        VerbatimHereStringPart,
        (VerbatimHereStringCharacters, VerbatimHereStringPart)
    )

class ExpandableStringLiteral(Grammar):

    """Expandable string literal (single-line double-quoted), which is
    a sequence of zero or more characters delimited by a pair of
    double-quote-characters.

    Examples are "" and "red".
    """
    grammar = (DoubleQuoteCharacter, OPTIONAL(ExpandableStringCharacters),
               OPTIONAL(Dollars), DoubleQuoteCharacter)

class ExpandableHereStringLiteral(Grammar):

    """Expandable here string literal (multi-line double-quoted), which is
    a sequence of zero or more characters delimited by the character pairs
    @double-quote-character and double-quote-character@, respectively,
    all contained on two or more source lines.

    Examples are:
    ::
    @"
    "@

    @"
    line 1
    "@

    @"
    line 1
    line 2
    "@
    """
    grammar = ("@", DoubleQuoteCharacter, OPTIONAL(WHITESPACE),
               NewLineCharacter, OPTIONAL(ExpandableHereStringCharacters),
               NewLineCharacter, DoubleQuoteCharacter, "@")

class VerbatimStringLiteral(Grammar):

    """Verbatim string literal (single-line single-quoted), which is a
    sequence of zero or more characters delimited by a pair
    of SingleQuoteCharacters.

    Examples are '' and 'red'.
    """

    grammar = (SingleQuoteCharacter, OPTIONAL(VerbatimStringCharacters),
               SingleQuoteCharacter)

class VerbatimHereStringLiteral(Grammar):

    """Verbatim here string literal (multi-line single-quoted), which is
    a sequence of zero or more characters delimited by the character pairs
    @single-quote-character and single-quote-character@, respectively, all
    contained on two or more source lines.

    Examples are:

    ::
    @'
    '@

    @'
    line 1
    line 2
    '@
    """
    grammar = ("@", SingleQuoteCharacter, OPTIONAL(WHITESPACE),
               NewLineCharacter, OPTIONAL(VerbatimHereStringCharacters),
               NewLineCharacter, SingleQuoteCharacter, "@")

class StringLiteral(Grammar):

    """String literal is one of the following:
        * ExpandableStringLiteral
        * ExpandableHereStringLiteral
        * VerbatimStringLiteral
        * VerbatimHereStringLiteral
    """
    grammar = OR(ExpandableStringLiteral,
                 ExpandableHereStringLiteral,
                 VerbatimStringLiteral,
                 VerbatimHereStringLiteral)

# Type names.

class TypeCharacter(Grammar):
    grammar = OR(
        WORD("A-Z", max=1),  # Letter, Uppercase
        WORD("a-z", max=1),  # Letter, Lowercase,

        # Letter, Titlecase
        WORD("\u01C5"), WORD("\u01C8"), WORD("\u01CB"), WORD("\u01F2"),
        WORD("\u1F88"), WORD("\u1F89"), WORD("\u1F8A"), WORD("\u1F8B"),
        WORD("\u1F8C"), WORD("\u1F8D"), WORD("\u1F8E"), WORD("\u1F8F"),
        WORD("\u1F98"), WORD("\u1F99"), WORD("\u1F9A"), WORD("\u1F9B"),
        WORD("\u1F9C"), WORD("\u1F9D"), WORD("\u1F9E"), WORD("\u1F9F"),
        WORD("\u1FA8"), WORD("\u1FA9"), WORD("\u1FAA"), WORD("\u1FAB"),
        WORD("\u1FAC"), WORD("\u1FAD"), WORD("\u1FAE"), WORD("\u1FAF"),
        WORD("\u1FBC"), WORD("\u1FCC"), WORD("\u1FFC"),

        # Letter, Modifier
        WORD("\u02B0-\u02EE", max=1),
        WORD("\u0374"), WORD("\u037A"), WORD("\u0559"), WORD("\u0640"),
        WORD("\u06E5"), WORD("\u06E6"), WORD("\u07F4"), WORD("\u07F5"),
        WORD("\u07FA"), WORD("\u081A"), WORD("\u0824"), WORD("\u0828"),
        # TODO: Add more characters from the 'Letter, Modifier` Category
        # TODO: Add characters from the 'Letter, Other' Category

        WORD("\u005F"),
    )

class TypeCharacters(Grammar):
    grammar = REPEAT(TypeCharacter)

class TypeIdentifier(Grammar):
    grammar = TypeCharacters

class TypeName(Grammar):
    grammar = OR(TypeIdentifier, (REF('TypeName'), '.', TypeIdentifier))

class ArrayTypeName(Grammar):
    grammar = (TypeName, "[")

class GenericTypeName(Grammar):
    grammar = ArrayTypeName
<|endoftext|>


<commit_before>#!/usr/bin/env python
import os
import rospy
import speech_recognition as sr
from std_msgs.msg import String

DEFAULT_MIC_SENSITIVITY = 300

GOOGLE_API_CREDENTIALS_JSON_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), '../google_api_credentials.json')

def init():
	rospy.init_node('dogbotVoiceRecognizer', anonymous=True)
	pub = rospy.Publisher('/dogbot/voice_command', String, queue_size=10)
	rate = rospy.Rate(1)
	recognizer = sr.Recognizer()

	microphone_sensitivity = rospy.get_param('mic_sensitivity', DEFAULT_MIC_SENSITIVITY)
	print 'Microphone sensitivity: ' + microphone_sensitivity

	with open(GOOGLE_API_CREDENTIALS_JSON_PATH, 'r') as credentialsFile:
		google_api_credentials = credentialsFile.read()

	with sr.Microphone() as source:
		while not rospy.is_shutdown():
			print 'Say something...'
			audio = recognizer.listen(source)
			print 'Got it! Processing...'
			try:
				phrase = recognizer.recognize_google_cloud(audio, credentials_json=google_api_credentials)
				print 'I think you said: "' + phrase + '"'
				pub.publish(phrase)
			except sr.UnknownValueError:
				print 'Google Cloud Speech could not understand audio'
				rospy.logerr('Google Cloud Speech could not understand audio')
			except sr.RequestError as e:
				print 'Could not request results from Google Cloud Speech service; {0}'.format(e)
				rospy.logerr('Could not request results from Google Cloud Speech service; {0}'.format(e))
			rate.sleep()

if __name__ == '__main__':
	init()
<commit_msg>added parameter to control mic sensitivity<commit_after>#!/usr/bin/env python
import os
import rospy
import speech_recognition as sr
from std_msgs.msg import String

DEFAULT_MIC_SENSITIVITY = 300

GOOGLE_API_CREDENTIALS_JSON_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), '../google_api_credentials.json')

def init():
	rospy.init_node('dogbotVoiceRecognizer', anonymous=True)
	pub = rospy.Publisher('/dogbot/voice_command', String, queue_size=10)
	rate = rospy.Rate(1)

	microphone_sensitivity = rospy.get_param('mic_sensitivity', DEFAULT_MIC_SENSITIVITY)
	print 'Microphone sensitivity: {}'.format(microphone_sensitivity)

	recognizer = sr.Recognizer()
	recognizer.energy_threshold = microphone_sensitivity

	with open(GOOGLE_API_CREDENTIALS_JSON_PATH, 'r') as credentialsFile:
		google_api_credentials = credentialsFile.read()

	with sr.Microphone() as source:
		while not rospy.is_shutdown():
			print 'Say something...'
			audio = recognizer.listen(source)
			print 'Got it! Processing...'
			try:
				phrase = recognizer.recognize_google_cloud(audio, credentials_json=google_api_credentials)
				print 'I think you said: "' + phrase + '"'
				pub.publish(phrase)
			except sr.UnknownValueError:
				print 'Google Cloud Speech could not understand audio'
				rospy.logerr('Google Cloud Speech could not understand audio')
			except sr.RequestError as e:
				print 'Could not request results from Google Cloud Speech service; {0}'.format(e)
				rospy.logerr('Could not request results from Google Cloud Speech service; {0}'.format(e))
			rate.sleep()

if __name__ == '__main__':
	init()
<|endoftext|>


<commit_before>import paramiko
import socket
import json

from exceptions import NoConnectionError, RemoteCommandFailedError

class RemoteClient(object):
    def __init__(self, hostname, port, username, key_path):
        self.hostname = hostname
        self.port = port
        self.username = username
        self.key_path = key_path

    def open(self):
        print("Starting SSH connection to client...")
        self._client = paramiko.SSHClient()
        self._client.set_missing_host_key_policy(paramiko.WarningPolicy())

        key = paramiko.RSAKey.from_private_key_file(self.key_path)

        self._client.connect(hostname=self.hostname, port=self.port,
                             username=self.username, pkey=key)

        self._transport = self._client.get_transport()
        self._channel = self._transport.open_session()

        print("Connected to client!")

    def _execute_command(self, method, arguments_dict):
        # Build up a dictionary to hold the command, dump it as json, append a newline to
        # the JSON string (to signify the end of a comamnd to the remote system) and then
        # send it down the SSH connection
        command = {"method": method, "arguments": arguments_dict}
        command_string = json.dumps(command) + "\n"

        # If self does not have the _channel attribute, then the client has not been
        # connected yet!
        try:
            self._channel.send(command_string)
        except AttributeError:
            raise NoConnectionError

        # Now receive the response from the channel - To ensure that we receive data even if
        # is is larger than the channel's buffer we keep receiving data until we recieve a
        # newline character signifying the end of the transmission
        recieve_buffer = bytearray()
        end_of_transmission = False
        while(not end_of_transmission):
            recieve_buffer.extend(self._channel.recv(1024))
            if recieve_buffer.endswith(b"\n"):
                end_of_transmission = True

        response = json.loads(recieve_buffer.decode("UTF-8"))
        if response["error"]:
            raise RemoteCommmandFailedError
        else:
            return response["result"]

    def check_version(self, plugin_name, plugin_version):
        arguments_dict = {  "plugin_name": plugin_name,
                            "plugin_version": plugin_version}

        return self._execute_command("check_version", arguments_dict)

    def close(self):
        print("Closing SSH connection to client...")
        self._channel.close()
        self._client.close()
        print("Connection closed!")

    def __enter__(self):
        self.open()
        return self

    def __exit__(self, type, value, traceback):
        self.close()

if __name__ == "__main__":
    with RemoteClient("127.0.0.1", 2200, "foo", "../client/keys/id_rsa.key") as client:
        print(client.check_version("test_plugin", 1.0))
<commit_msg>Added update_client and get_data methods to RemoteClient class<commit_after>import paramiko
import socket
import json

from exceptions import NoConnectionError, RemoteCommandFailedError

class RemoteClient(object):
    def __init__(self, hostname, port, username, key_path):
        self.hostname = hostname
        self.port = port
        self.username = username
        self.key_path = key_path

    def open(self):
        print("Starting SSH connection to client...")
        self._client = paramiko.SSHClient()
        self._client.set_missing_host_key_policy(paramiko.WarningPolicy())

        key = paramiko.RSAKey.from_private_key_file(self.key_path)

        self._client.connect(hostname=self.hostname, port=self.port,
                             username=self.username, pkey=key)

        self._transport = self._client.get_transport()
        self._channel = self._transport.open_session()

        print("Connected to client!")

    def _execute_command(self, method, arguments_dict):
        # Build up a dictionary to hold the command, dump it as json, append a newline to
        # the JSON string (to signify the end of a comamnd to the remote system) and then
        # send it down the SSH connection
        command = {"method": method, "arguments": arguments_dict}
        command_string = json.dumps(command) + "\n"

        # If self does not have the _channel attribute, then the client has not been
        # connected yet!
        try:
            self._channel.send(command_string)
        except AttributeError:
            raise NoConnectionError

        # Now receive the response from the channel - To ensure that we receive data even if
        # is is larger than the channel's buffer we keep receiving data until we recieve a
        # newline character signifying the end of the transmission
        recieve_buffer = bytearray()
        end_of_transmission = False
        while(not end_of_transmission):
            recieve_buffer.extend(self._channel.recv(1024))
            if recieve_buffer.endswith(b"\n"):
                end_of_transmission = True

        response = json.loads(recieve_buffer.decode("UTF-8"))
        if response["error"]:
            raise RemoteCommmandFailedError
        else:
            return response["result"]

    def check_version(self, plugin_name, plugin_version):
        arguments_dict = {  "plugin_name": plugin_name,
                            "plugin_version": plugin_version}

        return self._execute_command("check_version", arguments_dict)

    def update_client(self, plugin_name, payload):
        arguments_dict = {  "plugin_name": plugin_name,
                            "payload": payload}

        return self._execute_command("update_client", arguments_dict)

    def get_data(self, plugin_name):
        arguments_dict = {"plugin_name": plugin_name}

        return self._execute_command("get_data", arguments_dict)

    def close(self):
        print("Closing SSH connection to client...")
        self._channel.close()
        self._client.close()
        print("Connection closed!")

    def __enter__(self):
        self.open()
        return self

    def __exit__(self, type, value, traceback):
        self.close()

if __name__ == "__main__":
    with RemoteClient("127.0.0.1", 2200, "foo", "../client/keys/id_rsa.key") as client:
        print(client.get_data("test_plugin"))
<|endoftext|>


<commit_before>from ..kao_generator import KaoGenerator
import unittest
from unittest.mock import Mock

class init(unittest.TestCase):
    """ Test cases of initialization """

    def test_GeneratorBuilt(self):
        """ Test that the generator is built properly """
        generator = Mock()
        genFn = Mock(return_value=generator)
        args = range(3)
        kwargs = {'one': 1, 'two': 2, 'three': 3}

        wrapper = KaoGenerator(genFn, *args, **kwargs)
        genFn.assert_called_once_with(*args, **kwargs)
        self.assertEqual(wrapper.generator, generator)

    def test_EmptyQueue(self):
        """ Test that the queue is empty after initialization """
        genFn = Mock(return_value=None)

        wrapper = KaoGenerator(genFn)
        self.assertEqual(len(wrapper._queue), 0)

class pop(unittest.TestCase):
    """ Test cases of pop """

    def test_valueInQueue(self):
        """ Test that the value from the queue is retrieved """
        genFn = Mock(return_value=None)
        expected = 123

        wrapper = KaoGenerator(genFn)
        wrapper.queue(expected)
        actual = wrapper.pop()
        self.assertEqual(expected, actual)

    def test_noValueInQueue(self):
        """ Test that None is returned when the queue is empty """
        genFn = Mock(return_value=None)
        expected = None

        wrapper = KaoGenerator(genFn)
        self.assertEqual(len(wrapper._queue), 0)
        actual = wrapper.pop()
        self.assertEqual(expected, actual)<commit_msg>Added test for the queue method<commit_after>from ..kao_generator import KaoGenerator
import unittest
from unittest.mock import Mock

class init(unittest.TestCase):
    """ Test cases of initialization """

    def test_GeneratorBuilt(self):
        """ Test that the generator is built properly """
        generator = Mock()
        genFn = Mock(return_value=generator)
        args = range(3)
        kwargs = {'one': 1, 'two': 2, 'three': 3}

        wrapper = KaoGenerator(genFn, *args, **kwargs)
        genFn.assert_called_once_with(*args, **kwargs)
        self.assertEqual(wrapper.generator, generator)

    def test_EmptyQueue(self):
        """ Test that the queue is empty after initialization """
        genFn = Mock(return_value=None)

        wrapper = KaoGenerator(genFn)
        self.assertEqual(len(wrapper._queue), 0)

class queue(unittest.TestCase):
    """ Test cases of queue """

    def test_appended(self):
        """ Test that the value fis appended to the queue """
        genFn = Mock(return_value=None)
        expected = 123

        wrapper = KaoGenerator(genFn)
        wrapper.queue(expected)
        actual = wrapper._queue.popleft()
        self.assertEqual(expected, actual)

class pop(unittest.TestCase):
    """ Test cases of pop """

    def test_valueInQueue(self):
        """ Test that the value from the queue is retrieved """
        genFn = Mock(return_value=None)
        expected = 123

        wrapper = KaoGenerator(genFn)
        wrapper.queue(expected)
        actual = wrapper.pop()
        self.assertEqual(expected, actual)

    def test_noValueInQueue(self):
        """ Test that None is returned when the queue is empty """
        genFn = Mock(return_value=None)
        expected = None

        wrapper = KaoGenerator(genFn)
        self.assertEqual(len(wrapper._queue), 0)
        actual = wrapper.pop()
        self.assertEqual(expected, actual)<|endoftext|>


<commit_before>import simplejson
import urllib
import random
import csv
import zipfile
import tempfile
import urllib2
import os.path
import re

from django.conf import settings
from django.contrib.gis.geos import Point
from django.utils.translation import ugettext_noop
from django.utils.translation import ugettext as _

from molly.apps.places.providers import BaseMapsProvider
from molly.apps.places.models import Entity, EntityType, Source, EntityTypeCategory
from molly.utils.i18n import override, set_name_in_language

from molly.conf.settings import batch

class PostcodesMapsProvider(BaseMapsProvider):
    def __init__(self, codepoint_path, import_areas=None):
        self.codepoint_path = codepoint_path
        self.import_areas = import_areas

    CODEPOINT_OPEN_URL = 'http://freepostcodes.org.uk/static/code-point-open/codepo_gb.zip'

    def _download_codepoint_open(self):

            archive_url = urllib2.urlopen(self.CODEPOINT_OPEN_URL)
            archive_file = open(self.codepoint_path, 'wb')
            archive_file.write(archive_url.read())
            archive_file.close()

    @batch('%d 12 1 1 *' % random.randint(0, 59))
    def import_data(self, metadata, output):

        entity_type, source = self._get_entity_type(), self._get_source()

        if not os.path.exists(self.codepoint_path):
            self._download_codepoint_open()

        try:
            archive = zipfile.ZipFile(self.codepoint_path)
        except zipfile.BadZipfile:
            self._download_codepoint_open()
            archive = zipfile.ZipFile(self.codepoint_path)

        if self.import_areas:
            filenames = ['Code-Point Open/Data/%s.csv' % code.lower() for code in self.import_areas]
        else:
            filenames = [path for path in archive.namelist() if re.match(r'Code\-Point Open\/Data\/[a-z]{1,2}.csv', path)]

        for filename in filenames:
            if hasattr(archive, 'open'):
                f = archive.open(filename)
            else:
                f = tempfile.TemporaryFile()
                f.write(archive.read(filename))
                f.seek(0)
            reader = csv.reader(f)
            self._load_from_csv(reader, entity_type, source)
            del f

    def _load_from_csv(self, reader, entity_type, source):
        j = 0
        for i, line in enumerate(reader):
            postcode_abbrev, (easting, northing) = line[0], line[10:12]
            if postcode_abbrev[-4] != ' ':
                postcode = '%s %s' % (postcode_abbrev[:-3], postcode_abbrev[-3:])
            else:
                postcode = postcode_abbrev
            postcode_abbrev = postcode_abbrev.replace(' ', '')

            try:
                easting, northing = int(easting), int(northing)
            except ValueError:
                continue

            j += 1

            try:
                entity = Entity.objects.get(source=source, _identifiers__scheme='postcode', _identifiers__value=postcode_abbrev)
            except Entity.DoesNotExist:
                entity = Entity(source=source)

            entity.location = Point(easting, northing, srid=27700)
            entity.geometry = entity.location
            entity.primary_type = entity_type

            identifiers = {
                'postcode': postcode_abbrev,
                'postcode-canonical': postcode,
            }
            entity.save(identifiers=identifiers)
            set_name_in_language(entity, 'en',
                                 title = postcode)
            entity.all_types.add(entity_type)
            entity.update_all_types_completion()

    def _get_entity_type(self):
        category, created = EntityTypeCategory.objects.get_or_create(name=ugettext_noop('Uncategorised'))
        entity_type, created = EntityType.objects.get_or_create(
            slug='post-code', category=category)
        entity_type.slug = 'post-code'
        if created:
            entity_type.show_in_nearby_list = False
            entity_type.show_in_category_list = False
        entity_type.save()
        for lang_code, lang_name in settings.LANGUAGES:
            with override(lang_code):
                set_name_in_language(entity_type, lang_code,
                                     verbose_name=_('postcode'),
                                     verbose_name_singular=_('a postcode'),
                                     verbose_name_plural=_('postcodes'))
        return entity_type

    def _get_source(self):
        try:
            source = Source.objects.get(module_name="molly.providers.apps.maps.postcodes")
        except Source.DoesNotExist:
            source = Source(module_name="molly.providers.apps.maps.postcodes")

        source.name = "Postcodes"
        source.save()

        return source
<commit_msg>Correctly normalise postcodes which have districts different from 3 chars long<commit_after>import simplejson
import urllib
import random
import csv
import zipfile
import tempfile
import urllib2
import os.path
import re

from django.conf import settings
from django.contrib.gis.geos import Point
from django.utils.translation import ugettext_noop
from django.utils.translation import ugettext as _

from molly.apps.places.providers import BaseMapsProvider
from molly.apps.places.models import Entity, EntityType, Source, EntityTypeCategory
from molly.utils.i18n import override, set_name_in_language

from molly.conf.settings import batch

class PostcodesMapsProvider(BaseMapsProvider):
    def __init__(self, codepoint_path, import_areas=None):
        self.codepoint_path = codepoint_path
        self.import_areas = import_areas

    CODEPOINT_OPEN_URL = 'http://freepostcodes.org.uk/static/code-point-open/codepo_gb.zip'

    def _download_codepoint_open(self):

            archive_url = urllib2.urlopen(self.CODEPOINT_OPEN_URL)
            archive_file = open(self.codepoint_path, 'wb')
            archive_file.write(archive_url.read())
            archive_file.close()

    @batch('%d 12 1 1 *' % random.randint(0, 59))
    def import_data(self, metadata, output):

        entity_type, source = self._get_entity_type(), self._get_source()

        if not os.path.exists(self.codepoint_path):
            self._download_codepoint_open()

        try:
            archive = zipfile.ZipFile(self.codepoint_path)
        except zipfile.BadZipfile:
            self._download_codepoint_open()
            archive = zipfile.ZipFile(self.codepoint_path)

        if self.import_areas:
            filenames = ['Code-Point Open/Data/%s.csv' % code.lower() for code in self.import_areas]
        else:
            filenames = [path for path in archive.namelist() if re.match(r'Code\-Point Open\/Data\/[a-z]{1,2}.csv', path)]

        for filename in filenames:
            if hasattr(archive, 'open'):
                f = archive.open(filename)
            else:
                f = tempfile.TemporaryFile()
                f.write(archive.read(filename))
                f.seek(0)
            reader = csv.reader(f)
            self._load_from_csv(reader, entity_type, source)
            del f

    def _load_from_csv(self, reader, entity_type, source):
        j = 0
        for i, line in enumerate(reader):
            postcode_abbrev, (easting, northing) = line[0], line[10:12]
            postcode_abbrev = postcode_abbrev.replace(' ', '')

            # Now try to figure out where to put the space in
            if re.match(r'[A-Z][0-9]{2}[A-Z]{2}', postcode_abbrev):
                # A9 9AA
                postcode = '%s %s' % (postcode_abbrev[:2], postcode_abbrev[2:])
            elif re.match(r'[A-Z][0-9]{3}[A-Z]{2}', postcode_abbrev):
                # A99 9AA
                postcode = '%s %s' % (postcode_abbrev[:3], postcode_abbrev[3:])
            elif re.match(r'[A-Z]{2}[0-9]{2}[A-Z]{2}', postcode_abbrev):
                # AA9 9AA
                postcode = '%s %s' % (postcode_abbrev[:3], postcode_abbrev[3:])
            elif re.match(r'[A-Z]{2}[0-9]{3}[A-Z]{2}', postcode_abbrev):
                # AA99 9AA
                postcode = '%s %s' % (postcode_abbrev[:4], postcode_abbrev[4:])
            elif re.match(r'[A-Z][0-9][A-Z][0-9][A-Z]{2}', postcode_abbrev):
                # A9A 9AA
                postcode = '%s %s' % (postcode_abbrev[:3], postcode_abbrev[3:])
            elif re.match(r'[A-Z]{2}[0-9][A-Z][0-9][A-Z]{2}', postcode_abbrev):
                # AA9A 9AA
                postcode = '%s %s' % (postcode_abbrev[:4], postcode_abbrev[4:])
            else:
                postcode = postcode_abbrev

            try:
                easting, northing = int(easting), int(northing)
            except ValueError:
                continue

            j += 1

            try:
                entity = Entity.objects.get(source=source, _identifiers__scheme='postcode', _identifiers__value=postcode_abbrev)
            except Entity.DoesNotExist:
                entity = Entity(source=source)

            entity.location = Point(easting, northing, srid=27700)
            entity.geometry = entity.location
            entity.primary_type = entity_type

            identifiers = {
                'postcode': postcode_abbrev,
                'postcode-canonical': postcode,
            }
            entity.save(identifiers=identifiers)
            set_name_in_language(entity, 'en',
                                 title = postcode)
            entity.all_types.add(entity_type)
            entity.update_all_types_completion()

    def _get_entity_type(self):
        category, created = EntityTypeCategory.objects.get_or_create(name=ugettext_noop('Uncategorised'))
        entity_type, created = EntityType.objects.get_or_create(
            slug='post-code', category=category)
        entity_type.slug = 'post-code'
        if created:
            entity_type.show_in_nearby_list = False
            entity_type.show_in_category_list = False
        entity_type.save()
        for lang_code, lang_name in settings.LANGUAGES:
            with override(lang_code):
                set_name_in_language(entity_type, lang_code,
                                     verbose_name=_('postcode'),
                                     verbose_name_singular=_('a postcode'),
                                     verbose_name_plural=_('postcodes'))
        return entity_type

    def _get_source(self):
        try:
            source = Source.objects.get(module_name="molly.providers.apps.maps.postcodes")
        except Source.DoesNotExist:
            source = Source(module_name="molly.providers.apps.maps.postcodes")

        source.name = "Postcodes"
        source.save()

        return source
<|endoftext|>


<commit_before>"""
This module contains some code copied from unittest2/loader.py and other
code developed in reference to that module and others within unittest2.

unittest2 is Copyright (c) 2001-2010 Python Software Foundation; All
Rights Reserved. See: http://docs.python.org/license.html

"""
import os
import re
import sys
import traceback

try:
    from compiler.consts import CO_GENERATOR
except ImportError:
    # IronPython doesn't have a complier module
    CO_GENERATOR=0x20

try:
    from inspect import isgeneratorfunction # new in 2.6
except ImportError:
    import inspect
    # backported from Python 2.6
    def isgeneratorfunction(func):
        return bool((inspect.isfunction(func) or inspect.ismethod(func)) and
                    func.func_code.co_flags & CO_GENERATOR)

import six

IDENT_RE = re.compile(r'^[_a-zA-Z]\w*$', re.UNICODE)
VALID_MODULE_RE = re.compile(r'[_a-zA-Z]\w*\.py$', re.UNICODE)

def ln(label, char='-', width=70):
    """Draw a divider, with label in the middle.

    >>> ln('hello there')
    '---------------------------- hello there -----------------------------'

    Width and divider char may be specified. Defaults are 70 and '-'
    respectively.

    """
    label_len = len(label) + 2
    chunk = (width - label_len) // 2
    out = '%s %s %s' % (char * chunk, label, char * chunk)
    pad = width - len(out)
    if pad > 0:
        out = out + (char * pad)
    return out

def valid_module_name(path):
    return VALID_MODULE_RE.search(path)

def name_from_path(path):
    # back up to find module root
    parts = []
    path = os.path.normpath(path)
    base = os.path.splitext(path)[0]
    candidate, top = os.path.split(base)
    parts.append(top)
    while candidate:
        if ispackage(candidate):
            candidate, top = os.path.split(candidate)
            parts.append(top)
        else:
            break
    return '.'.join(reversed(parts))

def module_from_name(name):
    __import__(name)
    return sys.modules[name]

def test_from_name(name, module):
    pos = name.find(':')
    index = None
    if pos != -1:
        real_name, digits = name[:pos], name[pos+1:]
        try:
            index = int(digits)
        except ValueError:
            pass
        else:
            name = real_name
    try:
        parent, obj = object_from_name(name, module)
    except (AttributeError, ImportError):
        return None
    return parent, obj, name, index

def object_from_name(name, module=None):
    parts = name.split('.')
    if module is None:
        parts_copy = parts[:]
        while parts_copy:
            try:
                module = __import__('.'.join(parts_copy))
                break
            except ImportError:
                del parts_copy[-1]
                if not parts_copy:
                    raise
        parts = parts[1:]

    parent = None
    obj = module
    for part in parts:
        parent, obj = obj, getattr(obj, part)
    return parent, obj

def ispackage(path):
    """Is this path a package directory?"""
    if os.path.isdir(path):
        # at least the end of the path must be a legal python identifier
        # and __init__.py[co] must exist
        end = os.path.basename(path)
        if IDENT_RE.match(end):
            for init in ('__init__.py', '__init__.pyc', '__init__.pyo'):
                if os.path.isfile(os.path.join(path, init)):
                    return True
            if sys.platform.startswith('java') and \
                    os.path.isfile(os.path.join(path, '__init__$py.class')):
                return True
    return False

def ensure_importable(dirname):
    if not dirname in sys.path:
        sys.path.insert(0, dirname)

def isgenerator(obj):
    return (isgeneratorfunction(obj)
            or getattr(obj, 'testGenerator', None) is not None)

def safe_decode(string):
    if string is None:
        return string
    try:
        return string.decode()
    except UnicodeDecodeError:
        pass
    try:
        return string.decode('utf-8')
    except UnicodeDecodeError:
        return six.u('<unable to decode>')

def exc_info_to_string(err, test):
    formatTraceback = getattr(test, 'formatTraceback', None)
    if formatTraceback is not None:
        return test.formatTraceback(err)
    else:
        return format_traceback(test, err)

def format_traceback(test, err):
    """Converts a sys.exc_info()-style tuple of values into a string."""
    exctype, value, tb = err
    # Skip test runner traceback levels
    while tb and _is_relevant_tb_level(tb):
        tb = tb.tb_next
    failure = getattr(test, 'failureException', AssertionError)
    if exctype is failure:
        # Skip assert*() traceback levels
        length = _count_relevant_tb_levels(tb)
        msgLines = traceback.format_exception(exctype, value, tb, length)
    else:
        msgLines = traceback.format_exception(exctype, value, tb)
    return ''.join(msgLines)

def _is_relevant_tb_level(tb):
    return '__unittest' in tb.tb_frame.f_globals

def _count_relevant_tb_levels(tb):
    length = 0
    while tb and not _is_relevant_tb_level(tb):
        length += 1
        tb = tb.tb_next
    return length

class _WritelnDecorator(object):
    """Used to decorate file-like objects with a handy 'writeln' method"""
    def __init__(self, stream):
        self.stream = stream

    def __getattr__(self, attr):
        if attr in ('stream', '__getstate__'):
            raise AttributeError(attr)
        return getattr(self.stream, attr)

    def write(self, arg):
        self.stream.write(arg)

    def writeln(self, arg=None):
        if arg:
            self.stream.write(arg)
        self.stream.write('\n') # text-mode streams translate to \r\n if needed
<commit_msg>Move common func to util<commit_after>"""
This module contains some code copied from unittest2/loader.py and other
code developed in reference to that module and others within unittest2.

unittest2 is Copyright (c) 2001-2010 Python Software Foundation; All
Rights Reserved. See: http://docs.python.org/license.html

"""
import os
import re
import sys
import traceback

try:
    from compiler.consts import CO_GENERATOR
except ImportError:
    # IronPython doesn't have a complier module
    CO_GENERATOR=0x20

try:
    from inspect import isgeneratorfunction # new in 2.6
except ImportError:
    import inspect
    # backported from Python 2.6
    def isgeneratorfunction(func):
        return bool((inspect.isfunction(func) or inspect.ismethod(func)) and
                    func.func_code.co_flags & CO_GENERATOR)

import six

IDENT_RE = re.compile(r'^[_a-zA-Z]\w*$', re.UNICODE)
VALID_MODULE_RE = re.compile(r'[_a-zA-Z]\w*\.py$', re.UNICODE)

def ln(label, char='-', width=70):
    """Draw a divider, with label in the middle.

    >>> ln('hello there')
    '---------------------------- hello there -----------------------------'

    Width and divider char may be specified. Defaults are 70 and '-'
    respectively.

    """
    label_len = len(label) + 2
    chunk = (width - label_len) // 2
    out = '%s %s %s' % (char * chunk, label, char * chunk)
    pad = width - len(out)
    if pad > 0:
        out = out + (char * pad)
    return out

def valid_module_name(path):
    return VALID_MODULE_RE.search(path)

def name_from_path(path):
    # back up to find module root
    parts = []
    path = os.path.normpath(path)
    base = os.path.splitext(path)[0]
    candidate, top = os.path.split(base)
    parts.append(top)
    while candidate:
        if ispackage(candidate):
            candidate, top = os.path.split(candidate)
            parts.append(top)
        else:
            break
    return '.'.join(reversed(parts))

def module_from_name(name):
    __import__(name)
    return sys.modules[name]

def test_from_name(name, module):
    pos = name.find(':')
    index = None
    if pos != -1:
        real_name, digits = name[:pos], name[pos+1:]
        try:
            index = int(digits)
        except ValueError:
            pass
        else:
            name = real_name
    try:
        parent, obj = object_from_name(name, module)
    except (AttributeError, ImportError):
        return None
    return parent, obj, name, index

def object_from_name(name, module=None):
    parts = name.split('.')
    if module is None:
        parts_copy = parts[:]
        while parts_copy:
            try:
                module = __import__('.'.join(parts_copy))
                break
            except ImportError:
                del parts_copy[-1]
                if not parts_copy:
                    raise
        parts = parts[1:]

    parent = None
    obj = module
    for part in parts:
        parent, obj = obj, getattr(obj, part)
    return parent, obj

def name_from_args(name, index, args):
    summary = ', '.join(repr(arg) for arg in args)
    return '%s:%s\n%s' % (name, index + 1, summary[:79])

def ispackage(path):
    """Is this path a package directory?"""
    if os.path.isdir(path):
        # at least the end of the path must be a legal python identifier
        # and __init__.py[co] must exist
        end = os.path.basename(path)
        if IDENT_RE.match(end):
            for init in ('__init__.py', '__init__.pyc', '__init__.pyo'):
                if os.path.isfile(os.path.join(path, init)):
                    return True
            if sys.platform.startswith('java') and \
                    os.path.isfile(os.path.join(path, '__init__$py.class')):
                return True
    return False

def ensure_importable(dirname):
    if not dirname in sys.path:
        sys.path.insert(0, dirname)

def isgenerator(obj):
    return (isgeneratorfunction(obj)
            or getattr(obj, 'testGenerator', None) is not None)

def safe_decode(string):
    if string is None:
        return string
    try:
        return string.decode()
    except UnicodeDecodeError:
        pass
    try:
        return string.decode('utf-8')
    except UnicodeDecodeError:
        return six.u('<unable to decode>')

def exc_info_to_string(err, test):
    formatTraceback = getattr(test, 'formatTraceback', None)
    if formatTraceback is not None:
        return test.formatTraceback(err)
    else:
        return format_traceback(test, err)

def format_traceback(test, err):
    """Converts a sys.exc_info()-style tuple of values into a string."""
    exctype, value, tb = err
    # Skip test runner traceback levels
    while tb and _is_relevant_tb_level(tb):
        tb = tb.tb_next
    failure = getattr(test, 'failureException', AssertionError)
    if exctype is failure:
        # Skip assert*() traceback levels
        length = _count_relevant_tb_levels(tb)
        msgLines = traceback.format_exception(exctype, value, tb, length)
    else:
        msgLines = traceback.format_exception(exctype, value, tb)
    return ''.join(msgLines)

def _is_relevant_tb_level(tb):
    return '__unittest' in tb.tb_frame.f_globals

def _count_relevant_tb_levels(tb):
    length = 0
    while tb and not _is_relevant_tb_level(tb):
        length += 1
        tb = tb.tb_next
    return length

class _WritelnDecorator(object):
    """Used to decorate file-like objects with a handy 'writeln' method"""
    def __init__(self, stream):
        self.stream = stream

    def __getattr__(self, attr):
        if attr in ('stream', '__getstate__'):
            raise AttributeError(attr)
        return getattr(self.stream, attr)

    def write(self, arg):
        self.stream.write(arg)

    def writeln(self, arg=None):
        if arg:
            self.stream.write(arg)
        self.stream.write('\n') # text-mode streams translate to \r\n if needed
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# boto documentation build configuration file, created by
# sphinx-quickstart on Tue Sep 15 13:34:43 2009.
#
# This file is execfile()d with the current directory set to its containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys, os

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.append(os.path.abspath('.'))

# -- General configuration -----------------------------------------------------

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinx.ext.todo']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'boto'
copyright = u'2009, Mitch Garnaat'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '1.8'
# The full version, including alpha/beta/rc tags.
release = "HEAD" #'1.8d'
try:
    import subprocess
    p = subprocess.Popen(["svn info ../boto | grep Revision | awk '{print $2}'"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
    release = 'r%s' % p.stdout.read().strip()
except:
    pass

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of documents that shouldn't be included in the build.
#unused_docs = []

# List of directories, relative to source directory, that shouldn't be searched
# for source files.
exclude_trees = []

# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# -- Options for HTML output ---------------------------------------------------

# The theme to use for HTML and HTML Help pages.  Major themes that come with
# Sphinx are currently 'default' and 'sphinxdoc'.
html_theme = 'sphinxdoc'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_use_modindex = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = ''

# Output file base name for HTML help builder.
htmlhelp_basename = 'botodoc'

# -- Options for LaTeX output --------------------------------------------------

# The paper size ('letter' or 'a4').
#latex_paper_size = 'letter'

# The font size ('10pt', '11pt' or '12pt').
#latex_font_size = '10pt'

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
  ('index', 'boto.tex', u'boto Documentation',
   u'Mitch Garnaat', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# Additional stuff for the LaTeX preamble.
#latex_preamble = ''

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_use_modindex = True

# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {'http://docs.python.org/': None}
<commit_msg>Updated path.<commit_after># -*- coding: utf-8 -*-
#
# boto documentation build configuration file, created by
# sphinx-quickstart on Tue Sep 15 13:34:43 2009.
#
# This file is execfile()d with the current directory set to its containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys, os

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.append(os.path.abspath('.'))

# -- General configuration -----------------------------------------------------

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinx.ext.todo']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'boto'
copyright = u'2009, Mitch Garnaat'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '1.8'
# The full version, including alpha/beta/rc tags.
release = "HEAD" #'1.8d'
try:
    import subprocess
    p = subprocess.Popen(["svn info ../../boto | grep Revision | awk '{print $2}'"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
    release = 'r%s' % p.stdout.read().strip()
    print p.stderr.read()
except:
    pass

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of documents that shouldn't be included in the build.
#unused_docs = []

# List of directories, relative to source directory, that shouldn't be searched
# for source files.
exclude_trees = []

# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# -- Options for HTML output ---------------------------------------------------

# The theme to use for HTML and HTML Help pages.  Major themes that come with
# Sphinx are currently 'default' and 'sphinxdoc'.
html_theme = 'sphinxdoc'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
html_title = "boto v%s (r%s)" % (version, release)

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_use_modindex = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = ''

# Output file base name for HTML help builder.
htmlhelp_basename = 'botodoc'

# -- Options for LaTeX output --------------------------------------------------

# The paper size ('letter' or 'a4').
#latex_paper_size = 'letter'

# The font size ('10pt', '11pt' or '12pt').
#latex_font_size = '10pt'

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
  ('index', 'boto.tex', u'boto Documentation',
   u'Mitch Garnaat', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# Additional stuff for the LaTeX preamble.
#latex_preamble = ''

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_use_modindex = True

# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {'http://docs.python.org/': None}
<|endoftext|>


<commit_before>from kinetic_datanator.core import data_source
import sqlalchemy.ext.declarative
import pandas as pd
from sqlalchemy import Column, Integer, String, Float
# from six.moves.urllib.request import urlretrieve
# import gzip
# from six import BytesIO

Base = sqlalchemy.ext.declarative.declarative_base()
test = True

class UniprotData(Base):
    """ Represents protein interactions in from the IntAct Database

    Attributes:
        Index (:obj:`int`): Index of the DB
        interactor_a (:obj:`str`): represents participant A
        interactor_b (:obj:`str`): represents participant B
        publications (:obj:`str`): resource
        interaction (:obj:`str`): interaction ID
        feature_a (:obj:`str`): binding site of participant A
        feature_b (:obj:`str`): binding site of participant B
        stoich_a (:obj:`str`): stoichiometry of participant A
        stoich_b (:obj:`str`): stoichiometry of participant B

    """

    __tablename__ = 'uniprot'
    index = Column(Integer, primary_key = True)
    uniprot_id = Column(String(255), unique = True)
    entry_name = Column(String(255))
    gene_name = Column(String(255))
    protein_name = Column(String(255))
    canonical_sequence = Column(String(255))
    length = Column(Integer)
    mass = Column(String(255))
    ec_number = Column(String(255))
    entrez_id = Column(Integer)
    status  = Column(String(255))

class Uniprot(data_source.HttpDataSource):
    """

    """
    base_model = Base
    ENDPOINT_DOMAINS = {'uniprot' : 'http://www.uniprot.org/uniprot/?query=*&fil=reviewed%3Ayes#',
                        'uniprot-test': 'http://www.uniprot.org/uniprot/?sort=&desc=&compress=no&query=&fil=organism:%22Rattus%20norvegicus%20(Rat)%20[10116]%22%20AND%20reviewed:yes&limit=10&force=no&preview=true&format=tab&columns=id,entry%20name,genes(PREFERRED),protein%20names,sequence,length,mass,ec,database(GeneID),reviewed'}

    def load_content(self):

        if test:
            self.write_data_to_txt()
            pand = pd.read_csv('kinetic_datanator/data_source/cache/uniprot-test.txt', delimiter = '\t')
        else:
            pand = pd.read_csv('kinetic_datanator/data_source/cache/uniprot-all.txt', delimiter = '\t')

        new_columns = ['uniprot_id', 'entry_name', 'gene_name', 'protein_name', 'canonical_sequence', 'length', 'mass',
            'ec_number', 'entrez_id', 'status']
        pand.columns = new_columns

        pand['entrez_id'] = pand['entrez_id'].str.replace(';', '')

        if not self.max_entries == float('inf'):
            pand = pand[0:self.max_entries]

        pand.to_sql(name = 'uniprot', con=self.engine, if_exists = 'append', chunksize = 1000)
        self.session.commit()

    def write_data_to_txt(self):

        database_url = self.ENDPOINT_DOMAINS['uniprot-test']
        req = self.requests_session
        response = req.get(database_url)
        response.raise_for_status()
        content = str(response.content.decode('utf-8')).split('\n')
        with open(self.cache_dirname+'/uniprot-test.txt', 'w') as text_file:
            for line in content:
                text_file.write(line+'\n')
<commit_msg>fixing cache path causing error<commit_after>from kinetic_datanator.core import data_source
import sqlalchemy.ext.declarative
import pandas as pd
from sqlalchemy import Column, Integer, String, Float
# from six.moves.urllib.request import urlretrieve
# import gzip
# from six import BytesIO

Base = sqlalchemy.ext.declarative.declarative_base()
test = True

class UniprotData(Base):
    """ Represents protein interactions in from the IntAct Database

    Attributes:
        Index (:obj:`int`): Index of the DB
        interactor_a (:obj:`str`): represents participant A
        interactor_b (:obj:`str`): represents participant B
        publications (:obj:`str`): resource
        interaction (:obj:`str`): interaction ID
        feature_a (:obj:`str`): binding site of participant A
        feature_b (:obj:`str`): binding site of participant B
        stoich_a (:obj:`str`): stoichiometry of participant A
        stoich_b (:obj:`str`): stoichiometry of participant B

    """

    __tablename__ = 'uniprot'
    index = Column(Integer, primary_key = True)
    uniprot_id = Column(String(255), unique = True)
    entry_name = Column(String(255))
    gene_name = Column(String(255))
    protein_name = Column(String(255))
    canonical_sequence = Column(String(255))
    length = Column(Integer)
    mass = Column(String(255))
    ec_number = Column(String(255))
    entrez_id = Column(Integer)
    status  = Column(String(255))

class Uniprot(data_source.HttpDataSource):
    """

    """
    base_model = Base
    ENDPOINT_DOMAINS = {'uniprot' : 'http://www.uniprot.org/uniprot/?query=*&fil=reviewed%3Ayes#',
                        'uniprot-test': 'http://www.uniprot.org/uniprot/?sort=&desc=&compress=no&query=&fil=organism:%22Rattus%20norvegicus%20(Rat)%20[10116]%22%20AND%20reviewed:yes&limit=10&force=no&preview=true&format=tab&columns=id,entry%20name,genes(PREFERRED),protein%20names,sequence,length,mass,ec,database(GeneID),reviewed'}

    def load_content(self):

        if test:
            self.write_data_to_txt()
            pand = pd.read_csv(self.cache_dirname+'/uniprot-test.txt', delimiter = '\t')
        else:
            pand = pd.read_csv(self.cache_dirname+'/uniprot-all.txt', delimiter = '\t')

        new_columns = ['uniprot_id', 'entry_name', 'gene_name', 'protein_name', 'canonical_sequence', 'length', 'mass',
            'ec_number', 'entrez_id', 'status']
        pand.columns = new_columns

        pand['entrez_id'] = pand['entrez_id'].str.replace(';', '')

        if not self.max_entries == float('inf'):
            pand = pand[0:self.max_entries]

        pand.to_sql(name = 'uniprot', con=self.engine, if_exists = 'append', chunksize = 1000)
        self.session.commit()

    def write_data_to_txt(self):

        database_url = self.ENDPOINT_DOMAINS['uniprot-test']
        req = self.requests_session
        response = req.get(database_url)
        response.raise_for_status()
        content = str(response.content.decode('utf-8')).split('\n')
        with open(self.cache_dirname+'/uniprot-test.txt', 'w') as text_file:
            for line in content:
                text_file.write(line+'\n')
<|endoftext|>


<commit_before># -*- coding: cp949 -*-
from sympy import *

'''2.10.1.1'''
a = Rational(1, 2)
print "a =", a

print "pi**2 =", pi**2

print "pi.evalf() =", pi.evalf()

print "pi + exp(1) =", pi + exp(1)

print "(pi + exp(1)).evalf() =", (pi + exp(1)).evalf()

print "oo =", oo

print "oo > 99999 =", oo > 99999

'''2.10.1.2'''
print "sqrt(2) =", sqrt(2).evalf(100)

print "1/2 + 1/3 =", Rational(1, 2) + Rational(1, 3)

F, r, sigma_max, sf = symbols('F r sigma_max safety_factor')

area = pi * r * r
sigma = F / area
solution = solve([sigma - sigma_max/sf], r)
print "solution =", simplify(solution)

'''2.10.1.3'''

x = Symbol('x')
y = Symbol('y')

print "x+y+x-y =", x+y+x-y

print "(x+y)**2 =", (x+y)**2

'''2.10.2.1'''

print "expand((x+y)**3) =", expand((x+y)**3)

print "expand(x+y, complex=True) =", expand(x+y,complex=True)

print "expand(cos(x+y), trig=True) =", expand(cos(x+y), trig=True)

'''2.10.2.2'''

print "simplify((x+x*y)/x) =", simplify((x+x*y)/x)

'''2.10.3.1'''
print "limit(sin(x)/x,x,0) =", limit(sin(x)/x, x, 0)

print "limit(x,x,oo) =", limit(x, x, oo)

print "limit(1/x,x,oo) =", limit(1/x, x, oo)

print "limit(x**x,x,0) =", limit(x**x, x, 0)

'''2.10.3.2'''

print "diff(sin(x), x) =", diff(sin(x), x)

print "diff(sin(2*x), x) =", diff(sin(2*x), x)

print "diff(tan(x), x) =", diff(tan(x), x)

print "limit((tan(x+y)-tan(x))/y,y,0) =", limit((tan(x+y)-tan(x))/y,y,0)

print "diff(sin(2*x), x, 1) =", diff(sin(2*x), x, 1)

print "diff(sin(2*x), x, 2) =", diff(sin(2*x), x, 2)

print "diff(sin(2*x), x, 3) =", diff(sin(2*x), x, 3)

'''2.10.3.3'''

print "series(cos(x), x) =", series(cos(x), x)

print "series(1/cos(x), x) =", series(1/cos(x), x)

# Series  Ͽ ׷ ׸
import pylab
x_deg = pylab.arange(-90, 90+1)
x_rad = pylab.deg2rad(x_deg)
y_cos = pylab.cos(x_rad)
y_series_1 = 1*pylab.ones_like(x_rad)
y_series_2 = 1 - x_rad**2/2
y_series_3 = 1 - x_rad**2/2 + x_rad**4/24

pylab.plot(x_deg, y_cos, label='cos')
pylab.plot(x_deg, y_series_1, label='series 1')
pylab.plot(x_deg, y_series_2, label='series 2')
pylab.plot(x_deg, y_series_3, label='series 3')
pylab.grid()
pylab.legend(loc=0)
#pylab.show()

'''2.10.3.5'''
print "integrate(6*x**5, x) =", integrate(6*x**5, x)

print "integrate(sin(x), x) =", integrate(sin(x), x)

print "integrate(log(x), x) =", integrate(log(x), x)

print "integrate(2*x + sinh(x), x) =", integrate(2*x + sinh(x), x)

print "integrate(exp(-x**2)*erf(x), x) =", integrate(exp(-x**2)*erf(x), x)

print "integrate(x**3, (x, -1, 1)) =", integrate(x**3, (x, -1, 1))

print "integrate(sin(x), (x, 0, pi/2)) =", integrate(sin(x), (x, 0, pi/2))

print "integrate(cos(x), (x, -pi/2, pi/2)) =", integrate(cos(x), (x, -pi/2, pi/2))

print "integrate(exp(-x), (x, 0, oo)) =", integrate(exp(-x), (x, 0, oo))

print "integrate(exp(-x**2), (x, -oo, oo)) =", integrate(exp(-x**2), (x, -oo, oo))

'''2.10.4'''
print "solve(x**4 - 1, x) =", solve(x**4 - 1, x)

print "solve([x + 5*y - 2, -3*x + 6*y - 15], [x,y]) =", solve([x + 5*y - 2, -3*x + 6*y - 15], [x,y])

print "solve(exp(x) + 1, x) =", solve(exp(x) + 1, x)

f = x**4 - 3*x**2 + 1
print "factor(f) =", factor(f)

print "factor(f, modulus=5) =", factor(f, modulus=5)

print "satisfiable(x&y) =", satisfiable(x & y)
print "satisfiable(x^y) =", satisfiable(x ^ y)

'''2.10.5.1'''
print "Matrix([[1,0], [0,1]]) =", Matrix([[1,0], [0,1]])
A = Matrix([[1,x], [y,1]])
print "A =", A
print "A**2 =", A**2
<commit_msg>symbolic 예제 ~2.10.5.2<commit_after># -*- coding: cp949 -*-
from sympy import *

'''2.10.1.1'''
a = Rational(1, 2)
print "a =", a

print "pi**2 =", pi**2

print "pi.evalf() =", pi.evalf()

print "pi + exp(1) =", pi + exp(1)

print "(pi + exp(1)).evalf() =", (pi + exp(1)).evalf()

print "oo =", oo

print "oo > 99999 =", oo > 99999

'''2.10.1.2'''
print "sqrt(2) =", sqrt(2).evalf(100)

print "1/2 + 1/3 =", Rational(1, 2) + Rational(1, 3)

F, r, sigma_max, sf = symbols('F r sigma_max safety_factor')

area = pi * r * r
sigma = F / area
solution = solve([sigma - sigma_max/sf], r)
print "solution =", simplify(solution)

'''2.10.1.3'''

x = Symbol('x')
y = Symbol('y')

print "x+y+x-y =", x+y+x-y

print "(x+y)**2 =", (x+y)**2

'''2.10.2.1'''

print "expand((x+y)**3) =", expand((x+y)**3)

print "expand(x+y, complex=True) =", expand(x+y,complex=True)

print "expand(cos(x+y), trig=True) =", expand(cos(x+y), trig=True)

'''2.10.2.2'''

print "simplify((x+x*y)/x) =", simplify((x+x*y)/x)

'''2.10.3.1'''
print "limit(sin(x)/x,x,0) =", limit(sin(x)/x, x, 0)

print "limit(x,x,oo) =", limit(x, x, oo)

print "limit(1/x,x,oo) =", limit(1/x, x, oo)

print "limit(x**x,x,0) =", limit(x**x, x, 0)

'''2.10.3.2'''

print "diff(sin(x), x) =", diff(sin(x), x)

print "diff(sin(2*x), x) =", diff(sin(2*x), x)

print "diff(tan(x), x) =", diff(tan(x), x)

print "limit((tan(x+y)-tan(x))/y,y,0) =", limit((tan(x+y)-tan(x))/y,y,0)

print "diff(sin(2*x), x, 1) =", diff(sin(2*x), x, 1)

print "diff(sin(2*x), x, 2) =", diff(sin(2*x), x, 2)

print "diff(sin(2*x), x, 3) =", diff(sin(2*x), x, 3)

'''2.10.3.3'''

print "series(cos(x), x) =", series(cos(x), x)

print "series(1/cos(x), x) =", series(1/cos(x), x)

# Series  Ͽ ׷ ׸
import pylab
x_deg = pylab.arange(-90, 90+1)
x_rad = pylab.deg2rad(x_deg)
y_cos = pylab.cos(x_rad)
y_series_1 = 1*pylab.ones_like(x_rad)
y_series_2 = 1 - x_rad**2/2
y_series_3 = 1 - x_rad**2/2 + x_rad**4/24

pylab.plot(x_deg, y_cos, label='cos')
pylab.plot(x_deg, y_series_1, label='series 1')
pylab.plot(x_deg, y_series_2, label='series 2')
pylab.plot(x_deg, y_series_3, label='series 3')
pylab.grid()
pylab.legend(loc=0)
#pylab.show()

'''2.10.3.5'''
print "integrate(6*x**5, x) =", integrate(6*x**5, x)

print "integrate(sin(x), x) =", integrate(sin(x), x)

print "integrate(log(x), x) =", integrate(log(x), x)

print "integrate(2*x + sinh(x), x) =", integrate(2*x + sinh(x), x)

print "integrate(exp(-x**2)*erf(x), x) =", integrate(exp(-x**2)*erf(x), x)

print "integrate(x**3, (x, -1, 1)) =", integrate(x**3, (x, -1, 1))

print "integrate(sin(x), (x, 0, pi/2)) =", integrate(sin(x), (x, 0, pi/2))

print "integrate(cos(x), (x, -pi/2, pi/2)) =", integrate(cos(x), (x, -pi/2, pi/2))

print "integrate(exp(-x), (x, 0, oo)) =", integrate(exp(-x), (x, 0, oo))

print "integrate(exp(-x**2), (x, -oo, oo)) =", integrate(exp(-x**2), (x, -oo, oo))

'''2.10.4'''
print "solve(x**4 - 1, x) =", solve(x**4 - 1, x)

print "solve([x + 5*y - 2, -3*x + 6*y - 15], [x,y]) =", solve([x + 5*y - 2, -3*x + 6*y - 15], [x,y])

print "solve(exp(x) + 1, x) =", solve(exp(x) + 1, x)

f = x**4 - 3*x**2 + 1
print "factor(f) =", factor(f)

print "factor(f, modulus=5) =", factor(f, modulus=5)

print "satisfiable(x&y) =", satisfiable(x & y)
print "satisfiable(x^y) =", satisfiable(x ^ y)

'''2.10.5.1'''
print "Matrix([[1,0], [0,1]]) =", Matrix([[1,0], [0,1]])
A = Matrix([[1,x], [y,1]])
print "A =", A
print "A**2 =", A**2

'''2.10.5.2'''
f = Function('f')
print "f(x).diff(x, x) + f(x) =", f(x).diff(x, x) + f(x)
print "dsolve(f(x).diff(x, x) + f(x)) =", dsolve(f(x).diff(x, x) + f(x))
<|endoftext|>