<commit_before>from pyparsing import Literal, Word, Group, Combine, Optional, Forward, alphanums, SkipTo, LineEnd, nums, delimitedList
import string
import logging

from machine import Machine
from monoid import Monoid

import sys
class ParserException(Exception):
    pass

class DefinitionParser:
    _str = set([str, unicode])
    _deep_cases = [ "NOM" , "ACC" , "DAT" , "INS" , "OBL"
                    , "SUB", "SUE", "DEL"         # ON
                    , "ILL", "INE", "ELA"         # IN
                    , "ALL", "ADE", "ABL"         # AT
                    ]

    lb = "["
    rb = "]"
    lp = "("
    rp = ")"

    def_sep = ":"
    arg_sep = ","
    part_sep = ";"
    comment_sep = "%"
    prime = "'"
    hyphen = "-"

    def __init__(self):
        self.init_parser()

    def init_parser(self):
        self.id = Word(nums)
        self.lb_lit = Literal(DefinitionParser.lb)
        self.rb_lit = Literal(DefinitionParser.rb)
        self.lp_lit = Literal(DefinitionParser.lp)
        self.rp_lit = Literal(DefinitionParser.rp)

        self.def_sep_lit = Literal(DefinitionParser.def_sep)
        self.arg_sep_lit = Literal(DefinitionParser.arg_sep)
        self.part_sep_lit = Literal(DefinitionParser.part_sep)
        self.comment_sep_lit = Literal(DefinitionParser.comment_sep)
        self.prime_lit = Literal(DefinitionParser.prime)
        self.hyphen_lit = Literal(DefinitionParser.hyphen)

        #self.deep_cases = (Literal("NOM") | Literal("ACC") | Literal("DAT") | Literal("INS") | Literal("ABL"))
        self.deep_cases = reduce(lambda a, b: a | b, (Literal(dc) for dc in DefinitionParser._deep_cases))

        self.unary = Combine(Optional("-") + Word(string.lowercase + "_") + Optional(Word(nums))) | self.deep_cases
        self.binary = Word(string.uppercase + "_" + nums)
        self.dontcare = SkipTo(LineEnd())

        # main expression
        self.expression = Forward()

        # "enumerable expression"
        # D -> E | E, D
        self.definition = Group(delimitedList(self.expression, delim=DefinitionParser.arg_sep))
        self.expression << Group(
                            # E -> U
                            (self.unary) ^

                            # E -> U [ D ]
                            (self.unary + self.lb_lit.suppress() + self.definition + self.rb_lit.suppress() ) ^

                            # E -> U ( U ) | U ( U [ E ] )
                            (self.unary + self.lp_lit + self.unary + Optional(self.lb_lit.suppress() + self.expression + self.rb_lit.suppress()) + self.rp_lit ) ^

                            # E -> U B
                            (self.unary + self.binary) ^

                            # E -> U B E
                            (self.unary + self.binary + self.expression) ^

                            # E -> B E
                            (self.binary + self.expression) ^

                            # E -> B [ E ]
                            (self.binary + self.lb_lit.suppress() + self.expression + self.rb_lit.suppress()) ^

                            # E -> B [ E ; E ]
                            (self.binary + self.lb_lit.suppress() + self.expression + self.part_sep_lit.suppress() + self.expression + self.rb_lit.suppress()) ^

                            # E -> [ E ] B
                            (self.lb_lit.suppress() + self.expression + self.rb_lit.suppress() + self.binary) ^

                            # E -> [ E ] B [ E ]
                            (self.lb_lit.suppress() + self.expression + self.rb_lit.suppress() + self.binary + self.lb_lit.suppress() + self.expression + self.rb_lit.suppress()) ^

                            # E -> [ E ] B E
                            (self.lb_lit.suppress() + self.expression + self.rb_lit.suppress() + self.binary + self.expression ) ^

                            # E -> 'B
                            (self.prime_lit + self.binary) ^

                            # E -> B'
                            (self.binary + self.prime_lit)
                           )

        self.hu, self.pos, self.en, self.lt, self.pt = (Word(alphanums + "#-/_" ),) * 5
        self.word = Group(self.hu + self.pos + self.en + self.lt + self.pt)

        # S -> W : D | W : D % _
        self.sen = (self.id + self.word + self.def_sep_lit.suppress() + self.definition + Optional(self.comment_sep_lit + self.dontcare).suppress()) + LineEnd()

    def parse(self, s):
        return self.sen.parseString(s).asList()

    @classmethod
    def __flatten__(cls, _l):
        _str = cls._str
        result = None
        if type(_l) == list:
            if len(_l) == 1 and type(_l[0]) == list and len(_l[0]) == 1 and type(_l[0][0]) in _str:
                result = DefinitionParser.__flatten__(_l[0])
            else:
                result = [DefinitionParser.__flatten__(e) for e in _l]
        else:
            result = _l
        return result

    @classmethod
    def _is_binary(cls, s):
        return type(s) in cls._str and s.isupper() and not s in cls._deep_cases

    @classmethod
    def _is_unary(cls, s):
        # TODO unaries can contain hyphens
        return type(s) in cls._str and s.islower() or s in cls._deep_cases

    @classmethod
    def _is_deep_case(cls, s):
        return s in cls._deep_cases

    @classmethod
    def _parse_expr(cls, l):
        is_binary = cls._is_binary
        is_unary = cls._is_unary
        _str = cls._str
        if len(l) == 1:
            # unary
            # later here should be a lookup to find out if we already know this machine
            if type(l[0]) == list:
                return cls._parse_expr(l[0])
            else:
                if is_unary(l[0]):
                    return Machine(Monoid(l[0]))
                else:
                    raise ParserException("Only lower case strings or deep cases or filled binaries can be at unary position(" + str(l[0]) + ")")

        elif len(l) == 2:

            if type(l[0]) in _str:
                if is_unary(l[0]):
                    # unary with one partition
                    m = Machine(Monoid(l[0]))
                    m.base.partitions.append([])
                    m.base.partitions[1].append(cls._parse_expr(l[1]))
                    return m
                elif is_binary(l[0]):
                    # binary with right partition filled - prefix
                    m = Machine(Monoid(l[0]))
                    m.base.partitions.append([])
                    m.base.partitions.append([])
                    if type(l[1]) == list and len(l[1]) > 1:
                        raise ParserException("Binary primitives have to have one property per argument")
                    m.base.partitions[2].append(cls._parse_expr(l[1]))

                    return m
            elif type(l[1]) in _str:
                if is_binary(l[1]):
                    # binary with left partition filled - infix
                    m = Machine(Monoid(l[1]))
                    m.base.partitions.append([])
                    if type(l[0]) == list and len(l[0]) > 1:
                        raise ParserException("Binary primitives have to have one property per argument")
                    m.base.partitions[1].append(cls._parse_expr(l[0]))
                    m.base.partitions.append([])
                    return m

        elif len(l) == 3:
            # binary with two partitions
            later = None
            m = None
            if type(l[0]) in _str:
                if is_binary(l[0]):
                    m = Machine(Monoid(l[0]))
                    later = l[1], l[2]
                else:
                    raise ParserException("not uppercase string at binary position")
            elif type(l[1]) in _str:
                if is_binary(l[1]):
                    m = Machine(Monoid(l[1]))
                    later = l[0], l[2]
                else:
                    raise ParserException("not uppercase string at binary position")
            else:
                raise ParserException("not uppercase string at any possible binary position")

            m.base.partitions.append([cls._parse_expr(later[0])])
            m.base.partitions.append([cls._parse_expr(later[1])])
            return m

        elif len(l) == 4:
            # U ( U ) rule
            if is_unary(l[0]) and l[1] == "(" and is_unary(l[2]) and l[3] == ")":
                m = Machine(Monoid(l[2]))
                m.base.partitions.append([])
                m.base.partitions[1].append(Machine(Monoid(l[0])))
                return m
            else:
                pe = ParserException("there shouldn't be 4 nodes in a tree only if 2 of them are parentheses")
                logging.debug(str(pe))
                logging.debug(str(l))
                raise pe

    @classmethod
    def __parse_expr_v2(cls, expr):
        """
        creates machines from a parse node and its children
        there should be one handler for every rule
        """
        is_binary = cls._is_binary
        is_unary = cls._is_unary
        is_tree = lambda r: type(r) == list

        # E -> U [ D ]
        # ['unary', [['unary'], ['unary']]]
        if (len(expr) == 2 and
              is_unary(expr[0]) and
              is_tree(expr[1])):
            m = Machine(Monoid(expr[0]))
            for _property in expr[1]:
                m.append(1, cls.__parse_expr_v2(_property))
            return m

        # E -> U ( U ) | U ( U [ E ] )
        # ['unary', '(', 'unary', ')']
        if (len(expr) == 4 and
              is_unary(expr[0]) and
              expr[1] == cls.lp and
              is_unary(expr[2]) and
              expr[3] == cls.rp):
            m = Machine(Monoid(expr[2]))
            m.append(1, Machine(Monoid(expr[0])))
            return m
        # ???

        # E -> U B E
        # ['unary', 'BINARY', ['unary']]
        if (len(expr) == 3 and
              is_unary(expr[0]) and
              is_binary(expr[1]) and
              is_tree(expr[2])):
            m = Machine(Monoid(expr[1]))
            m.append(1, Machine(Monoid(expr[0])))
            m.append(2, cls.__parse_expr_v2(expr[2]))
            return m

        # E -> U B
        # ['unary', 'BINARY']
        if (len(expr) == 2 and
              is_unary(expr[0]) and
              is_binary(expr[1])):
            m = Machine(Monoid(expr[1]))
            m.append(1, Machine(Monoid(expr[0])))
            m.base.partitions.append([])
            return m

        # E -> U
        # ['unary']
        if (len(expr) == 1 and
              is_unary(expr[0])):
            return Machine(Monoid(expr[0]))

        # E -> B [ E ; E ]
        # ['BINARY', ['unary'], ['unary']]
        if (len(expr) == 3 and
              is_binary(expr[0]) and
              is_tree(expr[1]) and
              is_tree(expr[2])):
            m = Machine(Monoid(expr[0]))
            m.append(1, cls.__parse_expr_v2(expr[1]))
            m.append(2, cls.__parse_expr_v2(expr[2]))
            return m

        # E -> [ E ] B [ E ]
        # [['unary'], 'BINARY', ['unary']]
        if (len(expr) == 3 and
              is_tree(expr[0]) and
              is_binary(expr[1]) and
              is_tree(expr[2])):
            m = Machine(Monoid(expr[1]))
            m.append(1, cls.__parse_expr_v2(expr[0]))
            m.append(2, cls.__parse_expr_v2(expr[2]))
            return m

        # E -> B [ E ]
        # ['BINARY', ['unary']]
        if (len(expr) == 2 and
            is_binary(expr[0]) and
            is_tree(expr[1])):
            m = Machine(Monoid(expr[0]))
            m.append(2, cls.__parse_expr_v2(expr[1]))
            return m

        # E -> [ E ] B
        # [['unary'], 'BINARY']
        if (len(expr) == 2 and
            is_tree(expr[0]) and
            is_binary(expr[1])):
            m = Machine(Monoid(expr[1]))
            m.append(1, cls.__parse_expr_v2(expr[0]))
            m.base.partitions.append([])
            return m

        # E -> B E
        # ['BINARY', ['unary']]
        if (len(expr) == 2 and
            is_binary(expr[0]) and
            is_tree(expr[1])):
            m = Machine(Monoid(expr[0]))
            m.append(2, cls.__parse_expr_v2(expr[1]))
            return m

        # E -> 'B
        # ["'", 'BINARY']
        if (len(expr) == 2 and
            expr[0] == cls.prime and
            is_binary(expr[1])):
            raise NotImplementedError()

        # E -> B'
        # ['BINARY', "'"]
        if (len(expr) == 2 and
            is_binary(expr[0]) and
            expr[1] == cls.prime):
            raise NotImplementedError()

        pe = ParserException("Unknown expression in definition")
        logging.debug(str(pe))
        logging.debug(expr)
        raise pe

    def parse_into_machines(self, s):
        parsed = self.parse(s)
        #parsed = DefinitionParser.__flatten__(parsed)

        machine = Machine(Monoid(parsed[1][2]))
        machine.base.partitions.append([])
        for d in parsed[2]:
            machine.append(1, DefinitionParser.__parse_expr_v2(d))
        return (machine, tuple(parsed[1]))

def read(f):
    from langtools.utils import accents
    d = {}
    dp = DefinitionParser()
    for line in f:
        l = line.strip()
        if len(l) == 0:
            continue
        if l.startswith("#"):
            continue
        m, t = dp.parse_into_machines(line.strip())
        tl = list(t)
        tl[0] = accents.proszeky_to_utf(tl[0].encode("utf-8"))
        d[tuple(tl)] = m
    return d

if __name__ == "__main__":
    dp = DefinitionParser()
    pstr = sys.argv[-1]
    if sys.argv[1] == "-g":
      m, _ = dp.parse_into_machines(pstr)
      print m.to_dot(True)
    else:
      print dp.parse(pstr)
    #print dp.parse("1 a1bra1zat N expression facies mina: HAS mouth[open], ISA dog,   [a]HAS[b]")
    #m = dp.parse_into_machines("1 a1bra1zat N expression facies mina: HAS[mouth[open]], ISA dog,   [a]HAS[b]")
    #m = dp.parse_into_machines("1 a1bra1zat N expression facies mina: HAS[mouth[ACC]], ISA dog,   [a]HAS[b]")
    #print m.to_dot(True)
<commit_msg>definitionparser now gives an error message when no pyparsing is installed<commit_after>import logging
try:
    from pyparsing import Literal, Word, Group, Combine, Optional, Forward, alphanums, SkipTo, LineEnd, nums, delimitedList
except ImportError:
    import sys
    logging.critical("PyParsing have to be installed on the computer")
    sys.exit(-1)
import string

from machine import Machine
from monoid import Monoid

import sys
class ParserException(Exception):
    pass

class DefinitionParser:
    _str = set([str, unicode])
    _deep_cases = [ "NOM" , "ACC" , "DAT" , "INS" , "OBL"
                    , "SUB", "SUE", "DEL"         # ON
                    , "ILL", "INE", "ELA"         # IN
                    , "ALL", "ADE", "ABL"         # AT
                    ]

    lb = "["
    rb = "]"
    lp = "("
    rp = ")"

    def_sep = ":"
    arg_sep = ","
    part_sep = ";"
    comment_sep = "%"
    prime = "'"
    hyphen = "-"

    def __init__(self):
        self.init_parser()

    def init_parser(self):
        self.id = Word(nums)
        self.lb_lit = Literal(DefinitionParser.lb)
        self.rb_lit = Literal(DefinitionParser.rb)
        self.lp_lit = Literal(DefinitionParser.lp)
        self.rp_lit = Literal(DefinitionParser.rp)

        self.def_sep_lit = Literal(DefinitionParser.def_sep)
        self.arg_sep_lit = Literal(DefinitionParser.arg_sep)
        self.part_sep_lit = Literal(DefinitionParser.part_sep)
        self.comment_sep_lit = Literal(DefinitionParser.comment_sep)
        self.prime_lit = Literal(DefinitionParser.prime)
        self.hyphen_lit = Literal(DefinitionParser.hyphen)

        #self.deep_cases = (Literal("NOM") | Literal("ACC") | Literal("DAT") | Literal("INS") | Literal("ABL"))
        self.deep_cases = reduce(lambda a, b: a | b, (Literal(dc) for dc in DefinitionParser._deep_cases))

        self.unary = Combine(Optional("-") + Word(string.lowercase + "_") + Optional(Word(nums))) | self.deep_cases
        self.binary = Word(string.uppercase + "_" + nums)
        self.dontcare = SkipTo(LineEnd())

        # main expression
        self.expression = Forward()

        # "enumerable expression"
        # D -> E | E, D
        self.definition = Group(delimitedList(self.expression, delim=DefinitionParser.arg_sep))
        self.expression << Group(
                            # E -> U
                            (self.unary) ^

                            # E -> U [ D ]
                            (self.unary + self.lb_lit.suppress() + self.definition + self.rb_lit.suppress() ) ^

                            # E -> U ( U ) | U ( U [ E ] )
                            (self.unary + self.lp_lit + self.unary + Optional(self.lb_lit.suppress() + self.expression + self.rb_lit.suppress()) + self.rp_lit ) ^

                            # E -> U B
                            (self.unary + self.binary) ^

                            # E -> U B E
                            (self.unary + self.binary + self.expression) ^

                            # E -> B E
                            (self.binary + self.expression) ^

                            # E -> B [ E ]
                            (self.binary + self.lb_lit.suppress() + self.expression + self.rb_lit.suppress()) ^

                            # E -> B [ E ; E ]
                            (self.binary + self.lb_lit.suppress() + self.expression + self.part_sep_lit.suppress() + self.expression + self.rb_lit.suppress()) ^

                            # E -> [ E ] B
                            (self.lb_lit.suppress() + self.expression + self.rb_lit.suppress() + self.binary) ^

                            # E -> [ E ] B [ E ]
                            (self.lb_lit.suppress() + self.expression + self.rb_lit.suppress() + self.binary + self.lb_lit.suppress() + self.expression + self.rb_lit.suppress()) ^

                            # E -> [ E ] B E
                            (self.lb_lit.suppress() + self.expression + self.rb_lit.suppress() + self.binary + self.expression ) ^

                            # E -> 'B
                            (self.prime_lit + self.binary) ^

                            # E -> B'
                            (self.binary + self.prime_lit)
                           )

        self.hu, self.pos, self.en, self.lt, self.pt = (Word(alphanums + "#-/_" ),) * 5
        self.word = Group(self.hu + self.pos + self.en + self.lt + self.pt)

        # S -> W : D | W : D % _
        self.sen = (self.id + self.word + self.def_sep_lit.suppress() + self.definition + Optional(self.comment_sep_lit + self.dontcare).suppress()) + LineEnd()

    def parse(self, s):
        return self.sen.parseString(s).asList()

    @classmethod
    def __flatten__(cls, _l):
        _str = cls._str
        result = None
        if type(_l) == list:
            if len(_l) == 1 and type(_l[0]) == list and len(_l[0]) == 1 and type(_l[0][0]) in _str:
                result = DefinitionParser.__flatten__(_l[0])
            else:
                result = [DefinitionParser.__flatten__(e) for e in _l]
        else:
            result = _l
        return result

    @classmethod
    def _is_binary(cls, s):
        return type(s) in cls._str and s.isupper() and not s in cls._deep_cases

    @classmethod
    def _is_unary(cls, s):
        # TODO unaries can contain hyphens
        return type(s) in cls._str and s.islower() or s in cls._deep_cases

    @classmethod
    def _is_deep_case(cls, s):
        return s in cls._deep_cases

    @classmethod
    def _parse_expr(cls, l):
        is_binary = cls._is_binary
        is_unary = cls._is_unary
        _str = cls._str
        if len(l) == 1:
            # unary
            # later here should be a lookup to find out if we already know this machine
            if type(l[0]) == list:
                return cls._parse_expr(l[0])
            else:
                if is_unary(l[0]):
                    return Machine(Monoid(l[0]))
                else:
                    raise ParserException("Only lower case strings or deep cases or filled binaries can be at unary position(" + str(l[0]) + ")")

        elif len(l) == 2:

            if type(l[0]) in _str:
                if is_unary(l[0]):
                    # unary with one partition
                    m = Machine(Monoid(l[0]))
                    m.base.partitions.append([])
                    m.base.partitions[1].append(cls._parse_expr(l[1]))
                    return m
                elif is_binary(l[0]):
                    # binary with right partition filled - prefix
                    m = Machine(Monoid(l[0]))
                    m.base.partitions.append([])
                    m.base.partitions.append([])
                    if type(l[1]) == list and len(l[1]) > 1:
                        raise ParserException("Binary primitives have to have one property per argument")
                    m.base.partitions[2].append(cls._parse_expr(l[1]))

                    return m
            elif type(l[1]) in _str:
                if is_binary(l[1]):
                    # binary with left partition filled - infix
                    m = Machine(Monoid(l[1]))
                    m.base.partitions.append([])
                    if type(l[0]) == list and len(l[0]) > 1:
                        raise ParserException("Binary primitives have to have one property per argument")
                    m.base.partitions[1].append(cls._parse_expr(l[0]))
                    m.base.partitions.append([])
                    return m

        elif len(l) == 3:
            # binary with two partitions
            later = None
            m = None
            if type(l[0]) in _str:
                if is_binary(l[0]):
                    m = Machine(Monoid(l[0]))
                    later = l[1], l[2]
                else:
                    raise ParserException("not uppercase string at binary position")
            elif type(l[1]) in _str:
                if is_binary(l[1]):
                    m = Machine(Monoid(l[1]))
                    later = l[0], l[2]
                else:
                    raise ParserException("not uppercase string at binary position")
            else:
                raise ParserException("not uppercase string at any possible binary position")

            m.base.partitions.append([cls._parse_expr(later[0])])
            m.base.partitions.append([cls._parse_expr(later[1])])
            return m

        elif len(l) == 4:
            # U ( U ) rule
            if is_unary(l[0]) and l[1] == "(" and is_unary(l[2]) and l[3] == ")":
                m = Machine(Monoid(l[2]))
                m.base.partitions.append([])
                m.base.partitions[1].append(Machine(Monoid(l[0])))
                return m
            else:
                pe = ParserException("there shouldn't be 4 nodes in a tree only if 2 of them are parentheses")
                logging.debug(str(pe))
                logging.debug(str(l))
                raise pe

    @classmethod
    def __parse_expr_v2(cls, expr):
        """
        creates machines from a parse node and its children
        there should be one handler for every rule
        """
        is_binary = cls._is_binary
        is_unary = cls._is_unary
        is_tree = lambda r: type(r) == list

        # E -> U [ D ]
        # ['unary', [['unary'], ['unary']]]
        if (len(expr) == 2 and
              is_unary(expr[0]) and
              is_tree(expr[1])):
            m = Machine(Monoid(expr[0]))
            for _property in expr[1]:
                m.append(1, cls.__parse_expr_v2(_property))
            return m

        # E -> U ( U ) | U ( U [ E ] )
        # ['unary', '(', 'unary', ')']
        if (len(expr) == 4 and
              is_unary(expr[0]) and
              expr[1] == cls.lp and
              is_unary(expr[2]) and
              expr[3] == cls.rp):
            m = Machine(Monoid(expr[2]))
            m.append(1, Machine(Monoid(expr[0])))
            return m
        # ???

        # E -> U B E
        # ['unary', 'BINARY', ['unary']]
        if (len(expr) == 3 and
              is_unary(expr[0]) and
              is_binary(expr[1]) and
              is_tree(expr[2])):
            m = Machine(Monoid(expr[1]))
            m.append(1, Machine(Monoid(expr[0])))
            m.append(2, cls.__parse_expr_v2(expr[2]))
            return m

        # E -> U B
        # ['unary', 'BINARY']
        if (len(expr) == 2 and
              is_unary(expr[0]) and
              is_binary(expr[1])):
            m = Machine(Monoid(expr[1]))
            m.append(1, Machine(Monoid(expr[0])))
            m.base.partitions.append([])
            return m

        # E -> U
        # ['unary']
        if (len(expr) == 1 and
              is_unary(expr[0])):
            return Machine(Monoid(expr[0]))

        # E -> B [ E ; E ]
        # ['BINARY', ['unary'], ['unary']]
        if (len(expr) == 3 and
              is_binary(expr[0]) and
              is_tree(expr[1]) and
              is_tree(expr[2])):
            m = Machine(Monoid(expr[0]))
            m.append(1, cls.__parse_expr_v2(expr[1]))
            m.append(2, cls.__parse_expr_v2(expr[2]))
            return m

        # E -> [ E ] B [ E ]
        # [['unary'], 'BINARY', ['unary']]
        if (len(expr) == 3 and
              is_tree(expr[0]) and
              is_binary(expr[1]) and
              is_tree(expr[2])):
            m = Machine(Monoid(expr[1]))
            m.append(1, cls.__parse_expr_v2(expr[0]))
            m.append(2, cls.__parse_expr_v2(expr[2]))
            return m

        # E -> B [ E ]
        # ['BINARY', ['unary']]
        if (len(expr) == 2 and
            is_binary(expr[0]) and
            is_tree(expr[1])):
            m = Machine(Monoid(expr[0]))
            m.append(2, cls.__parse_expr_v2(expr[1]))
            return m

        # E -> [ E ] B
        # [['unary'], 'BINARY']
        if (len(expr) == 2 and
            is_tree(expr[0]) and
            is_binary(expr[1])):
            m = Machine(Monoid(expr[1]))
            m.append(1, cls.__parse_expr_v2(expr[0]))
            m.base.partitions.append([])
            return m

        # E -> B E
        # ['BINARY', ['unary']]
        if (len(expr) == 2 and
            is_binary(expr[0]) and
            is_tree(expr[1])):
            m = Machine(Monoid(expr[0]))
            m.append(2, cls.__parse_expr_v2(expr[1]))
            return m

        # E -> 'B
        # ["'", 'BINARY']
        if (len(expr) == 2 and
            expr[0] == cls.prime and
            is_binary(expr[1])):
            raise NotImplementedError()

        # E -> B'
        # ['BINARY', "'"]
        if (len(expr) == 2 and
            is_binary(expr[0]) and
            expr[1] == cls.prime):
            raise NotImplementedError()

        pe = ParserException("Unknown expression in definition")
        logging.debug(str(pe))
        logging.debug(expr)
        raise pe

    def parse_into_machines(self, s):
        parsed = self.parse(s)
        #parsed = DefinitionParser.__flatten__(parsed)

        machine = Machine(Monoid(parsed[1][2]))
        machine.base.partitions.append([])
        for d in parsed[2]:
            machine.append(1, DefinitionParser.__parse_expr_v2(d))
        return (machine, tuple(parsed[1]))

def read(f):
    from langtools.utils import accents
    d = {}
    dp = DefinitionParser()
    for line in f:
        l = line.strip()
        if len(l) == 0:
            continue
        if l.startswith("#"):
            continue
        m, t = dp.parse_into_machines(line.strip())
        tl = list(t)
        tl[0] = accents.proszeky_to_utf(tl[0].encode("utf-8"))
        d[tuple(tl)] = m
    return d

if __name__ == "__main__":
    dp = DefinitionParser()
    pstr = sys.argv[-1]
    if sys.argv[1] == "-g":
      m, _ = dp.parse_into_machines(pstr)
      print m.to_dot(True)
    else:
      print dp.parse(pstr)
    #print dp.parse("1 a1bra1zat N expression facies mina: HAS mouth[open], ISA dog,   [a]HAS[b]")
    #m = dp.parse_into_machines("1 a1bra1zat N expression facies mina: HAS[mouth[open]], ISA dog,   [a]HAS[b]")
    #m = dp.parse_into_machines("1 a1bra1zat N expression facies mina: HAS[mouth[ACC]], ISA dog,   [a]HAS[b]")
    #print m.to_dot(True)
<|endoftext|>


<commit_before>from django.conf.urls import patterns, url

from base import views

urlpatterns = patterns('',
    url(r'^$', views.IndexView.as_view(), name='index'),
    url(r'^add_project', views.AddProjectView.as_view(), name='add_project'),
    url(r'^(?P<slug>[\w-]+)/$', views.ProjectView.as_view(), name='project'),
)

<commit_msg>Namespace projects<commit_after>from django.conf.urls import patterns, url

from base import views

urlpatterns = patterns('',
    url(r'^$', views.IndexView.as_view(), name='index'),
    url(r'^project/add_project/$', views.AddProjectView.as_view(), name='add_project'),
    url(r'^project/(?P<slug>[\w-]+)/$', views.ProjectView.as_view(), name='project'),
)

<|endoftext|>


<commit_before>from typing import Callable
from time import sleep
from math import ceil
from abc import ABC, abstractmethod
from threading import Lock, Timer, Thread

class RateLimiter(ABC):
    @property
    @abstractmethod
    def permits_issued(self) -> int:
        pass

    @abstractmethod
    def reset_permits_issued(self) -> None:
        pass

    @abstractmethod
    def __enter__(self) -> "RateLimiter":
        pass

    @abstractmethod
    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
        pass

    def limit(self, method: Callable) -> Callable:
        def limited(*args, **kwargs):
            with self:
                return method(*args, **kwargs)
        return limited

class FixedWindowRateLimiter(RateLimiter):
    def __init__(self, window_seconds: int, window_permits: int) -> None:
        self._window_seconds = window_seconds
        self._window_permits = window_permits

        self._total_permits_issued = 0
        self._total_permits_issued_lock = Lock()

        self._permitter = Lock()
        self._permits = window_permits
        self._permits_lock = Lock()

        self._currently_processing = 0
        self._currently_processing_lock = Lock()

        self._resetter = None
        self._resetter_lock = Lock()

    def __enter__(self) -> "FixedWindowRateLimiter":
        # Grab the permit lock and decrement remaining permits. If this leaves it at 0, don't release the permit lock. It will be released by the resetter.
        self._permitter.acquire()
        with self._permits_lock:
            self._permits -= 1
            if self._permits > 0:
                self._permitter.release()

        # Increment total count
        with self._total_permits_issued_lock:
            self._total_permits_issued += 1

        # Increment current count
        with self._currently_processing_lock:
            self._currently_processing += 1

        return self

    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
        # Decrement current count
        with self._currently_processing_lock:
            self._currently_processing -= 1

        # The fixed window starts after processing the first permit ends. The presence of a resetter indicates whether we're in a fixed window already, or need to start one.
        with self._resetter_lock:
            if not self._resetter:
                self._resetter = Timer(self._window_seconds, self._reset)
                self._resetter.daemon = True
                self._resetter.start()

    def _reset(self):
        with self._resetter_lock:
            self._resetter = None

        with self._permits_lock:
            with self._currently_processing_lock:
                self._permits = self._window_permits - self._currently_processing
                try:
                    self._permitter.release()
                except RuntimeError:
                    # Wasn't waiting on any acquire
                    pass

    @property
    def permits_issued(self) -> int:
        with self._total_permits_issued_lock:
            return self._total_permits_issued

    def reset_permits_issued(self) -> None:
        with self._total_permits_issued_lock:
            self._total_permits_issued = 0

class TokenBucketRateLimiter(RateLimiter):
    def __init__(self, epoch_seconds: int, epoch_permits: int, max_burst: int, token_update_frequency: float) -> None:
        if max_burst < 1 or max_burst > epoch_permits:
            raise ValueError("Max burst must be >= 1 and <= epoch permits!")

        if token_update_frequency <= 0 or token_update_frequency > epoch_seconds:
            raise ValueError("Token update frequency must be > 0 and <= epoch seconds!")

        self._epoch_seconds = epoch_seconds
        self._epoch_permits = epoch_permits

        self._permitter = Lock()
        self._token_limit = max_burst
        self._tokens = max_burst
        self._tokens_lock = Lock()

        self._token_update = token_update_frequency

        self._token_provider = None
        self._token_provider_lock = Lock()

        self._total_permits_issued = 0
        self._total_permits_issued_lock = Lock()

    def __enter__(self) -> "TokenBucketRateLimiter":
        # Grab the permit lock and decrement remaining permits. If this leaves it at 0, don't release the permit lock. It will be released by the resetter.
        self._permitter.acquire()
        with self._tokens_lock:
            self._tokens -= 1
            if self._tokens >= 1:
                self._permitter.release()

        # Increment total count
        with self._total_permits_issued_lock:
            self._total_permits_issued += 1

        return self

    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
        # We don't want a thread sitting around dumping tokens into a filled bucket during downtimes. We construct one when needed and it cleans itself up if the bucket is full for an entire epoch.
        with self._token_provider_lock:
            if not self._token_provider:
                self._token_provider = Thread(target=self._provide_tokens)
                self._token_provider.daemon = True
                self._token_provider.start()

    def _provide_tokens(self):
        tokens_per_segment = self._epoch_permits / (self._epoch_seconds / self._token_update)
        segments_full = 0

        # If we go an entire epoch with a full bucket we'll stop this provider
        segments_per_epoch = int(ceil(self._epoch_seconds / self._token_update))
        while segments_full < segments_per_epoch:
            sleep(self._token_update)

            with self._tokens_lock:
                if self._tokens == self._token_limit:
                    segments_full += 1
                elif self._tokens < 1:
                    self._tokens = min(self._tokens + tokens_per_segment, self._token_limit)
                    segments_full = 0
                    if self._tokens >= 1:
                        try:
                            self._permitter.release()
                        except RuntimeError:
                            # Wasn't waiting on any acquire
                            pass
                else:
                    self._tokens = min(self._tokens + tokens_per_segment, self._token_limit)
                    segments_full = 0

        # Clean up
        with self._token_provider_lock:
            self._token_provider = None

    @property
    def permits_issued(self) -> int:
        with self._total_permits_issued_lock:
            return self._total_permits_issued

    def reset_permits_issued(self) -> None:
        with self._total_permits_issued_lock:
            self._total_permits_issued = 0
<commit_msg>Fixed Window race condition fix<commit_after>from typing import Callable
from time import sleep
from math import ceil
from abc import ABC, abstractmethod
from threading import Lock, Timer, Thread

class RateLimiter(ABC):
    @property
    @abstractmethod
    def permits_issued(self) -> int:
        pass

    @abstractmethod
    def reset_permits_issued(self) -> None:
        pass

    @abstractmethod
    def __enter__(self) -> "RateLimiter":
        pass

    @abstractmethod
    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
        pass

    def limit(self, method: Callable) -> Callable:
        def limited(*args, **kwargs):
            with self:
                return method(*args, **kwargs)
        return limited

class FixedWindowRateLimiter(RateLimiter):
    def __init__(self, window_seconds: int, window_permits: int) -> None:
        self._window_seconds = window_seconds
        self._window_permits = window_permits

        self._total_permits_issued = 0
        self._total_permits_issued_lock = Lock()

        self._permitter = Lock()
        self._permits = window_permits
        self._permits_lock = Lock()

        self._currently_processing = 0
        self._currently_processing_lock = Lock()

        self._resetter = None
        self._resetter_lock = Lock()

    def __enter__(self) -> "FixedWindowRateLimiter":
        # Grab the permit lock and decrement remaining permits. If this leaves it at 0, don't release the permit lock. It will be released by the resetter.
        self._permitter.acquire()
        with self._permits_lock:
            self._permits -= 1
            if self._permits > 0:
                self._permitter.release()

        # Increment total count
        with self._total_permits_issued_lock:
            self._total_permits_issued += 1

        # Increment current count
        with self._currently_processing_lock:
            self._currently_processing += 1

        return self

    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
        # Decrement current count
        with self._currently_processing_lock:
            self._currently_processing -= 1

        # The fixed window starts after processing the first permit ends. The presence of a resetter indicates whether we're in a fixed window already, or need to start one.
        with self._resetter_lock:
            if not self._resetter:
                self._resetter = Timer(self._window_seconds, self._reset)
                self._resetter.daemon = True
                self._resetter.start()

    def _reset(self):
        with self._permits_lock:
            with self._currently_processing_lock:
                self._permits = self._window_permits - self._currently_processing
                try:
                    self._permitter.release()
                except RuntimeError:
                    # Wasn't waiting on any acquire
                    pass

            with self._resetter_lock:
                self._resetter = None

    @property
    def permits_issued(self) -> int:
        with self._total_permits_issued_lock:
            return self._total_permits_issued

    def reset_permits_issued(self) -> None:
        with self._total_permits_issued_lock:
            self._total_permits_issued = 0

class TokenBucketRateLimiter(RateLimiter):
    def __init__(self, epoch_seconds: int, epoch_permits: int, max_burst: int, token_update_frequency: float) -> None:
        if max_burst < 1 or max_burst > epoch_permits:
            raise ValueError("Max burst must be >= 1 and <= epoch permits!")

        if token_update_frequency <= 0 or token_update_frequency > epoch_seconds:
            raise ValueError("Token update frequency must be > 0 and <= epoch seconds!")

        self._epoch_seconds = epoch_seconds
        self._epoch_permits = epoch_permits

        self._permitter = Lock()
        self._token_limit = max_burst
        self._tokens = max_burst
        self._tokens_lock = Lock()

        self._token_update = token_update_frequency

        self._token_provider = None
        self._token_provider_lock = Lock()

        self._total_permits_issued = 0
        self._total_permits_issued_lock = Lock()

    def __enter__(self) -> "TokenBucketRateLimiter":
        # Grab the permit lock and decrement remaining permits. If this leaves it at 0, don't release the permit lock. It will be released by the resetter.
        self._permitter.acquire()
        with self._tokens_lock:
            self._tokens -= 1
            if self._tokens >= 1:
                self._permitter.release()

        # Increment total count
        with self._total_permits_issued_lock:
            self._total_permits_issued += 1

        return self

    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
        # We don't want a thread sitting around dumping tokens into a filled bucket during downtimes. We construct one when needed and it cleans itself up if the bucket is full for an entire epoch.
        with self._token_provider_lock:
            if not self._token_provider:
                self._token_provider = Thread(target=self._provide_tokens)
                self._token_provider.daemon = True
                self._token_provider.start()

    def _provide_tokens(self):
        tokens_per_segment = self._epoch_permits / (self._epoch_seconds / self._token_update)
        segments_full = 0

        # If we go an entire epoch with a full bucket we'll stop this provider
        segments_per_epoch = int(ceil(self._epoch_seconds / self._token_update))
        while segments_full < segments_per_epoch:
            sleep(self._token_update)

            with self._tokens_lock:
                if self._tokens == self._token_limit:
                    segments_full += 1
                elif self._tokens < 1:
                    self._tokens = min(self._tokens + tokens_per_segment, self._token_limit)
                    segments_full = 0
                    if self._tokens >= 1:
                        try:
                            self._permitter.release()
                        except RuntimeError:
                            # Wasn't waiting on any acquire
                            pass
                else:
                    self._tokens = min(self._tokens + tokens_per_segment, self._token_limit)
                    segments_full = 0

        # Clean up
        with self._token_provider_lock:
            self._token_provider = None

    @property
    def permits_issued(self) -> int:
        with self._total_permits_issued_lock:
            return self._total_permits_issued

    def reset_permits_issued(self) -> None:
        with self._total_permits_issued_lock:
            self._total_permits_issued = 0
<|endoftext|>


<commit_before>import os

import numpy

from .._common import cell_data_from_raw, write_xml
from .._vtk import raw_from_cell_data
from .common import (
    dtype_to_format_string,
    meshio_to_xdmf_type,
    meshio_type_to_xdmf_index,
    numpy_to_xdmf_dtype,
    translate_mixed_cells,
    xdmf_to_meshio_type,
    xdmf_to_numpy_type,
)

try:
    from StringIO import cStringIO as BytesIO
except ImportError:
    from io import BytesIO

class XdmfTimeSeriesReader:
    def __init__(self, filename):
        from lxml import etree as ET

        self.filename = filename

        parser = ET.XMLParser(remove_comments=True, huge_tree=True)
        tree = ET.parse(self.filename, parser)
        root = tree.getroot()

        assert root.tag == "Xdmf"

        version = root.attrib["Version"]
        assert version.split(".")[0] == "3", "Unknown XDMF version {}.".format(version)

        domains = list(root)
        assert len(domains) == 1
        self.domain = domains[0]
        assert self.domain.tag == "Domain"

        grids = list(self.domain)

        # find the uniform grid
        self.mesh_grid = None
        for g in grids:
            if g.attrib["GridType"] == "Uniform":
                self.mesh_grid = g
        assert self.mesh_grid is not None, "Couldn't find the mesh grid"
        assert self.mesh_grid.tag == "Grid"

        # find the collection grid
        collection_grid = None
        for g in grids:
            if g.attrib["GridType"] == "Collection":
                collection_grid = g
        assert collection_grid is not None, "Couldn't find the mesh grid"
        assert collection_grid.tag == "Grid"
        assert collection_grid.attrib["CollectionType"] == "Temporal"
        # get the collection at once
        self.collection = list(collection_grid)
        self.num_steps = len(self.collection)
        self.cells = None
        self.hdf5_files = {}

    def __enter__(self):
        return self

    def __exit__(self, *args):
        # Those files are opened in _read_data_item()
        for f in self.hdf5_files.values():
            f.close()

    def read_points_cells(self):
        grid = self.mesh_grid

        points = None
        cells = {}

        for c in grid:
            if c.tag == "Topology":
                data_items = list(c)
                assert len(data_items) == 1
                data_item = data_items[0]

                data = self._read_data_item(data_item)

                # The XDMF2 key is `TopologyType`, just `Type` for XDMF3.
                # Allow both.
                if "Type" in c.attrib:
                    assert "TopologyType" not in c.attrib
                    cell_type = c.attrib["Type"]
                else:
                    cell_type = c.attrib["TopologyType"]

                if cell_type == "Mixed":
                    cells = translate_mixed_cells(data)
                else:
                    meshio_type = xdmf_to_meshio_type[cell_type]
                    cells[meshio_type] = data

            elif c.tag == "Geometry":
                try:
                    geometry_type = c.attrib["GeometryType"]
                except KeyError:
                    pass
                else:
                    assert geometry_type in ["XY", "XYZ"]

                data_items = list(c)
                assert len(data_items) == 1
                data_item = data_items[0]
                points = self._read_data_item(data_item)

        self.cells = cells
        return points, cells

    def read_data(self, k):
        point_data = {}
        cell_data_raw = {}

        t = None

        for c in list(self.collection[k]):
            if c.tag == "Time":
                t = float(c.attrib["Value"])
            elif c.tag == "Attribute":
                name = c.attrib["Name"]

                assert len(list(c)) == 1
                data_item = list(c)[0]
                data = self._read_data_item(data_item)

                if c.attrib["Center"] == "Node":
                    point_data[name] = data
                else:
                    assert c.attrib["Center"] == "Cell"
                    cell_data_raw[name] = data
            else:
                # skip the xi:included mesh
                continue

        assert self.cells is not None
        cell_data = cell_data_from_raw(self.cells, cell_data_raw)
        assert t is not None

        return t, point_data, cell_data

    def _read_data_item(self, data_item):
        dims = [int(d) for d in data_item.attrib["Dimensions"].split()]

        # Actually, `NumberType` is XDMF2 and `DataType` XDMF3, but many files out there
        # use both keys interchangeably.
        if "DataType" in data_item.attrib:
            assert "NumberType" not in data_item.attrib
            data_type = data_item.attrib["DataType"]
        elif "NumberType" in data_item.attrib:
            assert "DataType" not in data_item.attrib
            data_type = data_item.attrib["NumberType"]
        else:
            # Default, see
            # <http://www.xdmf.org/index.php/XDMF_Model_and_Format#XML_Element_.28Xdmf_ClassName.29_and_Default_XML_Attributes>
            data_type = "Float"

        try:
            precision = data_item.attrib["Precision"]
        except KeyError:
            precision = "4"

        if data_item.attrib["Format"] == "XML":
            return numpy.array(
                data_item.text.split(), dtype=xdmf_to_numpy_type[(data_type, precision)]
            ).reshape(dims)
        elif data_item.attrib["Format"] == "Binary":
            return numpy.fromfile(
                data_item.text.strip(), dtype=xdmf_to_numpy_type[(data_type, precision)]
            ).reshape(dims)

        assert data_item.attrib["Format"] == "HDF", "Unknown XDMF Format '{}'.".format(
            data_item.attrib["Format"]
        )

        info = data_item.text.strip()
        filename, h5path = info.split(":")

        # The HDF5 file path is given with respect to the XDMF (XML) file.
        full_hdf5_path = os.path.join(os.path.dirname(self.filename), filename)

        if full_hdf5_path in self.hdf5_files:
            f = self.hdf5_files[full_hdf5_path]
        else:
            import h5py

            f = h5py.File(full_hdf5_path, "r")
            self.hdf5_files[full_hdf5_path] = f

        assert h5path[0] == "/"

        for key in h5path[1:].split("/"):
            f = f[key]
        # `[()]` gives a numpy.ndarray
        return f[()]

class XdmfTimeSeriesWriter:
    def __init__(self, filename, pretty_xml=True, data_format="HDF"):
        from lxml import etree as ET

        assert data_format in ["XML", "Binary", "HDF"], (
            "Unknown XDMF data format "
            "'{}' (use 'XML', 'Binary', or 'HDF'.)".format(data_format)
        )

        self.filename = filename
        self.data_format = data_format
        self.data_counter = 0
        self.pretty_xml = pretty_xml

        self.xdmf_file = ET.Element("Xdmf", Version="3.0")

        self.domain = ET.SubElement(self.xdmf_file, "Domain")
        self.collection = ET.SubElement(
            self.domain,
            "Grid",
            Name="TimeSeries_meshio",
            GridType="Collection",
            CollectionType="Temporal",
        )

        ET.register_namespace("xi", "https://www.w3.org/2001/XInclude/")

        self.has_mesh = False
        self.mesh_name = None

    def __enter__(self):
        if self.data_format == "HDF":
            import h5py

            self.h5_filename = os.path.splitext(self.filename)[0] + ".h5"
            self.h5_file = h5py.File(self.h5_filename, "w")
        return self

    def __exit__(self, *args):
        self.h5_file.close()

    def write_points_cells(self, points, cells):
        # <Grid Name="mesh" GridType="Uniform">
        #   <Topology NumberOfElements="16757" TopologyType="Triangle" NodesPerElement="3">
        #     <DataItem Dimensions="16757 3" NumberType="UInt" Format="HDF">maxwell.h5:/Mesh/0/mesh/topology</DataItem>
        #   </Topology>
        #   <Geometry GeometryType="XY">
        #     <DataItem Dimensions="8457 2" Format="HDF">maxwell.h5:/Mesh/0/mesh/geometry</DataItem>
        #   </Geometry>
        # </Grid>
        from lxml import etree as ET

        self.mesh_name = "mesh"
        grid = ET.SubElement(
            self.domain, "Grid", Name=self.mesh_name, GridType="Uniform"
        )
        self.points(grid, points)
        self.cells(cells, grid)
        self.has_mesh = True

        write_xml(self.filename, self.xdmf_file, self.pretty_xml)

    def write_data(self, t, point_data=None, cell_data=None):
        # <Grid>
        #   <xi:include xpointer="xpointer(//Grid[@Name="TimeSeries_phi"]/Grid[1]/*[self::Topology or self::Geometry])" />
        #   <Time Value="3.3333333333333335e-05" />
        #   <Attribute Name="phi" AttributeType="Scalar" Center="Node">
        #     <DataItem Dimensions="8457 1" Format="HDF">maxwell.h5:/VisualisationVector/1</DataItem>
        #   </Attribute>
        # </Grid>
        from lxml import etree as ET

        grid = ET.SubElement(self.collection, "Grid")
        assert self.has_mesh
        ptr = 'xpointer(//Grid[@Name="{}"]/*[self::Topology or self::Geometry])'.format(
            self.mesh_name
        )
        ET.SubElement(grid, "{http://www.w3.org/2003/XInclude}include", xpointer=ptr)
        ET.SubElement(grid, "Time", Value="{}".format(t))

        if point_data:
            self.point_data(point_data, grid)
        if cell_data:
            self.cell_data(cell_data, grid)

        write_xml(self.filename, self.xdmf_file, self.pretty_xml)

    def numpy_to_xml_string(self, data):
        if self.data_format == "XML":
            s = BytesIO()
            fmt = dtype_to_format_string[data.dtype.name]
            numpy.savetxt(s, data.flatten(), fmt)
            return s.getvalue().decode()
        elif self.data_format == "Binary":
            bin_filename = "{}{}.bin".format(
                os.path.splitext(self.filename)[0], self.data_counter
            )
            self.data_counter += 1
            # write binary data to file
            with open(bin_filename, "wb") as f:
                data.tofile(f)
            return bin_filename

        assert self.data_format == "HDF"
        name = "data{}".format(self.data_counter)
        self.data_counter += 1
        self.h5_file.create_dataset(name, data=data)
        return os.path.basename(self.h5_filename) + ":/" + name

    def points(self, grid, points):
        from lxml import etree as ET

        if points.shape[1] == 2:
            geometry_type = "XY"
        else:
            assert points.shape[1] == 3
            geometry_type = "XYZ"

        geo = ET.SubElement(grid, "Geometry", GeometryType=geometry_type)
        dt, prec = numpy_to_xdmf_dtype[points.dtype.name]
        dim = "{} {}".format(*points.shape)
        data_item = ET.SubElement(
            geo,
            "DataItem",
            DataType=dt,
            Dimensions=dim,
            Format=self.data_format,
            Precision=prec,
        )
        data_item.text = self.numpy_to_xml_string(points)

    def cells(self, cells, grid):
        from lxml import etree as ET

        if len(cells) == 1:
            meshio_type = list(cells.keys())[0]
            num_cells = len(cells[meshio_type])
            xdmf_type = meshio_to_xdmf_type[meshio_type][0]
            topo = ET.SubElement(
                grid,
                "Topology",
                TopologyType=xdmf_type,
                NumberOfElements=str(num_cells),
            )
            dt, prec = numpy_to_xdmf_dtype[cells[meshio_type].dtype.name]
            dim = "{} {}".format(*cells[meshio_type].shape)
            data_item = ET.SubElement(
                topo,
                "DataItem",
                DataType=dt,
                Dimensions=dim,
                Format=self.data_format,
                Precision=prec,
            )
            data_item.text = self.numpy_to_xml_string(cells[meshio_type])
        elif len(cells) > 1:
            total_num_cells = sum(c.shape[0] for c in cells.values())
            topo = ET.SubElement(
                grid,
                "Topology",
                TopologyType="Mixed",
                NumberOfElements=str(total_num_cells),
            )
            total_num_cell_items = sum(numpy.prod(c.shape) for c in cells.values())
            dim = total_num_cell_items + total_num_cells
            # Lines translate to Polylines, and one needs to specify the exact
            # number of nodes. Hence, prepend 2.
            if "line" in cells:
                cells["line"] = numpy.insert(cells["line"], 0, 2, axis=1)
                dim += len(cells["line"])
            dim = str(dim)
            cd = numpy.concatenate(
                [
                    # prepend column with xdmf type index
                    numpy.insert(
                        value, 0, meshio_type_to_xdmf_index[key], axis=1
                    ).flatten()
                    for key, value in cells.items()
                ]
            )
            dt, prec = numpy_to_xdmf_dtype[cd.dtype.name]
            data_item = ET.SubElement(
                topo,
                "DataItem",
                DataType=dt,
                Dimensions=dim,
                Format=self.data_format,
                Precision=prec,
            )
            data_item.text = self.numpy_to_xml_string(cd)

    def point_data(self, point_data, grid):
        from lxml import etree as ET

        for name, data in point_data.items():
            att = ET.SubElement(grid, "Attribute", Name=name, Center="Node")
            dt, prec = numpy_to_xdmf_dtype[data.dtype.name]
            dim = " ".join([str(s) for s in data.shape])
            data_item = ET.SubElement(
                att,
                "DataItem",
                DataType=dt,
                Dimensions=dim,
                Format=self.data_format,
                Precision=prec,
            )
            data_item.text = self.numpy_to_xml_string(data)

    def cell_data(self, cell_data, grid):
        from lxml import etree as ET

        raw = raw_from_cell_data(cell_data)
        for name, data in raw.items():
            att = ET.SubElement(
                grid, "Attribute", Name=name, Type="None", Center="Cell"
            )
            dt, prec = numpy_to_xdmf_dtype[data.dtype.name]
            dim = " ".join([str(s) for s in data.shape])
            data_item = ET.SubElement(
                att,
                "DataItem",
                DataType=dt,
                Dimensions=dim,
                Format=self.data_format,
                Precision=prec,
            )
            data_item.text = self.numpy_to_xml_string(data)
<commit_msg>Allow reading XDMF time-series à la FEniCS<commit_after>import os

import numpy

from .._common import cell_data_from_raw, write_xml
from .._vtk import raw_from_cell_data
from .common import (
    dtype_to_format_string,
    meshio_to_xdmf_type,
    meshio_type_to_xdmf_index,
    numpy_to_xdmf_dtype,
    translate_mixed_cells,
    xdmf_to_meshio_type,
    xdmf_to_numpy_type,
)

try:
    from StringIO import cStringIO as BytesIO
except ImportError:
    from io import BytesIO

class XdmfTimeSeriesReader:
    def __init__(self, filename):
        from lxml import etree as ET

        self.filename = filename

        parser = ET.XMLParser(remove_comments=True, huge_tree=True)
        tree = ET.parse(self.filename, parser)
        root = tree.getroot()

        assert root.tag == "Xdmf"

        version = root.attrib["Version"]
        assert version.split(".")[0] == "3", "Unknown XDMF version {}.".format(version)

        domains = list(root)
        assert len(domains) == 1
        self.domain = domains[0]
        assert self.domain.tag == "Domain"

        grids = list(self.domain)

        # find the collection grid
        collection_grid = None
        for g in grids:
            if g.attrib["GridType"] == "Collection":
                collection_grid = g
        assert collection_grid is not None, "Couldn't find the mesh grid"
        assert collection_grid.tag == "Grid"
        assert collection_grid.attrib["CollectionType"] == "Temporal"

        # get the collection at once
        self.collection = list(collection_grid)
        self.num_steps = len(self.collection)
        self.cells = None
        self.hdf5_files = {}

        # find the uniform grid
        self.mesh_grid = None
        for g in grids:
            if g.attrib["GridType"] == "Uniform":
                self.mesh_grid = g
        # if not found, take the first uniform grid in the collection grid
        if self.mesh_grid is None:
            for g in self.collection:
                try:
                    if g.attrib["GridType"] == "Uniform":
                        self.mesh_grid = g
                        break
                except KeyError:
                    continue
        assert self.mesh_grid is not None, "Couldn't find the mesh grid"
        assert self.mesh_grid.tag == "Grid"

    def __enter__(self):
        return self

    def __exit__(self, *args):
        # Those files are opened in _read_data_item()
        for f in self.hdf5_files.values():
            f.close()

    def read_points_cells(self):
        grid = self.mesh_grid

        points = None
        cells = {}

        for c in grid:
            if c.tag == "Topology":
                data_items = list(c)
                assert len(data_items) == 1
                data_item = data_items[0]

                data = self._read_data_item(data_item)

                # The XDMF2 key is `TopologyType`, just `Type` for XDMF3.
                # Allow both.
                if "Type" in c.attrib:
                    assert "TopologyType" not in c.attrib
                    cell_type = c.attrib["Type"]
                else:
                    cell_type = c.attrib["TopologyType"]

                if cell_type == "Mixed":
                    cells = translate_mixed_cells(data)
                else:
                    meshio_type = xdmf_to_meshio_type[cell_type]
                    cells[meshio_type] = data

            elif c.tag == "Geometry":
                try:
                    geometry_type = c.attrib["GeometryType"]
                except KeyError:
                    pass
                else:
                    assert geometry_type in ["XY", "XYZ"]

                data_items = list(c)
                assert len(data_items) == 1
                data_item = data_items[0]
                points = self._read_data_item(data_item)

        self.cells = cells
        return points, cells

    def read_data(self, k):
        point_data = {}
        cell_data_raw = {}

        t = None

        for c in list(self.collection[k]):
            if c.tag == "Time":
                t = float(c.attrib["Value"])
            elif c.tag == "Attribute":
                name = c.attrib["Name"]

                assert len(list(c)) == 1
                data_item = list(c)[0]
                data = self._read_data_item(data_item)

                if c.attrib["Center"] == "Node":
                    point_data[name] = data
                else:
                    assert c.attrib["Center"] == "Cell"
                    cell_data_raw[name] = data
            else:
                # skip the xi:included mesh
                continue

        assert self.cells is not None
        cell_data = cell_data_from_raw(self.cells, cell_data_raw)
        assert t is not None

        return t, point_data, cell_data

    def _read_data_item(self, data_item):
        dims = [int(d) for d in data_item.attrib["Dimensions"].split()]

        # Actually, `NumberType` is XDMF2 and `DataType` XDMF3, but many files out there
        # use both keys interchangeably.
        if "DataType" in data_item.attrib:
            assert "NumberType" not in data_item.attrib
            data_type = data_item.attrib["DataType"]
        elif "NumberType" in data_item.attrib:
            assert "DataType" not in data_item.attrib
            data_type = data_item.attrib["NumberType"]
        else:
            # Default, see
            # <http://www.xdmf.org/index.php/XDMF_Model_and_Format#XML_Element_.28Xdmf_ClassName.29_and_Default_XML_Attributes>
            data_type = "Float"

        try:
            precision = data_item.attrib["Precision"]
        except KeyError:
            precision = "4"

        if data_item.attrib["Format"] == "XML":
            return numpy.array(
                data_item.text.split(), dtype=xdmf_to_numpy_type[(data_type, precision)]
            ).reshape(dims)
        elif data_item.attrib["Format"] == "Binary":
            return numpy.fromfile(
                data_item.text.strip(), dtype=xdmf_to_numpy_type[(data_type, precision)]
            ).reshape(dims)

        assert data_item.attrib["Format"] == "HDF", "Unknown XDMF Format '{}'.".format(
            data_item.attrib["Format"]
        )

        info = data_item.text.strip()
        filename, h5path = info.split(":")

        # The HDF5 file path is given with respect to the XDMF (XML) file.
        full_hdf5_path = os.path.join(os.path.dirname(self.filename), filename)

        if full_hdf5_path in self.hdf5_files:
            f = self.hdf5_files[full_hdf5_path]
        else:
            import h5py

            f = h5py.File(full_hdf5_path, "r")
            self.hdf5_files[full_hdf5_path] = f

        assert h5path[0] == "/"

        for key in h5path[1:].split("/"):
            f = f[key]
        # `[()]` gives a numpy.ndarray
        return f[()]

class XdmfTimeSeriesWriter:
    def __init__(self, filename, pretty_xml=True, data_format="HDF"):
        from lxml import etree as ET

        assert data_format in ["XML", "Binary", "HDF"], (
            "Unknown XDMF data format "
            "'{}' (use 'XML', 'Binary', or 'HDF'.)".format(data_format)
        )

        self.filename = filename
        self.data_format = data_format
        self.data_counter = 0
        self.pretty_xml = pretty_xml

        self.xdmf_file = ET.Element("Xdmf", Version="3.0")

        self.domain = ET.SubElement(self.xdmf_file, "Domain")
        self.collection = ET.SubElement(
            self.domain,
            "Grid",
            Name="TimeSeries_meshio",
            GridType="Collection",
            CollectionType="Temporal",
        )

        ET.register_namespace("xi", "https://www.w3.org/2001/XInclude/")

        self.has_mesh = False
        self.mesh_name = None

    def __enter__(self):
        if self.data_format == "HDF":
            import h5py

            self.h5_filename = os.path.splitext(self.filename)[0] + ".h5"
            self.h5_file = h5py.File(self.h5_filename, "w")
        return self

    def __exit__(self, *args):
        self.h5_file.close()

    def write_points_cells(self, points, cells):
        # <Grid Name="mesh" GridType="Uniform">
        #   <Topology NumberOfElements="16757" TopologyType="Triangle" NodesPerElement="3">
        #     <DataItem Dimensions="16757 3" NumberType="UInt" Format="HDF">maxwell.h5:/Mesh/0/mesh/topology</DataItem>
        #   </Topology>
        #   <Geometry GeometryType="XY">
        #     <DataItem Dimensions="8457 2" Format="HDF">maxwell.h5:/Mesh/0/mesh/geometry</DataItem>
        #   </Geometry>
        # </Grid>
        from lxml import etree as ET

        self.mesh_name = "mesh"
        grid = ET.SubElement(
            self.domain, "Grid", Name=self.mesh_name, GridType="Uniform"
        )
        self.points(grid, points)
        self.cells(cells, grid)
        self.has_mesh = True

        write_xml(self.filename, self.xdmf_file, self.pretty_xml)

    def write_data(self, t, point_data=None, cell_data=None):
        # <Grid>
        #   <xi:include xpointer="xpointer(//Grid[@Name="TimeSeries_phi"]/Grid[1]/*[self::Topology or self::Geometry])" />
        #   <Time Value="3.3333333333333335e-05" />
        #   <Attribute Name="phi" AttributeType="Scalar" Center="Node">
        #     <DataItem Dimensions="8457 1" Format="HDF">maxwell.h5:/VisualisationVector/1</DataItem>
        #   </Attribute>
        # </Grid>
        from lxml import etree as ET

        grid = ET.SubElement(self.collection, "Grid")
        assert self.has_mesh
        ptr = 'xpointer(//Grid[@Name="{}"]/*[self::Topology or self::Geometry])'.format(
            self.mesh_name
        )
        ET.SubElement(grid, "{http://www.w3.org/2003/XInclude}include", xpointer=ptr)
        ET.SubElement(grid, "Time", Value="{}".format(t))

        if point_data:
            self.point_data(point_data, grid)
        if cell_data:
            self.cell_data(cell_data, grid)

        write_xml(self.filename, self.xdmf_file, self.pretty_xml)

    def numpy_to_xml_string(self, data):
        if self.data_format == "XML":
            s = BytesIO()
            fmt = dtype_to_format_string[data.dtype.name]
            numpy.savetxt(s, data.flatten(), fmt)
            return s.getvalue().decode()
        elif self.data_format == "Binary":
            bin_filename = "{}{}.bin".format(
                os.path.splitext(self.filename)[0], self.data_counter
            )
            self.data_counter += 1
            # write binary data to file
            with open(bin_filename, "wb") as f:
                data.tofile(f)
            return bin_filename

        assert self.data_format == "HDF"
        name = "data{}".format(self.data_counter)
        self.data_counter += 1
        self.h5_file.create_dataset(name, data=data)
        return os.path.basename(self.h5_filename) + ":/" + name

    def points(self, grid, points):
        from lxml import etree as ET

        if points.shape[1] == 2:
            geometry_type = "XY"
        else:
            assert points.shape[1] == 3
            geometry_type = "XYZ"

        geo = ET.SubElement(grid, "Geometry", GeometryType=geometry_type)
        dt, prec = numpy_to_xdmf_dtype[points.dtype.name]
        dim = "{} {}".format(*points.shape)
        data_item = ET.SubElement(
            geo,
            "DataItem",
            DataType=dt,
            Dimensions=dim,
            Format=self.data_format,
            Precision=prec,
        )
        data_item.text = self.numpy_to_xml_string(points)

    def cells(self, cells, grid):
        from lxml import etree as ET

        if len(cells) == 1:
            meshio_type = list(cells.keys())[0]
            num_cells = len(cells[meshio_type])
            xdmf_type = meshio_to_xdmf_type[meshio_type][0]
            topo = ET.SubElement(
                grid,
                "Topology",
                TopologyType=xdmf_type,
                NumberOfElements=str(num_cells),
            )
            dt, prec = numpy_to_xdmf_dtype[cells[meshio_type].dtype.name]
            dim = "{} {}".format(*cells[meshio_type].shape)
            data_item = ET.SubElement(
                topo,
                "DataItem",
                DataType=dt,
                Dimensions=dim,
                Format=self.data_format,
                Precision=prec,
            )
            data_item.text = self.numpy_to_xml_string(cells[meshio_type])
        elif len(cells) > 1:
            total_num_cells = sum(c.shape[0] for c in cells.values())
            topo = ET.SubElement(
                grid,
                "Topology",
                TopologyType="Mixed",
                NumberOfElements=str(total_num_cells),
            )
            total_num_cell_items = sum(numpy.prod(c.shape) for c in cells.values())
            dim = total_num_cell_items + total_num_cells
            # Lines translate to Polylines, and one needs to specify the exact
            # number of nodes. Hence, prepend 2.
            if "line" in cells:
                cells["line"] = numpy.insert(cells["line"], 0, 2, axis=1)
                dim += len(cells["line"])
            dim = str(dim)
            cd = numpy.concatenate(
                [
                    # prepend column with xdmf type index
                    numpy.insert(
                        value, 0, meshio_type_to_xdmf_index[key], axis=1
                    ).flatten()
                    for key, value in cells.items()
                ]
            )
            dt, prec = numpy_to_xdmf_dtype[cd.dtype.name]
            data_item = ET.SubElement(
                topo,
                "DataItem",
                DataType=dt,
                Dimensions=dim,
                Format=self.data_format,
                Precision=prec,
            )
            data_item.text = self.numpy_to_xml_string(cd)

    def point_data(self, point_data, grid):
        from lxml import etree as ET

        for name, data in point_data.items():
            att = ET.SubElement(grid, "Attribute", Name=name, Center="Node")
            dt, prec = numpy_to_xdmf_dtype[data.dtype.name]
            dim = " ".join([str(s) for s in data.shape])
            data_item = ET.SubElement(
                att,
                "DataItem",
                DataType=dt,
                Dimensions=dim,
                Format=self.data_format,
                Precision=prec,
            )
            data_item.text = self.numpy_to_xml_string(data)

    def cell_data(self, cell_data, grid):
        from lxml import etree as ET

        raw = raw_from_cell_data(cell_data)
        for name, data in raw.items():
            att = ET.SubElement(
                grid, "Attribute", Name=name, Type="None", Center="Cell"
            )
            dt, prec = numpy_to_xdmf_dtype[data.dtype.name]
            dim = " ".join([str(s) for s in data.shape])
            data_item = ET.SubElement(
                att,
                "DataItem",
                DataType=dt,
                Dimensions=dim,
                Format=self.data_format,
                Precision=prec,
            )
            data_item.text = self.numpy_to_xml_string(data)
<|endoftext|>


<commit_before># Copyright 2013-present Barefoot Networks, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# -----------------------------------------------------------------------------
# stf_parser.py
#
# Parse an STF file.
# -----------------------------------------------------------------------------

import ply.yacc as yacc
from .stf_lexer import STFLexer

# PARSER GRAMMAR --------------------------------------------------------------

# statement : ADD qualified_name priority match_list action [= ID]
#           | ADD qualified_name match_list action [= ID]
#           | REMOVE [ALL | entry]
#           | CHECK_COUNTER ID(id_or_index) [count_type logical_cond number]
#           | EXPECT port expect_data
#           | EXPECT port
#           | NO_PACKET
#           | PACKET port packet_data
#           | SETDEFAULT qualified_name action
#           | WAIT
#
# match_list : match
#            | match_list match
# match : qualified_name COLON number_or_lpm
# number_or_lpm: number SLASH number
#              | number
# number : INT_CONST_DEC
#        | INT_CONST_BIN
#        | INT_CONST_HEX
#        | TERN_CONST_HEX
#
# qualified_name : ID
#                | ID '[' INT_CONST_DEC ']'
#                | qualified_name DOT ID
#
# id_or_index: ID
#            | number
# count_type : BYTES
#            | PACKETS
# logical_cond : EQEQ
#              | NEQ
#              | LEQ
#              | LE
#              | GEQ
#              | GT
#
# action : qualified_name LPAREN args RPAREN
# args :
#      | arg_list
# arg_list: arg
#      | arg_list COMMA arg
# arg : ID COLON number
#
# port : DATA_DEC
#
# priority : INT_CONST_DEC
#
# expect_data : expect_datum
#             | expect_data expect_datum
# packet_data : packet_datum
#             | packet_data packet_datum
#
# expect_datum : packet_datum | DATA_TERN
# packet_datum : DATA_DEC | DATA_HEX

# PARSER ----------------------------------------------------------------------

class STFParser:
    def __init__(self):
        self.lexer = STFLexer()
        self.lexer.build()
        self.tokens = self.lexer.tokens
        self.parser = yacc.yacc(module = self)
        self.errors_cnt = 0

    def parse(self, data = None, filename=''):
        if data is None and filename == '':
            raise ValueError("Please specify either a filename or data")

        # if we specified only the filename, initialize the data
        if data is None:
            with file(filename) as f: data = f.read()

        self.lexer.filename = filename
        self.lexer.reset_lineno()
        stf_ast = self.parser.parse(input = data,
                                    lexer = self.lexer)
        self.errors_cnt += self.lexer.errors_cnt
        return stf_ast, self.errors_cnt

    def print_error(self, lineno, lexpos, msg):
        self.errors_cnt += 1
        print("parse error (%s%s:%s): %s" % (
            '%s:' % self.lexer.filename if self.lexer.filename else '',
            lineno,
            lexpos,
            msg))

    def get_filename(self):
        return self.lexer.filename

    def p_error(self, p):
        if p is None:
            self.print_error(self.lexer.get_lineno(), 0,
                             "Unexpected end-of-file.  Missing paren?")
        else:
            self.print_error(p.lineno, self.lexer.get_colno(),
                             "Syntax error while parsing at token '%s' (of type %s)." % (p.value, p.type))
            # Skip to the next statement.
            while True:
                tok = self.parser.token()
                if not tok or tok.type in self.lexer.keywords:
                    break
                self.parser.restart()

    def p_statements_none(self, p):
        'statements : '
        p[0] = []

    def p_statements_one(self, p):
        'statements : statement'
        p[0] = [p[1]]

    def p_statements_many(self, p):
        'statements : statements statement'
        p[0] = p[1] + [p[2]]

    def p_statement_add(self, p):
        'statement : ADD qualified_name match_list action'
        p[0] = (p[1].lower(), p[2], None, p[3], p[4], None)

    def p_statement_add_with_priority(self, p):
        'statement : ADD qualified_name priority match_list action'
        p[0] = (p[1].lower(), p[2], p[3], p[4], p[5], None)

    def p_statement_add_with_id(self, p):
        'statement : ADD qualified_name match_list action EQUAL ID'
        p[0] = (p[1].lower(), p[2], None, p[3], p[4], p[6])

    def p_statement_add_with_priority_and_id(self, p):
        'statement : ADD qualified_name priority match_list action EQUAL ID'
        p[0] = (p[1].lower(), p[2], p[3], p[4], p[5], p[7])

    # remove all entries.
    # \TODO: removal of individual entries
    def p_statement_remove(self, p):
        'statement : REMOVE ALL'
        p[0] = (p[1].lower(), p[2])

    def p_statement_check_counter(self, p):
        'statement : CHECK_COUNTER ID LPAREN id_or_index RPAREN'
        p[0] = (p[1].lower(), p[2], p[4], (None, '==', 0))

    def p_statement_check_counter_with_check(self, p):
        'statement : CHECK_COUNTER ID LPAREN id_or_index RPAREN count_type logical_cond number'
        p[0] = (p[1].lower(), p[2], p[4], (p[6], p[7], p[8]))

    def p_statement_expect(self, p):
        'statement : EXPECT port'
        p[0] = (p[1].lower(), p[2], None)

    def p_statement_expect_data(self, p):
        'statement : EXPECT port expect_data'
        p[0] = (p[1].lower(), p[2], ''.join(p[3]))

    def p_statement_no_packet(self, p):
        'statement : NO_PACKET'
        p[0] = ('expect', None, None)

    def p_statement_packet(self, p):
        'statement : PACKET port packet_data'
        p[0] = (p[1].lower(), p[2], ''.join(p[3]))

    def p_statement_setdefault(self, p):
        'statement : SETDEFAULT qualified_name action'
        p[0] = (p[1].lower(), p[2], p[3])

    def p_statement_wait(self, p):
        'statement : WAIT'
        p[0] = (p[1].lower(),)

    def p_id_or_index(self, p):
        '''id_or_index : ID
                       | number
        '''
        p[0] = p[1]

    def p_count_type(self, p):
        '''count_type : BYTES
                      | PACKETS
        '''
        p[0] = p[1]

    def p_logical_cond(self, p):
        '''logical_cond : EQEQ
                        | NEQ
                        | LEQ
                        | LE
                        | GEQ
                        | GT
        '''
        p[0] = p[1]

    def p_match_list_one(self, p):
        'match_list : match'
        p[0] = [p[1]]

    def p_match_list_many(self, p):
        'match_list : match_list match'
        p[0] = p[1] + [p[2]]

    def p_match(self, p):
        'match : qualified_name COLON number_or_lpm'
        p[0] = (p[1], p[3])

    def p_qualified_name_one(self, p):
        'qualified_name : ID'
        p[0] = p[1]

    # \TODO: how do we handle stacks in either PD or PI??
    # for now we escape the brackets ...
    def p_qualified_name_array(self, p):
        'qualified_name : ID LBRACKET INT_CONST_DEC RBRACKET'
        p[0] = p[1] + '[' + p[3] + ']'

    def p_qualified_name_many(self, p):
        'qualified_name : qualified_name DOT ID'
        p[0] = p[1] + '.' + p[3]

    def p_action(self, p):
        'action : qualified_name LPAREN args RPAREN'
        p[0] = (p[1], p[3])

    def p_args_none(self, p):
        'args :'
        p[0] = []

    def p_args_many(self, p):
        'args : arg_list'
        p[0] = p[1]

    def p_arg_list_one(self, p):
        'arg_list : arg'
        p[0] = [p[1]]

    def p_arg_list_many(self, p):
        'arg_list : arg_list COMMA arg'
        p[0] = p[1] + [p[3]]

    def p_arg(self, p):
        'arg : ID COLON number'
        p[0] = (p[1], p[3])

    def p_priority(self, p):
        'priority : INT_CONST_DEC'
        p[0] = p[1]

    def p_lpm(self, p):
        'number_or_lpm : number SLASH number'
        p[0] = (p[1], p[3])

    def p_number_or_lpm(self, p):
        'number_or_lpm : number'
        p[0] = p[1]

    def p_number(self, p):
        '''number : INT_CONST_DEC
                  | INT_CONST_BIN
                  | TERN_CONST_HEX
                  | INT_CONST_HEX'''
        p[0] = p[1]

    def p_packet_data_port(self, p):
        'port : DATA_DEC'
        p[0] = p[1]

    def p_packet_data_one(self, p):
        'packet_data : packet_datum'
        p[0] = [p[1]]

    def p_packet_data_many(self, p):
        'packet_data : packet_data packet_datum'
        p[0] = p[1] + [p[2]]

    def p_packet_datum(self, p):
        '''packet_datum : DATA_HEX
                        | DATA_DEC'''
        p[0] = p[1]

    def p_expect_data_one(self, p):
        'expect_data : expect_datum'
        p[0] = [p[1]]

    def p_expect_data_many(self, p):
        'expect_data : expect_data expect_datum'
        p[0] = p[1] + [p[2]]

    def p_expect_dataum(self, p):
        '''expect_datum : packet_datum
                        | DATA_TERN'''
        p[0] = p[1]

# TESTING ---------------------------------------------------------------------
#

if __name__ == '__main__':
    data = '''
    ADD test2 100 data.f2:0x04040404 c4_6(val4:0x4040, val5:0x5050, val6:0x6060, port:2)
    add tab1 dstAddr:0xa1a2a3a4a5a6 act(port:2) = A
    # multiple match fields and ids with $
    add test1 0 data.$valid:0b1 data.f1:0x****0101 setb1(val:0x7f, port:2)

    setdefault ethertype_match l2_packet()

    expect 1 *1010101 03030303 0101 0202 0303 0404 0505 0606
    packet 0 01010101 03030303 9999 8888 7777 6666 5555 4444

    # expect with out data
    expect 5

    check_counter cntDum(10) packets == 2
    check_counter cntDum($A)
    wait
    '''

    parser = STFParser()
    stf, errs = parser.parse(data)
    if errs == 0:
        print('\n'.join(map(str, stf)))
<commit_msg>Simple stf parser fix. (#2328)<commit_after># Copyright 2013-present Barefoot Networks, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# -----------------------------------------------------------------------------
# stf_parser.py
#
# Parse an STF file.
# -----------------------------------------------------------------------------

import ply.yacc as yacc
from .stf_lexer import STFLexer

# PARSER GRAMMAR --------------------------------------------------------------

# statement : ADD qualified_name priority match_list action [= ID]
#           | ADD qualified_name match_list action [= ID]
#           | REMOVE [ALL | entry]
#           | CHECK_COUNTER ID(id_or_index) [count_type logical_cond number]
#           | EXPECT port expect_data
#           | EXPECT port
#           | NO_PACKET
#           | PACKET port packet_data
#           | SETDEFAULT qualified_name action
#           | WAIT
#
# match_list : match
#            | match_list match
# match : qualified_name COLON number_or_lpm
# number_or_lpm: number SLASH number
#              | number
# number : INT_CONST_DEC
#        | INT_CONST_BIN
#        | INT_CONST_HEX
#        | TERN_CONST_HEX
#
# qualified_name : ID
#                | ID '[' INT_CONST_DEC ']'
#                | qualified_name DOT ID
#
# id_or_index: ID
#            | number
# count_type : BYTES
#            | PACKETS
# logical_cond : EQEQ
#              | NEQ
#              | LEQ
#              | LE
#              | GEQ
#              | GT
#
# action : qualified_name LPAREN args RPAREN
# args :
#      | arg_list
# arg_list: arg
#      | arg_list COMMA arg
# arg : ID COLON number
#
# port : DATA_DEC
#
# priority : INT_CONST_DEC
#
# expect_data : expect_datum
#             | expect_data expect_datum
# packet_data : packet_datum
#             | packet_data packet_datum
#
# expect_datum : packet_datum | DATA_TERN
# packet_datum : DATA_DEC | DATA_HEX

# PARSER ----------------------------------------------------------------------

class STFParser:
    def __init__(self):
        self.lexer = STFLexer()
        self.lexer.build()
        self.tokens = self.lexer.tokens
        self.parser = yacc.yacc(module=self)
        self.errors_cnt = 0

    def parse(self, data=None, filename=''):
        if data is None and filename == '':
            raise ValueError("Please specify either a filename or data")

        # if we specified only the filename, initialize the data
        if data is None:
            with open(filename) as f:
                data = f.read()

        self.lexer.filename = filename
        self.lexer.reset_lineno()
        stf_ast = self.parser.parse(input=data,
                                    lexer=self.lexer)
        self.errors_cnt += self.lexer.errors_cnt
        return stf_ast, self.errors_cnt

    def print_error(self, lineno, lexpos, msg):
        self.errors_cnt += 1
        print("parse error (%s%s:%s): %s" % (
            '%s:' % self.lexer.filename if self.lexer.filename else '',
            lineno,
            lexpos,
            msg))

    def get_filename(self):
        return self.lexer.filename

    def p_error(self, p):
        if p is None:
            self.print_error(self.lexer.get_lineno(), 0,
                             "Unexpected end-of-file.  Missing paren?")
        else:
            self.print_error(p.lineno, self.lexer.get_colno(),
                             "Syntax error while parsing at token '%s' (of type %s)." % (p.value, p.type))
            # Skip to the next statement.
            while True:
                tok = self.parser.token()
                if not tok or tok.type in self.lexer.keywords:
                    break
                self.parser.restart()

    def p_statements_none(self, p):
        'statements : '
        p[0] = []

    def p_statements_one(self, p):
        'statements : statement'
        p[0] = [p[1]]

    def p_statements_many(self, p):
        'statements : statements statement'
        p[0] = p[1] + [p[2]]

    def p_statement_add(self, p):
        'statement : ADD qualified_name match_list action'
        p[0] = (p[1].lower(), p[2], None, p[3], p[4], None)

    def p_statement_add_with_priority(self, p):
        'statement : ADD qualified_name priority match_list action'
        p[0] = (p[1].lower(), p[2], p[3], p[4], p[5], None)

    def p_statement_add_with_id(self, p):
        'statement : ADD qualified_name match_list action EQUAL ID'
        p[0] = (p[1].lower(), p[2], None, p[3], p[4], p[6])

    def p_statement_add_with_priority_and_id(self, p):
        'statement : ADD qualified_name priority match_list action EQUAL ID'
        p[0] = (p[1].lower(), p[2], p[3], p[4], p[5], p[7])

    # remove all entries.
    # \TODO: removal of individual entries
    def p_statement_remove(self, p):
        'statement : REMOVE ALL'
        p[0] = (p[1].lower(), p[2])

    def p_statement_check_counter(self, p):
        'statement : CHECK_COUNTER ID LPAREN id_or_index RPAREN'
        p[0] = (p[1].lower(), p[2], p[4], (None, '==', 0))

    def p_statement_check_counter_with_check(self, p):
        'statement : CHECK_COUNTER ID LPAREN id_or_index RPAREN count_type logical_cond number'
        p[0] = (p[1].lower(), p[2], p[4], (p[6], p[7], p[8]))

    def p_statement_expect(self, p):
        'statement : EXPECT port'
        p[0] = (p[1].lower(), p[2], None)

    def p_statement_expect_data(self, p):
        'statement : EXPECT port expect_data'
        p[0] = (p[1].lower(), p[2], ''.join(p[3]))

    def p_statement_no_packet(self, p):
        'statement : NO_PACKET'
        p[0] = ('expect', None, None)

    def p_statement_packet(self, p):
        'statement : PACKET port packet_data'
        p[0] = (p[1].lower(), p[2], ''.join(p[3]))

    def p_statement_setdefault(self, p):
        'statement : SETDEFAULT qualified_name action'
        p[0] = (p[1].lower(), p[2], p[3])

    def p_statement_wait(self, p):
        'statement : WAIT'
        p[0] = (p[1].lower(),)

    def p_id_or_index(self, p):
        '''id_or_index : ID
                       | number
        '''
        p[0] = p[1]

    def p_count_type(self, p):
        '''count_type : BYTES
                      | PACKETS
        '''
        p[0] = p[1]

    def p_logical_cond(self, p):
        '''logical_cond : EQEQ
                        | NEQ
                        | LEQ
                        | LE
                        | GEQ
                        | GT
        '''
        p[0] = p[1]

    def p_match_list_one(self, p):
        'match_list : match'
        p[0] = [p[1]]

    def p_match_list_many(self, p):
        'match_list : match_list match'
        p[0] = p[1] + [p[2]]

    def p_match(self, p):
        'match : qualified_name COLON number_or_lpm'
        p[0] = (p[1], p[3])

    def p_qualified_name_one(self, p):
        'qualified_name : ID'
        p[0] = p[1]

    # \TODO: how do we handle stacks in either PD or PI??
    # for now we escape the brackets ...
    def p_qualified_name_array(self, p):
        'qualified_name : ID LBRACKET INT_CONST_DEC RBRACKET'
        p[0] = p[1] + '[' + p[3] + ']'

    def p_qualified_name_many(self, p):
        'qualified_name : qualified_name DOT ID'
        p[0] = p[1] + '.' + p[3]

    def p_action(self, p):
        'action : qualified_name LPAREN args RPAREN'
        p[0] = (p[1], p[3])

    def p_args_none(self, p):
        'args :'
        p[0] = []

    def p_args_many(self, p):
        'args : arg_list'
        p[0] = p[1]

    def p_arg_list_one(self, p):
        'arg_list : arg'
        p[0] = [p[1]]

    def p_arg_list_many(self, p):
        'arg_list : arg_list COMMA arg'
        p[0] = p[1] + [p[3]]

    def p_arg(self, p):
        'arg : ID COLON number'
        p[0] = (p[1], p[3])

    def p_priority(self, p):
        'priority : INT_CONST_DEC'
        p[0] = p[1]

    def p_lpm(self, p):
        'number_or_lpm : number SLASH number'
        p[0] = (p[1], p[3])

    def p_number_or_lpm(self, p):
        'number_or_lpm : number'
        p[0] = p[1]

    def p_number(self, p):
        '''number : INT_CONST_DEC
                  | INT_CONST_BIN
                  | TERN_CONST_HEX
                  | INT_CONST_HEX'''
        p[0] = p[1]

    def p_packet_data_port(self, p):
        'port : DATA_DEC'
        p[0] = p[1]

    def p_packet_data_one(self, p):
        'packet_data : packet_datum'
        p[0] = [p[1]]

    def p_packet_data_many(self, p):
        'packet_data : packet_data packet_datum'
        p[0] = p[1] + [p[2]]

    def p_packet_datum(self, p):
        '''packet_datum : DATA_HEX
                        | DATA_DEC'''
        p[0] = p[1]

    def p_expect_data_one(self, p):
        'expect_data : expect_datum'
        p[0] = [p[1]]

    def p_expect_data_many(self, p):
        'expect_data : expect_data expect_datum'
        p[0] = p[1] + [p[2]]

    def p_expect_dataum(self, p):
        '''expect_datum : packet_datum
                        | DATA_TERN'''
        p[0] = p[1]

# TESTING ---------------------------------------------------------------------
#

if __name__ == '__main__':
    data = '''
    ADD test2 100 data.f2:0x04040404 c4_6(val4:0x4040, val5:0x5050, val6:0x6060, port:2)
    add tab1 dstAddr:0xa1a2a3a4a5a6 act(port:2) = A
    # multiple match fields and ids with $
    add test1 0 data.$valid:0b1 data.f1:0x****0101 setb1(val:0x7f, port:2)

    setdefault ethertype_match l2_packet()

    expect 1 *1010101 03030303 0101 0202 0303 0404 0505 0606
    packet 0 01010101 03030303 9999 8888 7777 6666 5555 4444

    # expect with out data
    expect 5

    check_counter cntDum(10) packets == 2
    check_counter cntDum($A)
    wait
    '''

    parser = STFParser()
    stf, errs = parser.parse(data)
    if errs == 0:
        print('\n'.join(map(str, stf)))
<|endoftext|>


<commit_before>#!/usr/bin/env python

import re
import subprocess

db_admin_user = 'postgres'
original_db = 'original'

def psql(*args):
    subprocess.check_call(['psql', '-U', db_admin_user] + list(args))

def load(path):
    subprocess.check_call('zcat %s | psql -U %s -d %s' % (
            path, db_admin_user, original_db), shell=True)

def pg_dump(to_file):
    pg = subprocess.Popen(['pg_dump', '--schema-only', '-U', db_admin_user,
                           original_db], stdout=subprocess.PIPE)

    comments = []
    columns = []
    in_table = False

    with open(to_file, 'w') as out:
        for line in pg.stdout:
            # skip SQL comments and blank lines
            if line.startswith('--') or line == '\n':
                continue

            # normalize conditions (for some reason pg_dump dumps them
            # differently before and after an upgrade)
            if line.startswith('    CONSTRAINT'):
                line = re.sub(r"\('(\w+)'::character varying\)::text",
                              r"'\1'::character varying", line)
                line = re.sub(r"ANY \(\(ARRAY\[(.*?)\]\)::text\[\]\)",
                              r"ANY (ARRAY[\1])", line)

            # reorder table columns
            if in_table:
                if line == ');\n':
                    # table finished
                    in_table = False
                else:
                    # accumulate table lines
                    line = re.sub(r",\n", r"\n", line)
                    columns.append(line)
                    continue

                # write table lines sorted
                for column in sorted(columns):
                    out.write(column)

            # move comments to the end
            if line.startswith('COMMENT '):
                comments.append(line)
                continue

            if line.startswith('CREATE TABLE'):
                in_table = True;
                columns = []

            out.write(line)

        # write out comments sorted
        for comment in sorted(comments):
            out.write(comment)

# DB from create_oq_schema
psql('-c', "DROP DATABASE IF EXISTS %s" % original_db)
psql('-c', "CREATE DATABASE %s" % original_db)

subprocess.check_call(['bin/create_oq_schema', '--yes',
                       '--db-name=%s' % original_db,
                       '--db-user=%s' % db_admin_user,
                       '--schema-path=%s' % 'openquake/db/schema'])

pg_dump('/tmp/fresh.sql')

# DB from dbmaint.py
psql('-c', "DROP DATABASE IF EXISTS %s" % original_db)
psql('-c', "CREATE DATABASE %s" % original_db)
load('tools/test_migration_base.sql.gz')

subprocess.check_call(['tools/dbmaint.py', '--db', original_db,
                       '-U', db_admin_user])

pg_dump('/tmp/after.sql')

subprocess.call('diff -u /tmp/fresh.sql /tmp/after.sql | less -F', shell=True)
<commit_msg>Exclude public schemas from the diff (suggested by angri).<commit_after>#!/usr/bin/env python

import re
import subprocess

db_admin_user = 'postgres'
original_db = 'original'

def psql(*args):
    subprocess.check_call(['psql', '-U', db_admin_user] + list(args))

def load(path):
    subprocess.check_call('zcat %s | psql -U %s -d %s' % (
            path, db_admin_user, original_db), shell=True)

def pg_dump(to_file):
    pg = subprocess.Popen(['pg_dump', '--schema-only', '-U', db_admin_user,
                           '--exclude-schema', 'public',
                           original_db], stdout=subprocess.PIPE)

    comments = []
    columns = []
    in_table = False

    with open(to_file, 'w') as out:
        for line in pg.stdout:
            # skip SQL comments and blank lines
            if line.startswith('--') or line == '\n':
                continue

            # normalize conditions (for some reason pg_dump dumps them
            # differently before and after an upgrade)
            if line.startswith('    CONSTRAINT'):
                line = re.sub(r"\('(\w+)'::character varying\)::text",
                              r"'\1'::character varying", line)
                line = re.sub(r"ANY \(\(ARRAY\[(.*?)\]\)::text\[\]\)",
                              r"ANY (ARRAY[\1])", line)

            # reorder table columns
            if in_table:
                if line == ');\n':
                    # table finished
                    in_table = False
                else:
                    # accumulate table lines
                    line = re.sub(r",\n", r"\n", line)
                    columns.append(line)
                    continue

                # write table lines sorted
                for column in sorted(columns):
                    out.write(column)

            # move comments to the end
            if line.startswith('COMMENT '):
                comments.append(line)
                continue

            if line.startswith('CREATE TABLE'):
                in_table = True;
                columns = []

            out.write(line)

        # write out comments sorted
        for comment in sorted(comments):
            out.write(comment)

# DB from create_oq_schema
psql('-c', "DROP DATABASE IF EXISTS %s" % original_db)
psql('-c', "CREATE DATABASE %s" % original_db)

subprocess.check_call(['bin/create_oq_schema', '--yes',
                       '--db-name=%s' % original_db,
                       '--db-user=%s' % db_admin_user,
                       '--schema-path=%s' % 'openquake/db/schema'])

pg_dump('/tmp/fresh.sql')

# DB from dbmaint.py
psql('-c', "DROP DATABASE IF EXISTS %s" % original_db)
psql('-c', "CREATE DATABASE %s" % original_db)
load('tools/test_migration_base.sql.gz')

subprocess.check_call(['tools/dbmaint.py', '--db', original_db,
                       '-U', db_admin_user])

pg_dump('/tmp/after.sql')

subprocess.call('diff -u /tmp/fresh.sql /tmp/after.sql | less -F', shell=True)
<|endoftext|>