<commit_before>from common import PynException

def parse(msg, fname):
    return NinjaParser().parse(msg, fname)

class NinjaParser(object):
    """Parse the contents of a .ninja file and return an AST."""

    def parse(self, msg, fname):
        v, p, err = self.grammar_(msg, 0, len(msg))
        if err:
            lineno = 1
            colno = 1
            for c in xrange(p):
                if msg[c] == '\n':
                    lineno += 1
                    colno = 1
                else:
                    colno += 1

            pos_str = '%s:%d:%d' % (fname, lineno, colno)
            raise PynException("%s %s" % (pos_str, err))
        else:
            return v

    def expect(self, msg, start, end, substr):
        l = len(substr)
        if (end - start) < l or msg[start:start + l] != substr:
            return None, start, 'expecting "%s" at %d' % (substr, start)
        return substr, start + l, None

    def grammar_(self, msg, start, end):
        """ ((empty_line* decl)*:ds empty_line*) end -> ds """
        ds = []
        err = None
        p = start
        while not err and p < end:
            while not err and p < end:
                _, p, err = self.empty_line_(msg, p, end)
            orig_p = p
            if p < end:
                v, p, err = self.decl_(msg, orig_p, end)
                if not err:
                    ds.append(v)
        if not err:
            return ds, p, err
        else:
            return None, p, err

    def decl_(self, msg, start, end):
        """ build | rule | subninja | include | pool | default | var """
        v, p, err = self.build_(msg, start, end)
        if not err:
            return v, p, None

        v, p, err = self.rule_(msg, start, end)
        if not err:
            return v, p, None

        v, p, err = self.subninja_(msg, start, end)
        if not err:
            return v, p, None

        v, p, err = self.include_(msg, start, end)
        if not err:
            return v, p, None

        v, p, err = self.pool_(msg, start, end)
        if not err:
            return v, p, None

        v, p, err = self.default_(msg, start, end)
        if not err:
            return v, p, None

        v, p, err = self.var_(msg, start, end)
        if not err:
            return v, p, None

        return None, p, ("expecting one of 'build', 'rule', 'subninja', "
                         "'include', 'pool', 'default', or a a variable name")

    def build_(self, msg, start, end):
        """ "build" ws paths:os ws? ':' ws name:rule
            explicit_deps:eds implicit_deps:ids order_only_deps:ods eol
            ws_vars:vs -> ['build', os, rule, eds, ids, ods, vs] """
        p = start
        v, p, err = self.expect(msg, p, end, 'build')
        if err:
            return v, p, err

        _, p, err = self.ws_(msg, p, end)
        if err:
            return None, p, err

        os, p, err = self.paths_(msg, p, end)
        if err:
            return None, p, err

        _, p, _ = self.ws_(msg, p, end)

        v, p, err = self.expect(msg, p, end, ':')
        if err:
            return None, p, err

        _, p, _ = self.ws_(msg, p, end)

        rule, p, err = self.name_(msg, p, end)
        if err:
            return None, p, err

        eds, p, _ = self.explicit_deps_(msg, p, end)

        ids, p, err = self.implicit_deps_(msg, p, end)
        if err:
            return None, p, err

        ods, p, err = self.order_only_deps_(msg, p, end)
        if err:
            return None, p, err

        _, p, err = self.eol_(msg, p, end)
        if err:
            return None, p, err

        vs, p, _ = self.ws_vars_(msg, p, end)

        return ['build', os, rule, eds, ids, ods, vs], p, None

    def rule_(self, msg, start, end):
        """ "rule" ws name:n eol ws_vars:vs -> ['rule', n, vs] """
        p = start

        _, p, err = self.expect(msg, p, end, 'rule')
        if err:
            return None, p, err

        _, p, err = self.ws_(msg, p, end)
        if err:
            return None, p, err

        n, p, err = self.name_(msg, p, end)
        if err:
            return None, p, err

        _, p, err = self.eol_(msg, p, end)
        if err:
            return None, p, err

        vs, p, _ = self.ws_vars_(msg, p, end)

        return ['rule', n, vs], p, None

    def ws_vars_(self, msg, start, end):
        """ (ws var)*:vs -> vs """
        p = start
        vs = []
        err = None
        while p < end and not err:
            _, p, err = self.ws_(msg, p, end)
            if not err:
                v, p, err = self.var_(msg, p, end)
                if not err:
                    vs.append(v)
        return vs, p, None

    def var_(self, msg, start, end):
        """ name:n ws? '=' ws? value:v eol -> ['var', n, v] """
        n, p, err = self.name_(msg, start, end)
        if err:
            return None, p, err

        _, p, _ = self.ws_(msg, p, end)

        _, p, err = self.expect(msg, p, end, '=')
        if err:
            return None, p, err

        _, p, _ = self.ws_(msg, p, end)

        v, p, _ = self.value_(msg, p, end)

        _, p, _ = self.eol_(msg, p, end)
        return ['var', n, v], p, None

    def value_(self, msg, start, end):
        """ (~eol (('$' ' ' -> ' ')|('$' '\n' ' '+ -> '')|anything))*:vs
            -> ''.join(vs) """
        p = start
        vs = []
        _, _, err = self.eol_(msg, p, end)
        while err and p < end:
            if p < (end - 1) and msg[p:p + 2] == '$ ':
                vs.append(' ')
                p += 2
            elif p < (end - 1) and msg[p:p + 2] == '$\n':
                p += 2
                _, p, _ = self.ws_(msg, p, end)
            else:
                vs.append(msg[p])
                p += 1
            _, _, err = self.eol_(msg, p, end)
        return ''.join(vs), p, None

    def subninja_(self, msg, start, end):
        """ "subninja" ws path:p eol -> ['subninja', p] """
        v, p, err = self.expect(msg, start, end, 'subninja')
        if err:
            return v, p, err

        v, p, err = self.ws_(msg, p, end)
        if err:
            return None, p, err

        v, p, err = self.path_(msg, p, end)
        if err:
            return None, p, err

        _, p, err = self.eol_(msg, p, end)
        return ['subninja', v], p, None

    def include_(self, msg, start, end):
        """ "include" ws path:p eol -> ['include', p] """
        v, p, err = self.expect(msg, start, end, 'include')
        if err:
            return v, p, err

        v, p, err = self.ws_(msg, p, end)
        if err:
            return None, p, err

        v, p, err = self.path_(msg, p, end)
        if err:
            return None, p, err

        _, p, err = self.eol_(msg, p, end)
        return ['include', v], p, None

    def pool_(self, msg, start, end):
        """ "pool" ws name:n eol ws_vars:vs -> ['pool', n, vs] """
        _, p, err = self.expect(msg, start, end, 'pool')
        if err:
            return None, p, err
        _, p, err = self.ws_(msg, p, end)
        if err:
            return None, p, err
        n, p, err = self.name_(msg, p, end)
        if err:
            return None, p, err
        _, p, err = self.eol_(msg, p, end)
        if err:
            return None, p, err
        vs, p, _ = self.ws_vars_(msg, p, end)
        return ['pool', n, vs], p, None

    def default_(self, msg, start, end):
        """ "default" ws paths:ps eol  -> ['default', ps] """
        v, p, err = self.expect(msg, start, end, 'default')
        if err:
            return v, p, err

        v, p, err = self.ws_(msg, p, end)
        if err:
            return None, p, err

        v, p, err = self.paths_(msg, p, end)
        if err:
            return None, p, err

        _, p, err = self.eol_(msg, p, end)
        return ['default', v], p, None

    def paths_(self, msg, start, end):
        """ path:hd (ws path)*:tl -> [hd] + tl """
        hd, p, err = self.path_(msg, start, end)
        tl = []
        if err:
            return None, p, err

        while p < end and not err:
            v, p, err = self.ws_(msg, p, end)
            if not err:
                v, p, err = self.path_(msg, p, end)
                if not err:
                    tl.append(v)
        return [hd] + tl, p, None

    def path_(self, msg, start, end):
        """(('$' ' ')|(~(' '|':'|'='|'|'|eol) anything))+:vs -> ''.join(vs)"""
        vs = []
        p = start
        while p < end:
            c = msg[p]
            if c.isalpha():
                vs.append(c)
                p += 1
            elif c == ' ':
                break
            elif c == '\n':
                break
            elif c == '$' and (p < end - 1) and msg[p+1] == ' ':
                vs.append(' ')
                p += 2
            elif c in (':', '=', '|'):
                break
            else:
                vs.append(c)
                p += 1
        if len(vs) == 0:
            return None, start, 'expecting a path'
        else:
            return ''.join(vs), p, None

    def name_(self, msg, start, end):
        """ (letter|'_'):hd (letter|digit|'_')*:tl -> ''.join([hd] + tl) """
        p = start
        if p == end:
            return None, p, 'expecting a name'
        hd = msg[p]
        if not hd.isalpha() and hd != '_':
            return None, p, 'expecting a letter or "_" to start a name'
        p += 1
        tl = []
        while p < end and (msg[p].isalpha() or msg[p].isdigit() or
                           msg[p] == '_'):
            tl.append(msg[p])
            p += 1
        return hd + ''.join(tl), p, None

    def explicit_deps_(self, msg, start, end):
        """ ws? paths:ps -> ps | -> [] """
        if start == end:
            return [], start, None

        _, p, _ = self.ws_(msg, start, end)

        ps, p, err = self.paths_(msg, p, end)
        if err:
            return [], start, None
        return ps, p, None

    def implicit_deps_(self, msg, start, end):
        """ ws? (~'|' '|') '|' ws? paths:ps -> ps | -> [] """
        if start == end:
            return [], start, None

        _, p, _ = self.ws_(msg, start, end)

        if (p < end - 1) and msg[p] == '|' and msg[p + 1] == '|':
            return [], start, None

        _, p, err = self.expect(msg, p, end, '|')
        if err:
            return [], start, None

        _, p, _ = self.ws_(msg, p, end)

        ps, p, err = self.paths_(msg, p, end)
        if err:
            return None, p, err
        return ps, p, None

    def order_only_deps_(self, msg, start, end):
        """ ws? '|' '|' ws? paths:ps -> ps | -> [] """
        if start == end:
            return [], start, None

        _, p, _ = self.ws_(msg, start, end)

        _, p, err = self.expect(msg, p, end, '||')
        if err:
            return [], start, None

        _, p, _ = self.ws_(msg, p, end)

        ps, p, err = self.paths_(msg, p, end)
        if err:
            return None, p, err
        return ps, p, None

    def empty_line_(self, msg, start, end):
        """ ws? comment? ('\n'|end) """
        _, p, _ = self.ws_(msg, start, end)
        if p < end:
            _, p, _ = self.comment_(msg, p, end)
        if p < end and msg[p] == '\n':
            return '\n', p + 1, None
        elif p >= end:
            return None, p, None
        return None, p, "expecting a '\n' or EOF"

    def eol_(self, msg, start, end):
        """ ws? ('\n' | end) """
        if start < end and (msg[start] == ' ' or msg[start] == '$'):
            _, p, _ = self.ws_(msg, start, end)
        else:
            p = start
        if p < end:
            if msg[p] == '\n':
                return '\n', p + 1, None
            else:
                return None, p, "expecting a newline, or EOF"
        else:
            return None, p, None

    def ws_(self, msg, start, end):
        """ (' '|('$' '\n'))+ """
        p = start
        if start == end:
            return None, p, "expecting either ' ' or '$\n'"
        err = None
        while not err and p < end:
            if msg[p] == ' ':
                p += 1
            elif msg[p] == '$' and p < end - 1 and msg[p+1] == '\n':
                p += 2
            else:
                err = "expecting either ' ' or '$\n'"
        if p == start:
            return None, p, err
        else:
            return None, p, None

    def comment_(self, msg, start, end):
        """ '#' (~'\n' anything)* ('\n'|end) """
        p = start
        if msg[p] != '#':
            return None, start, "expecting a '#'"
        p += 1
        while p < end and msg[p] != '\n':
            p += 1
        if p < end:
            return '\n', p, None
        else:
            return None, p, None
<commit_msg>refactor parser a bit<commit_after>from common import PynException

def parse(msg, fname):
    return NinjaParser(msg, fname).parse()

class NinjaParser(object):
    """Parse the contents of a .ninja file and return an AST."""
    def __init__(self, msg, fname):
        self.msg = msg
        self.end = len(msg)
        self.fname = fname

    def parse(self):
        msg = self.msg
        v, p, err = self.grammar_(0)
        if err:
            lineno = 1
            colno = 1
            i = 0
            while i < p:
                if msg[i] == '\n':
                    lineno += 1
                    colno = 1
                else:
                    colno += 1
                i += 1

            pos_str = '%s:%d:%d' % (self.fname, lineno, colno)
            raise PynException("%s %s" % (pos_str, err))
        else:
            return v

    def expect(self, start, substr):
        l = len(substr)
        if (self.end - start) < l or self.msg[start:start + l] != substr:
            return None, start, 'expecting "%s" at %d' % (substr, start)
        return substr, start + l, None

    def grammar_(self, start):
        """ ((empty_line* decl)*:ds empty_line*) end -> ds """
        ds = []
        err = None
        p = start
        while not err and p < self.end:
            while not err and p < self.end:
                _, p, err = self.empty_line_(p)
            orig_p = p
            if p < self.end:
                v, p, err = self.decl_(orig_p)
                if not err:
                    ds.append(v)
        if not err:
            return ds, p, err
        else:
            return None, p, err

    def decl_(self, start):
        """ build | rule | subninja | include | pool | default | var """
        v, p, err = self.build_(start)
        if not err:
            return v, p, None

        v, p, err = self.rule_(start)
        if not err:
            return v, p, None

        v, p, err = self.subninja_(start)
        if not err:
            return v, p, None

        v, p, err = self.include_(start)
        if not err:
            return v, p, None

        v, p, err = self.pool_(start)
        if not err:
            return v, p, None

        v, p, err = self.default_(start)
        if not err:
            return v, p, None

        v, p, err = self.var_(start)
        if not err:
            return v, p, None

        return None, p, ("expecting one of 'build', 'rule', 'subninja', "
                         "'include', 'pool', 'default', or a a variable name")

    def build_(self, start):
        """ "build" ws paths:os ws? ':' ws name:rule
            explicit_deps:eds implicit_deps:ids order_only_deps:ods eol
            ws_vars:vs -> ['build', os, rule, eds, ids, ods, vs] """
        p = start
        v, p, err = self.expect(p, 'build')
        if err:
            return v, p, err

        _, p, err = self.ws_(p)
        if err:
            return None, p, err

        os, p, err = self.paths_(p)
        if err:
            return None, p, err

        _, p, _ = self.ws_(p)

        v, p, err = self.expect(p, ':')
        if err:
            return None, p, err

        _, p, _ = self.ws_(p)

        rule, p, err = self.name_(p)
        if err:
            return None, p, err

        eds, p, _ = self.explicit_deps_(p)

        ids, p, err = self.implicit_deps_(p)
        if err:
            return None, p, err

        ods, p, err = self.order_only_deps_(p)
        if err:
            return None, p, err

        _, p, err = self.eol_(p)
        if err:
            return None, p, err

        vs, p, _ = self.ws_vars_(p)

        return ['build', os, rule, eds, ids, ods, vs], p, None

    def rule_(self, start):
        """ "rule" ws name:n eol ws_vars:vs -> ['rule', n, vs] """
        p = start

        _, p, err = self.expect(p, 'rule')
        if err:
            return None, p, err

        _, p, err = self.ws_(p)
        if err:
            return None, p, err

        n, p, err = self.name_(p)
        if err:
            return None, p, err

        _, p, err = self.eol_(p)
        if err:
            return None, p, err

        vs, p, _ = self.ws_vars_(p)

        return ['rule', n, vs], p, None

    def ws_vars_(self, start):
        """ (ws var)*:vs -> vs """
        p = start
        vs = []
        err = None
        while p < self.end and not err:
            _, p, err = self.ws_(p)
            if not err:
                v, p, err = self.var_(p)
                if not err:
                    vs.append(v)
        return vs, p, None

    def var_(self, start):
        """ name:n ws? '=' ws? value:v eol -> ['var', n, v] """
        n, p, err = self.name_(start)
        if err:
            return None, p, err

        _, p, _ = self.ws_(p)

        _, p, err = self.expect(p, '=')
        if err:
            return None, p, err

        _, p, _ = self.ws_(p)

        v, p, _ = self.value_(p)

        _, p, _ = self.eol_(p)
        return ['var', n, v], p, None

    def value_(self, start):
        """ (~eol (('$' ' ' -> ' ')|('$' '\n' ' '+ -> '')|anything))*:vs
            -> ''.join(vs) """
        msg, p, end = self.msg, start, self.end

        vs = []
        _, _, err = self.eol_(p)
        while err and p < end:
            if p < (end - 1) and msg[p:p + 2] == '$ ':
                vs.append(' ')
                p += 2
            elif p < (end - 1) and msg[p:p + 2] == '$\n':
                p += 2
                _, p, _ = self.ws_(p)
            else:
                vs.append(msg[p])
                p += 1
            _, _, err = self.eol_(p)
        return ''.join(vs), p, None

    def subninja_(self, start):
        """ "subninja" ws path:p eol -> ['subninja', p] """
        v, p, err = self.expect(start, 'subninja')
        if err:
            return v, p, err

        v, p, err = self.ws_(p)
        if err:
            return None, p, err

        v, p, err = self.path_(p)
        if err:
            return None, p, err

        _, p, err = self.eol_(p)
        return ['subninja', v], p, None

    def include_(self, start):
        """ "include" ws path:p eol -> ['include', p] """
        v, p, err = self.expect(start, 'include')
        if err:
            return v, p, err

        v, p, err = self.ws_(p)
        if err:
            return None, p, err

        v, p, err = self.path_(p)
        if err:
            return None, p, err

        _, p, err = self.eol_(p)
        return ['include', v], p, None

    def pool_(self, start):
        """ "pool" ws name:n eol ws_vars:vs -> ['pool', n, vs] """
        _, p, err = self.expect(start, 'pool')
        if err:
            return None, p, err
        _, p, err = self.ws_(p)
        if err:
            return None, p, err
        n, p, err = self.name_(p)
        if err:
            return None, p, err
        _, p, err = self.eol_(p)
        if err:
            return None, p, err
        vs, p, _ = self.ws_vars_(p)
        return ['pool', n, vs], p, None

    def default_(self, start):
        """ "default" ws paths:ps eol  -> ['default', ps] """
        v, p, err = self.expect(start, 'default')
        if err:
            return v, p, err

        v, p, err = self.ws_(p)
        if err:
            return None, p, err

        v, p, err = self.paths_(p)
        if err:
            return None, p, err

        _, p, err = self.eol_(p)
        return ['default', v], p, None

    def paths_(self, start):
        """ path:hd (ws path)*:tl -> [hd] + tl """
        hd, p, err = self.path_(start)
        tl = []
        if err:
            return None, p, err

        while p < self.end and not err:
            v, p, err = self.ws_(p)
            if not err:
                v, p, err = self.path_(p)
                if not err:
                    tl.append(v)
        return [hd] + tl, p, None

    def path_(self, start):
        """(('$' ' ')|(~(' '|':'|'='|'|'|eol) anything))+:vs -> ''.join(vs)"""
        msg, p, end = self.msg, start, self.end
        vs = []

        while p < end:
            c = msg[p]
            if c.isalpha():
                vs.append(c)
                p += 1
            elif c == ' ':
                break
            elif c == '\n':
                break
            elif c == '$' and (p < end - 1) and msg[p+1] == ' ':
                vs.append(' ')
                p += 2
            elif c in (':', '=', '|'):
                break
            else:
                vs.append(c)
                p += 1
        if len(vs) == 0:
            return None, start, 'expecting a path'
        else:
            return ''.join(vs), p, None

    def name_(self, start):
        """ (letter|'_'):hd (letter|digit|'_')*:tl -> ''.join([hd] + tl) """
        if start == self.end:
            return None, start, 'expecting a name'

        msg, p, end = self.msg, start, self.end
        hd = msg[p]
        if not hd.isalpha() and hd != '_':
            return None, p, 'expecting a letter or "_" to start a name'
        p += 1
        tl = []
        while p < end:
            c = msg[p]
            if c.isalpha() or c.isdigit() or c == '_':
                tl.append(c)
                p += 1
            else:
                break
        return hd + ''.join(tl), p, None

    def explicit_deps_(self, start):
        """ ws? paths:ps -> ps | -> [] """
        if start == self.end:
            return [], start, None

        _, p, _ = self.ws_(start)

        ps, p, err = self.paths_(p)
        if err:
            return [], start, None
        return ps, p, None

    def implicit_deps_(self, start):
        """ ws? (~'|' '|') '|' ws? paths:ps -> ps | -> [] """
        msg = self.msg
        if start == self.end:
            return [], start, None

        _, p, _ = self.ws_(start)

        if (p < self.end - 1) and msg[p] == '|' and msg[p + 1] == '|':
            return [], start, None

        _, p, err = self.expect(p, '|')
        if err:
            return [], start, None

        _, p, _ = self.ws_(p)

        ps, p, err = self.paths_(p)
        if err:
            return None, p, err
        return ps, p, None

    def order_only_deps_(self, start):
        """ ws? '|' '|' ws? paths:ps -> ps | -> [] """
        if start == self.end:
            return [], start, None

        _, p, _ = self.ws_(start)

        _, p, err = self.expect(p, '||')
        if err:
            return [], start, None

        _, p, _ = self.ws_(p)

        ps, p, err = self.paths_(p)
        if err:
            return None, p, err
        return ps, p, None

    def empty_line_(self, start):
        """ ws? comment? ('\n'|end) """
        msg, p, end = self.msg, start, self.end
        _, p, _ = self.ws_(start)
        if p < end:
            _, p, _ = self.comment_(p)
        if p < end and msg[p] == '\n':
            return '\n', p + 1, None
        elif p >= end:
            return None, p, None
        return None, p, "expecting a '\n' or EOF"

    def eol_(self, start):
        """ ws? ('\n' | end) """
        msg, p, end = self.msg, start, self.end
        if start < end and (msg[start] == ' ' or msg[start] == '$'):
            _, p, _ = self.ws_(start)
        else:
            p = start
        if p < end:
            if msg[p] == '\n':
                return '\n', p + 1, None
            else:
                return None, p, "expecting a newline, or EOF"
        else:
            return None, p, None

    def ws_(self, start):
        """ (' '|('$' '\n'))+ """
        msg, p, end = self.msg, start, self.end
        if start == end:
            return None, p, "expecting either ' ' or '$\n'"
        err = None
        while not err and p < end:
            if msg[p] == ' ':
                p += 1
            elif msg[p] == '$' and p < end - 1 and msg[p+1] == '\n':
                p += 2
            else:
                err = "expecting either ' ' or '$\n'"
        if p == start:
            return None, p, err
        else:
            return None, p, None

    def comment_(self, start):
        """ '#' (~'\n' anything)* ('\n'|end) """
        msg, p, end = self.msg, start, self.end
        if msg[p] != '#':
            return None, start, "expecting a '#'"
        p += 1
        while p < end and msg[p] != '\n':
            p += 1
        if p < end:
            return '\n', p, None
        else:
            return None, p, None
<|endoftext|>


<commit_before>import re

class CorruptedData(Exception):
    def __init__(self, message, data):
        super(CorruptedData, self).__init__(message)
        # clean the data form empty keys
        data_celaned = {k: v for k, v in data.items() if v != []}
        self.parsed_data = data_celaned

# regexp unita di misura
unit = r'(\(?(?:Ry|a\.u\.|bohr|\/|ang)+\)?(?:\^|\*\*)?\d*\)?)'

# information of scf
scf_set = dict(
    r_pseudopotential=r'^ *file *([\w_\-\.]+\.UPF)',
    r_bli=r'^ *bravais-lattice index *= *(\d+)',
    r_alat=r'^ *lattice parameter \(alat\) *= *([\d\.\+\-]+) *{}'.format(unit),
    r_unit_cell_volume=r'^ *unit-cell volume *= *([\d\.\+\-]+) *{}'.format(
                       unit),
    r_cell_side=r' *a\((\d+)\) *= *\( *(-?[\d.]+) *(-?[\d.]+) *'
                '(-?[\d.]+) *\)',
    r_natoms=r'^ *number of atoms/cell *= *(\d+)',
    r_nspecies=r'^ *number of atomic types *= *(\d+)',
    r_dspecies=r'^ +(\w{1,2}) +(-?[\d.]+) +(-?[\d.]+)',
    r_nelectrons=r'^ *number of electrons *= *([\d\.\+\-]+)',
    r_nksstates=r'^ *number of Kohn-Sham states *= *(\d+)',
    r_cutoff=r'^ *kinetic-energy cutoff *= *([\d\.\+\-]+) *{}'.format(unit),
    r_charge_cutoff=r'^ *charge density cutoff *= *([\d\.\+\-]+) *{}'.format(
                    unit),
    r_treshold=r'^ *convergence threshold *= *(\d+.\d+E?-?\d*)',
    r_mixing=r'^ *mixing beta *= *([\d\.\+\-]+)',
    r_apos=r'^ +(\d+) +(\w{1,2})[^=]+= \( +([\d\+\-\.]+) +([\d\+\-\.]+) +'
           '([\d\+\-\.]+) +\)')

# data of scf
# TODO, the pressure calculation is not always carried out so it should be
# putted in another dictionary
scf_data_out = dict(
    r_total_energy=r'^![\w =]+([\d\.\+\- ]+){}'.format(unit),
    r_E_hartree=r'^ +hartree \w+ += +([\d\.\+\-]+) +{}'.format(unit),
    r_E_onelectron=r'^ +one-electron \w+ += +([\d\.\+\-]+) +{}'.format(unit),
    r_E_xc=r'^ +xc \w+ += +([\d\.\+\-]+) +{}'.format(unit),
    r_E_ewald=r'^ +ewald \w+ += +([\d\.\+\-]+) +{}'.format(unit),
    r_E_paw=r'^ +one-center paw \w+\. += +([\d\.\+\-]+) +{}'.format(unit),
    r_force=r'atom +([\d\.\+\-]+) +type +([\d\.\+\-]+)'
            ' +force = +([\d\.\+\-]+) +([\d\.\+\-]+) +([\d\.\+\-]+)',
    r_stress_units=r' +total +stress +{}'.format(unit),
    r_pressure=r'\(kbar\) +P= +([\d\.\+\-]+)',
    r_stress_and_kbar_tensor=r'^ {2,3}([\d\.\+\-]+) +([\d\.\+\-]+) +'
                             r'([\d\.\+\-]+) {9,10}([\d\.\+\-]+) +'
                             r'([\d\.\+\-]+) +([\d\.\+\-]+)$')

# information of BFGS
bfgs_set = dict(
    r_bfgs_converged=r' +bfgs converged in +(\d+) +scf cycles and +(\d+) +'
                     'bfgs steps',
    t_bfgs_not_converged=r'^ +The maximum number of steps has been reached.',
    r_criteria=r'^ +\(criteria: +(energy) < ([\d\.\+\-E]+),'
               ' +(force) < ([\d\.\+\-E]+),'
               ' +(cell) < ([\d\.\+\-E]+)\)',
    r_end=r'^ +End of BFGS Geometry Optimization',
    r_start=r'^ +BFGS Geometry Optimization',
    r_nstep=r'^ *nstep *= *(\d+)',
    r_recalculation=r'The G-vectors are recalculated for the final unit cell')

# data of bfgs:
bfgs_data_out = dict(
    r_unit_cell_volume=r'^ *new unit-cell volume *= *([\d\.\+\-]+) *{}'.format(
                       unit),
    r_cell_side_units=r'CELL_PARAMETERS \(([\w ]+)= ([\d\.\+\-]+)\)',
    r_cell_side=r'^ {2,3}([\d\+\-\.]+) +([\d\+\-\.]+) +([\d\+\-\.]+)$',
    r_apos_units=r'ATOMIC_POSITIONS \((.+)\)',
    r_apos=r'^(\w{1,2}) +([\d\+\-\.]+) +([\d\+\-\.]+) +([\d\+\-\.]+)$')

# closing string:
r_close = r'JOB DONE'

def complete_scf(text):
    simulation = {}
    for x in scf_set:
        data = re.findall(scf_set[x], text, re.MULTILINE)
        if len(data) == 1:
            simulation[x[2:]] = data.pop()
        else:
            simulation[x[2:]] = data
    for x in scf_data_out:
        data = re.findall(scf_data_out[x], text, re.MULTILINE)
        if len(data) == 1:
            simulation[x[2:]] = data.pop()
        else:
            simulation[x[2:]] = data

    # scf fix
    # print('ATTENTION: POSITONS ARE ON Crystallographic axes')

    # the normalizations MUST BE DONE in the same order as the data are
    # collected becouse the first that fails will rise an error and all the
    # others wont be applied.

    # normalization of atom description
    simulation['atom_description'] = []
    for x, y in zip(simulation.pop('dspecies'),
                    simulation.pop('pseudopotential')):
        simulation['atom_description'].append((x[0], x[1], x[2], y))

    conversion = {i + 1: x[0] for i, x in
                  enumerate(simulation['atom_description'])}

    # normalization of cell description
    cell_side = simulation.pop('cell_side')
    simulation['cell_side'] = [x[1:] for x in cell_side]

    # normalization of force and positions
    simulation['atom'] = []
    nat = int(simulation['natoms'])
    pos = simulation.pop('apos')[nat:]
    force = simulation.pop('force')[:nat]
    if len(force) < nat:
        # try to save as much data as possible
        for x in pos:
            simulation['atom'].append((x[0], x[1], x[2:]))
        raise CorruptedData('not enought forces, damage data', simulation)
    for x, y in zip(pos, force):
        if x[0] == y[0] and x[1] == conversion[int(y[1])]:
            simulation['atom'].append((x[0], x[1], x[2:], y[2:]))
        else:
            raise ValueError('Error in conversion step')

    # normalization of stress and pressure information
    simulation['stress_tesnsor'] = []
    simulation['pressure_tesnsor'] = []
    for x in simulation.pop('stress_and_kbar_tensor'):
        simulation['stress_tesnsor'].append(x[:3])
        simulation['pressure_tesnsor'].append(x[3:])
    return simulation

def bfgs_complete(text):
    simulation = {}
    for x in scf_data_out:
        data = re.findall(scf_data_out[x], text, re.MULTILINE)
        if len(data) == 1:
            simulation[x[2:]] = data.pop()
        else:
            simulation[x[2:]] = data
    for x in bfgs_data_out:
        data = re.findall(bfgs_data_out[x], text, re.MULTILINE)
        if len(data) == 1:
            simulation[x[2:]] = data.pop()
        else:
            simulation[x[2:]] = data

    # normalization of force and positions
    simulation['atom'] = []
    pos = simulation.pop('apos')
    conversion = []
    for x in pos:
        if x[0] not in conversion:
            conversion.append(x[0])
    conversion = {i + 1: j for i, j in enumerate(conversion)}
    nat = len(pos)
    force = simulation.pop('force')[:nat]
    if len(force) < nat:
        if len(force) < nat:
            for x in pos:
                simulation['atom'].append((x[0], x[1:]))
        raise CorruptedData('not enought forces, damage data', simulation)
    for x, y in zip(pos, force):
        if x[0] == conversion[int(y[1])]:
            simulation['atom'].append((y[0], x[0], x[1:], y[2:]))
        else:
            raise ValueError('Error in conversion step')
    return simulation
<commit_msg>now atoms type are mathched by name not \w{1,2}<commit_after>import re

class CorruptedData(Exception):
    def __init__(self, message, data):
        super(CorruptedData, self).__init__(message)
        # clean the data form empty keys
        data_celaned = {k: v for k, v in data.items() if v != []}
        self.parsed_data = data_celaned

# util to regexp
unit = r'(\(?(?:Ry|a\.u\.|bohr|\/|ang)+\)?(?:\^|\*\*)?\d*\)?)'
atoms_name = r'(?:C|H|O|N)'

# information of scf
scf_set = dict(
    r_pseudopotential=r'^ *file *([\w_\-\.]+\.UPF)',
    r_bli=r'^ *bravais-lattice index *= *(\d+)',
    r_alat=r'^ *lattice parameter \(alat\) *= *([\d\.\+\-]+) *{}'.format(unit),
    r_unit_cell_volume=r'^ *unit-cell volume *= *([\d\.\+\-]+) *{}'.format(
                       unit),
    r_cell_side=r' *a\((\d+)\) *= *\( *(-?[\d.]+) *(-?[\d.]+) *'
                '(-?[\d.]+) *\)',
    r_natoms=r'^ *number of atoms/cell *= *(\d+)',
    r_nspecies=r'^ *number of atomic types *= *(\d+)',
    r_dspecies=r'^ +(\w{1,2}) +(-?[\d.]+) +(-?[\d.]+)',
    r_nelectrons=r'^ *number of electrons *= *([\d\.\+\-]+)',
    r_nksstates=r'^ *number of Kohn-Sham states *= *(\d+)',
    r_cutoff=r'^ *kinetic-energy cutoff *= *([\d\.\+\-]+) *{}'.format(unit),
    r_charge_cutoff=r'^ *charge density cutoff *= *([\d\.\+\-]+) *{}'.format(
                    unit),
    r_treshold=r'^ *convergence threshold *= *(\d+.\d+E?-?\d*)',
    r_mixing=r'^ *mixing beta *= *([\d\.\+\-]+)',
    r_apos=r'^ +(\d+) +({})[^=]+= \( +([\d\+\-\.]+) +([\d\+\-\.]+) +'
           '([\d\+\-\.]+) +\)'.format(atoms_name))

# data of scf
# TODO, the pressure calculation is not always carried out so it should be
# putted in another dictionary
scf_data_out = dict(
    r_total_energy=r'^![\w =]+([\d\.\+\- ]+){}'.format(unit),
    r_E_hartree=r'^ +hartree \w+ += +([\d\.\+\-]+) +{}'.format(unit),
    r_E_onelectron=r'^ +one-electron \w+ += +([\d\.\+\-]+) +{}'.format(unit),
    r_E_xc=r'^ +xc \w+ += +([\d\.\+\-]+) +{}'.format(unit),
    r_E_ewald=r'^ +ewald \w+ += +([\d\.\+\-]+) +{}'.format(unit),
    r_E_paw=r'^ +one-center paw \w+\. += +([\d\.\+\-]+) +{}'.format(unit),
    r_force=r'atom +([\d\.\+\-]+) +type +([\d\.\+\-]+)'
            ' +force = +([\d\.\+\-]+) +([\d\.\+\-]+) +([\d\.\+\-]+)',
    r_stress_units=r' +total +stress +{}'.format(unit),
    r_pressure=r'\(kbar\) +P= +([\d\.\+\-]+)',
    r_stress_and_kbar_tensor=r'^ {2,3}([\d\.\+\-]+) +([\d\.\+\-]+) +'
                             r'([\d\.\+\-]+) {9,10}([\d\.\+\-]+) +'
                             r'([\d\.\+\-]+) +([\d\.\+\-]+)$')

# information of BFGS
bfgs_set = dict(
    r_bfgs_converged=r' +bfgs converged in +(\d+) +scf cycles and +(\d+) +'
                     'bfgs steps',
    t_bfgs_not_converged=r'^ +The maximum number of steps has been reached.',
    r_criteria=r'^ +\(criteria: +(energy) < ([\d\.\+\-E]+),'
               ' +(force) < ([\d\.\+\-E]+),'
               ' +(cell) < ([\d\.\+\-E]+)\)',
    r_end=r'^ +End of BFGS Geometry Optimization',
    r_start=r'^ +BFGS Geometry Optimization',
    r_nstep=r'^ *nstep *= *(\d+)',
    r_recalculation=r'The G-vectors are recalculated for the final unit cell')

# data of bfgs:
bfgs_data_out = dict(
    r_unit_cell_volume=r'^ *new unit-cell volume *= *([\d\.\+\-]+) *{}'.format(
                       unit),
    r_cell_side_units=r'CELL_PARAMETERS \(([\w ]+)= ([\d\.\+\-]+)\)',
    r_cell_side=r'^ {2,3}([\d\+\-\.]+) +([\d\+\-\.]+) +([\d\+\-\.]+)$',
    r_apos_units=r'ATOMIC_POSITIONS \((.+)\)',
    r_apos=r'^({}) +([\d\+\-\.]+) +([\d\+\-\.]+) +([\d\+\-\.]+)$'
                .format(atoms_name))

# closing string:
r_close = r'JOB DONE'

def complete_scf(text):
    simulation = {}
    for x in scf_set:
        data = re.findall(scf_set[x], text, re.MULTILINE)
        if len(data) == 1:
            simulation[x[2:]] = data.pop()
        else:
            simulation[x[2:]] = data
    for x in scf_data_out:
        data = re.findall(scf_data_out[x], text, re.MULTILINE)
        if len(data) == 1:
            simulation[x[2:]] = data.pop()
        else:
            simulation[x[2:]] = data

    # scf fix
    # print('ATTENTION: POSITONS ARE ON Crystallographic axes')

    # the normalizations MUST BE DONE in the same order as the data are
    # collected becouse the first that fails will rise an error and all the
    # others wont be applied.

    # normalization of atom description
    simulation['atom_description'] = []
    for x, y in zip(simulation.pop('dspecies'),
                    simulation.pop('pseudopotential')):
        simulation['atom_description'].append((x[0], x[1], x[2], y))

    conversion = {i + 1: x[0] for i, x in
                  enumerate(simulation['atom_description'])}

    # normalization of cell description
    cell_side = simulation.pop('cell_side')
    simulation['cell_side'] = [x[1:] for x in cell_side]

    # normalization of force and positions
    simulation['atom'] = []
    nat = int(simulation['natoms'])
    pos = simulation.pop('apos')[nat:]
    force = simulation.pop('force')[:nat]
    if len(force) < nat:
        # try to save as much data as possible
        for x in pos:
            simulation['atom'].append((x[0], x[1], x[2:]))
        raise CorruptedData('not enought forces, damage data', simulation)
    for x, y in zip(pos, force):
        if x[0] == y[0] and x[1] == conversion[int(y[1])]:
            simulation['atom'].append((x[0], x[1], x[2:], y[2:]))
        else:
            raise ValueError('Error in conversion step')

    # normalization of stress and pressure information
    simulation['stress_tesnsor'] = []
    simulation['pressure_tesnsor'] = []
    for x in simulation.pop('stress_and_kbar_tensor'):
        simulation['stress_tesnsor'].append(x[:3])
        simulation['pressure_tesnsor'].append(x[3:])
    return simulation

def bfgs_complete(text):
    simulation = {}
    for x in scf_data_out:
        data = re.findall(scf_data_out[x], text, re.MULTILINE)
        if len(data) == 1:
            simulation[x[2:]] = data.pop()
        else:
            simulation[x[2:]] = data
    for x in bfgs_data_out:
        data = re.findall(bfgs_data_out[x], text, re.MULTILINE)
        if len(data) == 1:
            simulation[x[2:]] = data.pop()
        else:
            simulation[x[2:]] = data

    # normalization of force and positions
    simulation['atom'] = []
    pos = simulation.pop('apos')
    conversion = []
    for x in pos:
        if x[0] not in conversion:
            conversion.append(x[0])
    conversion = {i + 1: j for i, j in enumerate(conversion)}
    nat = len(pos)
    force = simulation.pop('force')[:nat]
    if len(force) < nat:
        if len(force) < nat:
            for x in pos:
                simulation['atom'].append((x[0], x[1:]))
        raise CorruptedData('not enought forces, damage data', simulation)
    for x, y in zip(pos, force):
        if x[0] == conversion[int(y[1])]:
            simulation['atom'].append((y[0], x[0], x[1:], y[2:]))
        else:
            print(text)
            print(conversion)
            print(x[0])
            print(y[1])
            raise ValueError('Error in conversion step')
    return simulation
<|endoftext|>


<commit_before>import math

stack = []

digits = list("0123456789")
set_var = list("ABCDEF")
get_var = list("abcdef")

# Retrieve data from the stack
# When the stack is empty, ask for input

def pop():
    if len(stack):
        return stack.pop()
    else:
        return float(input("Value required: "))

# Define basic maths operations

def add():
    if len(stack):
        return pop() + pop()
    else:
        return pop() * 2

def sub():
    if len(stack):
        return -1 * (pop() - pop())
    else:
        return -1 * pop()

def mult():
    if len(stack):
        return pop() * pop()
    else:
        return math.pow(pop(), 2)

def div():
    if len(stack):
        return 1 / (pop() / pop())
    else:
        return 1 / pop()

def exp():
    if len(stack):
        return math.pow(pop(), pop())
    else:
        a = pop()
        return math.pow(a, a)

def mod():
    a,b = pop(),pop()
    return b % a

# Parse the input into a list of instructions

def parse(code):

    pointer = 0

    c = lambda: code[pointer]

    parsed = []

    while pointer < len(code):

        if c() in digits:

            number = ""

            while pointer < len(code) and (c() in digits or (c() == '.' and '.' not in number)):
                number += c()
                pointer += 1
            pointer -= 1
            parsed.append(("pushnum", float(number)))

        elif c() in operations:
            parsed.append(("operation", operations[c()]))

        elif c() in set_var:
            parsed.append(("setvar", c()))

        elif c() in get_var:
            parsed.append(("getvar", c()))

        pointer += 1

    return parsed

# a dict of operations

operations = {'+':add,
              '-':sub,
              '*':mult,
              '/':div,
              '^':exp,
              '%':mod}

# vars the user has access to

scope = {'a':0,
         'b':0,
         'c':0,
         'd':0,
         'e':0,
         'f':0}

code = input("Input expression: ")

instructions = parse(code)

print(instructions)

for i in instructions:

    if i[0] == "pushnum":
        stack.append(i[1])

    elif i[0] == "operation":
        stack.append(i[1]())

    elif i[0] == "setvar":
        scope[i[1].lower()] = pop()

    elif i[0] == "getvar":
        stack.append(scope[i[1]])

print(stack[-1])
<commit_msg>Adjustements to implicit input<commit_after>import math

stack = []

digits = list("0123456789")
set_var = list("ABCDEF")
get_var = list("abcdef")

# Retrieve data from the stack
# When the stack is empty, ask for input

def pop():
    if len(stack):
        return stack.pop()
    else:
        return float(input("Value required: "))

# Define basic maths operations

def add():
    if len(stack) > 1:
        return pop() + pop()
    else:
        return pop() * 2

def sub():
    if len(stack) > 1:
        return -1 * (pop() - pop())
    else:
        return -1 * pop()

def mult():
    if len(stack) > 1:
        return pop() * pop()
    else:
        return math.pow(pop(), 2)

def div():
    if len(stack) > 1:
        return 1 / (pop() / pop())
    else:
        return 1 / pop()

def exp():
    if len(stack) > 1:
        a = pop()
        b = pop()
        return math.pow(b, a)
    else:
        a = pop()
        return math.pow(a, a)

def mod():
    a,b = pop(),pop()
    return b % a

# Parse the input into a list of instructions

def parse(code):

    pointer = 0

    c = lambda: code[pointer]

    parsed = []

    while pointer < len(code):

        if c() in digits:

            number = ""

            while pointer < len(code) and (c() in digits or (c() == '.' and '.' not in number)):
                number += c()
                pointer += 1
            pointer -= 1
            parsed.append(("pushnum", float(number)))

        elif c() in operations:
            parsed.append(("operation", operations[c()]))

        elif c() in set_var:
            parsed.append(("setvar", c().lower()))

        elif c() in get_var:
            parsed.append(("getvar", c()))

        pointer += 1

    return parsed

# a dict of operations

operations = {'+':add,
              '-':sub,
              '*':mult,
              '/':div,
              '^':exp,
              '%':mod}

# vars the user has access to

scope = {'a':0,
         'b':0,
         'c':0,
         'd':0,
         'e':0,
         'f':0}

code = input("Input expression: ")

instructions = parse(code)

print(instructions)

for i in instructions:

    if i[0] == "pushnum":
        stack.append(i[1])

    elif i[0] == "operation":
        stack.append(i[1]())

    elif i[0] == "setvar":
        scope[i[1]] = pop()

    elif i[0] == "getvar":
        stack.append(scope[i[1]])

print(stack[-1])
<|endoftext|>


<commit_before>from itertools import zip_longest
import math
import string

_EOF = '\0'

def _search_text_state(lex):
    """Lex state for handling text.

    Args:
        lex: The _Lexer object.
    Returns:
        The next lex state.
    """
    lex.acceptRun(string.whitespace)
    lex.ignore()

    n = lex.next()
    if n == _EOF:
        return None
    if n == '"':
        return _search_quote_state

    while True:
        lex.acceptRunIgnoring(string.whitespace + '\\:')
        if lex.peek() == ':':
            # TODO(pope): Remove this out of here and make this the job of the
            # parser.
            if lex.curstr().lower() in ('file', 'case'):
                lex.emit('flag')
                lex.next()  # advance the ':'.
                lex.ignore()  # drop it.
                return _search_text_state
            else:
                lex.next()
        elif lex.accept('\\'):
            lex.next()  # advance what we're escaping.
        else:
            break
    lex.emit('text')

    return _search_text_state

def _search_quote_state(lex):
    """Lex state for handling double quoted text.

    Args:
        lex: The _Lexer object.
    Returns:
        The next lex state.
    """
    # TODO(pope): Handle the error case when we try to leave without seeing a
    # closing quote.
    while True:
        lex.acceptRunIgnoring('\\"')
        c = lex.next()
        if c == '\\':
            lex.next()  # Advance over whatever is being escaped.
        else:
            break
    lex.emit('quote')
    return _search_text_state

class _OutputToken(object):
    FileName, LineNumber, Line = range(0, 3)

def _output_start_state(lex):
    """Lex start state for tokenizing search output.

    Args:
        lex: The _Lexer object.
    Returns:
        The next lex state function.
    """
    if lex.peek() == _EOF:
        return None
    return _output_filename_state

def _output_filename_state(lex):
    """Lex state for tokenizing the filename.

    Args:
        lex: The _Lexer object.
    Returns:
        The next lex state function.
    """
    lex.acceptRunIgnoring(':')
    if not lex.hasstr():
        lex.error('Expected to read a filename.')
    lex.emit(_OutputToken.FileName)
    if lex.next() != ':':
        lex.error('Expected to see a ":" after reading the filename.')
    lex.ignore()
    return _output_linenumber_state(lex)

def _output_linenumber_state(lex):
    """Lex state for tokenizing the line number.

    Args:
        lex: The _Lexer object.
    Returns:
        The next lex state function.
    """
    lex.acceptRun(string.digits)
    if not lex.hasstr():
        lex.error('Expected line number as digits.')
    lex.emit(_OutputToken.LineNumber)
    if lex.next() != ':':
        lex.error('Expected to see a ":" after reading the line number.')
    lex.ignore()
    return _output_line_state

def _output_line_state(lex):
    """Lex state for tokenizing the found line.

    Args:
        lex: The _Lexer object.
    Returns:
        The next lex state function.
    """
    lex.acceptRunIgnoring('\n')
    if not lex.hasstr():
        lex.error('Expected the found line.')
    lex.emit(_OutputToken.Line)
    c = lex.next()
    if not (c == '\n' or c == _EOF):
        lex.error('Expected a newline after reading the found line.')
    lex.ignore()
    return _output_start_state

class _LexerException(Exception):
    """Exception class when there is a lexing error."""
    pass

class _Lexer(object):
    """The simple lexer for tokenizing strings."""

    def __init__(self, text, start_state):
        self._text = text
        self._start_state = start_state

    def run(self):
        """Runs the lexer returning a list of tokens.

        Returns:
            A list of token tuples, where the first entry is the token type and
            the second value is the text value.
        """
        self._start = 0
        self._pos = 0
        self._width = 0
        self._tokens = []
        state = self._start_state
        while state is not None:
            state = state(self)
        return self._tokens

    def error(self, msg):
        """Emits an error token with the message.

        Args:
            msg: The error message to emit.
        Raises:
            _LexerException
        """
        # TODO(pope): Keep track of the line number and emit that too.
        raise _LexerException(msg)

    def curstr(self):
        """The string as it's been parsed before being emitted."""
        return self._text[self._start:self._pos]

    def hasstr(self):
        """Checks to see if a string has been parsed before being emitted."""
        return self._pos > self._start

    def emit(self, tokType):
        """Emit appends the current string to the list of tokens.

        Args:
            tokType: The token type describing the current string.
        """
        self._tokens.append((tokType, self.curstr()))
        self._start = self._pos

    def ignore(self):
        """Ignores the current string."""
        self._start = self._pos

    def next(self):
        """Moves the cursor to the next character in the string.

        Returns:
            The character that was just consumed or _EOF if there's nothing
            left.
        """
        if self._pos >= len(self._text):
            self._width = 0
            return _EOF
        c = self._text[self._pos]
        self._width = 1
        self._pos += self._width
        return c

    def backup(self):
        """Undo one next() call."""
        self._pos -= self._width

    def peek(self):
        """Returns the next character in the string."""
        c = self.next()
        self.backup()
        return c

    def accept(self, text):
        """Advances the cursor to the next character if within the string.

        Args:
            text: A string of expected characters to see next.
        Returns:
            True if the next character was accepted, or False if it was not.
        """
        if self.next() in text:
            return True
        self.backup()
        return False

    def acceptRun(self, text):
        """Advances the cursor forward while the next character is in the string.

        Args:
            text: A string of expected characters to see next.
        """
        while self.next() in text:
            pass
        self.backup()

    def acceptRunIgnoring(self, text):
        """Advances the cursor forward while next is not within the string.

        Args:
            text: A string of characters to ignore while calling next.
        """
        while True:
            c = self.next()
            if c == _EOF or c in text:
                break
        self.backup()

class Search(object):
    """A search object.

    Used to organize phrases to search and for while files.

    Attributes:
        query: A list of search terms to hunt down.
        file: A string pattern for files to limit the search to.
        case: A boolean value for if the search is case sensitive or not.
    """

    def __init__(self, query=None, file=None, case=True):
        if query:
            self.query = query
        else:
            self.query = []
        self.file = file
        self.case = case

    def args(self):
        """Prints out the command arguments for csearch.

        Returns:
            A list of command line arguments from the query.
        Raises:
            AttributeError: If there is no query for the command.
        """
        if not self.query:
            raise AttributeError('No query to run')
        args = []
        if self.file:
            args.extend(['-f', self.file])
        if not self.case:
            args.append('-i')
        args.append(self.query_re())
        return args

    def query_re(self):
        if len(self.query) == 1:
            return self.query[0]
        else:
            return '({0})'.format('|'.join(self.query))

    def __eq__(self, other):
        return (isinstance(other, self.__class__) and
                self.query == other.query and
                self.file == other.file and
                self.case == other.case)

    def __ne__(self, other):
        return not self.__eq__(other)

    def __hash__(self):
        # Not really needed, so a very dumb implementation to just be correct.
        return 42

    def __repr__(self):
        return '{0}(query={1}; file={2}; case={3})'.format(self.__class__,
                                                           self.query,
                                                           self.file,
                                                           self.case)

def parse_query(text):
    """Parse a search string into a Search object.

    Search example queries include:

        "A literal string"
        ^.*someRegex.*$
        MyClass file:.*c$
        myclass case:no

    Args:
        text: The search query string to parse.
    Returns:
        A Search object representing the parsed string.
    Raises:
        Exception: If there was a problem parsing the search query.
    """
    res = Search()
    lex = _Lexer(text, _search_text_state)
    tokens = iter(lex.run())
    try:
        while True:
            (tokType, tokValue) = next(tokens)
            if tokType in ('text', 'quote'):
                res.query.append(tokValue)
            elif tokType == 'flag':
                flag = tokValue.lower()
                (ignore_type, value) = next(tokens)
                if flag == 'case':
                    if value.lower() == 'no':
                        res.case = False
                    else:
                        res.case = True
                elif flag == 'file':
                    res.file = value
                else:
                    raise Exception('Unsupported flag value: {0}'.format(flag))
            else:
                raise Exception('Unsupported token type: {0}'.format(tokType))
    except StopIteration:
        pass
    return res

class FileResults(object):
    """The parsed search output for a file.

    Used to organize all of the matched lines for a particular file.

    Attributes:
        filename: The location of the file.
        matches: A list of tuple pairs where the first item is the line number,
            and the second value is the matched line.
    """

    def __init__(self, filename, matches):
        assert matches
        assert filename
        self.filename = filename
        self.matches = matches

    def __eq__(self, other):
        return (isinstance(other, self.__class__) and
                self.filename == other.filename and
                self.matches == other.matches)

    def __ne__(self, other):
        return not self.__eq__(other)

    def __hash__(self):
        # Not really needed, so a very dumb implementation to just be correct.
        return 42

    def __repr__(self):
        msg = '{0}(filename={1}; matches={2})'
        return msg.format(self.__class__, self.filename, self.matches)

    def __str__(self):
        line_tmpl = '{0: >5}: {1}'
        res_matches = []
        matches = iter(self.matches)
        (linenum, line) = next(matches)
        res_matches.append(line_tmpl.format(linenum, line))
        prev_linenum = linenum
        for (linenum, line) in matches:
            if prev_linenum + 1 != linenum:
                num_digits = int(math.log10(prev_linenum)) + 1
                res_matches.append('{0: >5}'.format('.' * num_digits))
            res_matches.append(line_tmpl.format(linenum, line))
            prev_linenum = linenum
        return '{0}:\n{1}'.format(self.filename, '\n'.join(res_matches))

def parse_search_output(text):
    """Parse the output text from a search command.

    The format of the text should be:

        #File_Name:Line_Number:Line_Contents
        a.txt:1:Too many cooks
        a.txt:2:TOO MANY cooks
        b.txt:34:How to cook

    Args:
        text: The search output string.
    Returns:
        A list of FileResults objects.
    Raises:
        Exception: If there was a problem parsing the output.
    """
    res = []
    lex = _Lexer(text, _output_start_state)
    tokens = lex.run()
    if not tokens:
        return res
    # See grouper recipe in itertools docs. Going to group this list into 3s --
    # filename, line number, line.
    tokens = iter(tokens)
    token_groups = zip_longest(tokens, tokens, tokens)

    (filename, linenum, line) = _line_parts(next(token_groups))
    cur_filename = filename
    cur_matches = [(linenum, line)]
    for (filename, linenum, line) in map(_line_parts, token_groups):
        if cur_filename != filename:
            res.append(FileResults(cur_filename, cur_matches))
            cur_filename = filename
            cur_matches = [(linenum, line)]
        else:
            cur_matches.append((linenum, line))
    res.append(FileResults(cur_filename, cur_matches))
    return res

def _line_parts(token_group):
    """Pulls out the data from the token group.

    Args:
        token_group:  The token group of filename, linenum, and line.
    Returns:
        A tuple for the filename, the line number, and the matched line.
    Raises:
        ValueError: If the line number could not be parsed as an int. This
            should not happen.
    """
    (file_token, linenum_token, line_token) = token_group
    # TODO(pope): Add assertions that the tokens are what we expect.
    (unused_tok, filename) = file_token
    (unused_tok, linenum) = linenum_token
    (unused_tok, line) = line_token
    return (filename, int(linenum), line)
<commit_msg>PEP8 compliance<commit_after>from itertools import zip_longest
import math
import string

_EOF = '\0'

def _search_text_state(lex):
    """Lex state for handling text.

    Args:
        lex: The _Lexer object.
    Returns:
        The next lex state.
    """
    lex.acceptRun(string.whitespace)
    lex.ignore()

    n = lex.next()
    if n == _EOF:
        return None
    if n == '"':
        return _search_quote_state

    while True:
        lex.acceptRunIgnoring(string.whitespace + '\\:')
        if lex.peek() == ':':
            # TODO(pope): Remove this out of here and make this the job of the
            # parser.
            if lex.curstr().lower() in ('file', 'case'):
                lex.emit('flag')
                lex.next()  # advance the ':'.
                lex.ignore()  # drop it.
                return _search_text_state
            else:
                lex.next()
        elif lex.accept('\\'):
            lex.next()  # advance what we're escaping.
        else:
            break
    lex.emit('text')

    return _search_text_state

def _search_quote_state(lex):
    """Lex state for handling double quoted text.

    Args:
        lex: The _Lexer object.
    Returns:
        The next lex state.
    """
    # TODO(pope): Handle the error case when we try to leave without seeing a
    # closing quote.
    while True:
        lex.acceptRunIgnoring('\\"')
        c = lex.next()
        if c == '\\':
            lex.next()  # Advance over whatever is being escaped.
        else:
            break
    lex.emit('quote')
    return _search_text_state

class _OutputToken(object):
    FileName, LineNumber, Line = range(0, 3)

def _output_start_state(lex):
    """Lex start state for tokenizing search output.

    Args:
        lex: The _Lexer object.
    Returns:
        The next lex state function.
    """
    if lex.peek() == _EOF:
        return None
    return _output_filename_state

def _output_filename_state(lex):
    """Lex state for tokenizing the filename.

    Args:
        lex: The _Lexer object.
    Returns:
        The next lex state function.
    """
    lex.acceptRunIgnoring(':')
    if not lex.hasstr():
        lex.error('Expected to read a filename.')
    lex.emit(_OutputToken.FileName)
    if lex.next() != ':':
        lex.error('Expected to see a ":" after reading the filename.')
    lex.ignore()
    return _output_linenumber_state(lex)

def _output_linenumber_state(lex):
    """Lex state for tokenizing the line number.

    Args:
        lex: The _Lexer object.
    Returns:
        The next lex state function.
    """
    lex.acceptRun(string.digits)
    if not lex.hasstr():
        lex.error('Expected line number as digits.')
    lex.emit(_OutputToken.LineNumber)
    if lex.next() != ':':
        lex.error('Expected to see a ":" after reading the line number.')
    lex.ignore()
    return _output_line_state

def _output_line_state(lex):
    """Lex state for tokenizing the found line.

    Args:
        lex: The _Lexer object.
    Returns:
        The next lex state function.
    """
    lex.acceptRunIgnoring('\n')
    if not lex.hasstr():
        lex.error('Expected the found line.')
    lex.emit(_OutputToken.Line)
    c = lex.next()
    if not (c == '\n' or c == _EOF):
        lex.error('Expected a newline after reading the found line.')
    lex.ignore()
    return _output_start_state

class _LexerException(Exception):
    """Exception class when there is a lexing error."""
    pass

class _Lexer(object):
    """The simple lexer for tokenizing strings."""

    def __init__(self, text, start_state):
        self._text = text
        self._start_state = start_state

    def run(self):
        """Runs the lexer returning a list of tokens.

        Returns:
            A list of token tuples, where the first entry is the token type and
            the second value is the text value.
        """
        self._start = 0
        self._pos = 0
        self._width = 0
        self._tokens = []
        state = self._start_state
        while state is not None:
            state = state(self)
        return self._tokens

    def error(self, msg):
        """Emits an error token with the message.

        Args:
            msg: The error message to emit.
        Raises:
            _LexerException
        """
        # TODO(pope): Keep track of the line number and emit that too.
        raise _LexerException(msg)

    def curstr(self):
        """The string as it's been parsed before being emitted."""
        return self._text[self._start:self._pos]

    def hasstr(self):
        """Checks to see if a string has been parsed before being emitted."""
        return self._pos > self._start

    def emit(self, tokType):
        """Emit appends the current string to the list of tokens.

        Args:
            tokType: The token type describing the current string.
        """
        self._tokens.append((tokType, self.curstr()))
        self._start = self._pos

    def ignore(self):
        """Ignores the current string."""
        self._start = self._pos

    def next(self):
        """Moves the cursor to the next character in the string.

        Returns:
            The character that was just consumed or _EOF if there's nothing
            left.
        """
        if self._pos >= len(self._text):
            self._width = 0
            return _EOF
        c = self._text[self._pos]
        self._width = 1
        self._pos += self._width
        return c

    def backup(self):
        """Undo one next() call."""
        self._pos -= self._width

    def peek(self):
        """Returns the next character in the string."""
        c = self.next()
        self.backup()
        return c

    def accept(self, text):
        """Advances the cursor to the next character if within the string.

        Args:
            text: A string of expected characters to see next.
        Returns:
            True if the next character was accepted, or False if it was not.
        """
        if self.next() in text:
            return True
        self.backup()
        return False

    def acceptRun(self, text):
        """Advances the cursor forward while the next character is within the
        string.

        Args:
            text: A string of expected characters to see next.
        """
        while self.next() in text:
            pass
        self.backup()

    def acceptRunIgnoring(self, text):
        """Advances the cursor forward while next is not within the string.

        Args:
            text: A string of characters to ignore while calling next.
        """
        while True:
            c = self.next()
            if c == _EOF or c in text:
                break
        self.backup()

class Search(object):
    """A search object.

    Used to organize phrases to search and for while files.

    Attributes:
        query: A list of search terms to hunt down.
        file: A string pattern for files to limit the search to.
        case: A boolean value for if the search is case sensitive or not.
    """

    def __init__(self, query=None, file=None, case=True):
        if query:
            self.query = query
        else:
            self.query = []
        self.file = file
        self.case = case

    def args(self):
        """Prints out the command arguments for csearch.

        Returns:
            A list of command line arguments from the query.
        Raises:
            AttributeError: If there is no query for the command.
        """
        if not self.query:
            raise AttributeError('No query to run')
        args = []
        if self.file:
            args.extend(['-f', self.file])
        if not self.case:
            args.append('-i')
        args.append(self.query_re())
        return args

    def query_re(self):
        if len(self.query) == 1:
            return self.query[0]
        else:
            return '({0})'.format('|'.join(self.query))

    def __eq__(self, other):
        return (isinstance(other, self.__class__) and
                self.query == other.query and
                self.file == other.file and
                self.case == other.case)

    def __ne__(self, other):
        return not self.__eq__(other)

    def __hash__(self):
        # Not really needed, so a very dumb implementation to just be correct.
        return 42

    def __repr__(self):
        return '{0}(query={1}; file={2}; case={3})'.format(self.__class__,
                                                           self.query,
                                                           self.file,
                                                           self.case)

def parse_query(text):
    """Parse a search string into a Search object.

    Search example queries include:

        "A literal string"
        ^.*someRegex.*$
        MyClass file:.*c$
        myclass case:no

    Args:
        text: The search query string to parse.
    Returns:
        A Search object representing the parsed string.
    Raises:
        Exception: If there was a problem parsing the search query.
    """
    res = Search()
    lex = _Lexer(text, _search_text_state)
    tokens = iter(lex.run())
    try:
        while True:
            (tokType, tokValue) = next(tokens)
            if tokType in ('text', 'quote'):
                res.query.append(tokValue)
            elif tokType == 'flag':
                flag = tokValue.lower()
                (ignore_type, value) = next(tokens)
                if flag == 'case':
                    if value.lower() == 'no':
                        res.case = False
                    else:
                        res.case = True
                elif flag == 'file':
                    res.file = value
                else:
                    raise Exception('Unsupported flag value: {0}'.format(flag))
            else:
                raise Exception('Unsupported token type: {0}'.format(tokType))
    except StopIteration:
        pass
    return res

class FileResults(object):
    """The parsed search output for a file.

    Used to organize all of the matched lines for a particular file.

    Attributes:
        filename: The location of the file.
        matches: A list of tuple pairs where the first item is the line number,
            and the second value is the matched line.
    """

    def __init__(self, filename, matches):
        assert matches
        assert filename
        self.filename = filename
        self.matches = matches

    def __eq__(self, other):
        return (isinstance(other, self.__class__) and
                self.filename == other.filename and
                self.matches == other.matches)

    def __ne__(self, other):
        return not self.__eq__(other)

    def __hash__(self):
        # Not really needed, so a very dumb implementation to just be correct.
        return 42

    def __repr__(self):
        msg = '{0}(filename={1}; matches={2})'
        return msg.format(self.__class__, self.filename, self.matches)

    def __str__(self):
        line_tmpl = '{0: >5}: {1}'
        res_matches = []
        matches = iter(self.matches)
        (linenum, line) = next(matches)
        res_matches.append(line_tmpl.format(linenum, line))
        prev_linenum = linenum
        for (linenum, line) in matches:
            if prev_linenum + 1 != linenum:
                num_digits = int(math.log10(prev_linenum)) + 1
                res_matches.append('{0: >5}'.format('.' * num_digits))
            res_matches.append(line_tmpl.format(linenum, line))
            prev_linenum = linenum
        return '{0}:\n{1}'.format(self.filename, '\n'.join(res_matches))

def parse_search_output(text):
    """Parse the output text from a search command.

    The format of the text should be:

        #File_Name:Line_Number:Line_Contents
        a.txt:1:Too many cooks
        a.txt:2:TOO MANY cooks
        b.txt:34:How to cook

    Args:
        text: The search output string.
    Returns:
        A list of FileResults objects.
    Raises:
        Exception: If there was a problem parsing the output.
    """
    res = []
    lex = _Lexer(text, _output_start_state)
    tokens = lex.run()
    if not tokens:
        return res
    # See grouper recipe in itertools docs. Going to group this list into 3s --
    # filename, line number, line.
    tokens = iter(tokens)
    token_groups = zip_longest(tokens, tokens, tokens)

    (filename, linenum, line) = _line_parts(next(token_groups))
    cur_filename = filename
    cur_matches = [(linenum, line)]
    for (filename, linenum, line) in map(_line_parts, token_groups):
        if cur_filename != filename:
            res.append(FileResults(cur_filename, cur_matches))
            cur_filename = filename
            cur_matches = [(linenum, line)]
        else:
            cur_matches.append((linenum, line))
    res.append(FileResults(cur_filename, cur_matches))
    return res

def _line_parts(token_group):
    """Pulls out the data from the token group.

    Args:
        token_group:  The token group of filename, linenum, and line.
    Returns:
        A tuple for the filename, the line number, and the matched line.
    Raises:
        ValueError: If the line number could not be parsed as an int. This
            should not happen.
    """
    (file_token, linenum_token, line_token) = token_group
    # TODO(pope): Add assertions that the tokens are what we expect.
    (unused_tok, filename) = file_token
    (unused_tok, linenum) = linenum_token
    (unused_tok, line) = line_token
    return (filename, int(linenum), line)
<|endoftext|>


<commit_before>import sys
import optparse
import os

from result_file import ResultFile
from queries_summary_file import QueriesSummaryFile
from log_file import LogFile
from document import Document
from query import Query
from topic import Topic
from user import User
from condition import Condition
from action import Action

def error_exit():
  sys.stderr.write( 'ERROR: missing argument(s).\n' )
  CLI.parser.print_help()
  sys.exit(1)

class CLI: pass
CLI.parser = optparse.OptionParser()
CLI.parser.add_option( "-q", "--queries", dest = "queries", help = "Queries csv file", metavar="FILE")
CLI.parser.add_option( "-l", "--logs", dest = "logs", help = "Path to log files containing directory", metavar="DIR")
CLI.parser.add_option( "-r", "--results", dest = "results", help = "Path to result files containing directory", metavar="DIR")
CLI.parsedArgs = CLI.parser.parse_args()

def get_cli_options():
  (options, args) = CLI.parsedArgs
  return options

def parse_result_files():
  result_files = []
  opts = get_cli_options()
  results_dir = opts.results
  if results_dir is None: error_exit()
  for dirname, dirnames, filenames in os.walk( results_dir ):
    for filename in filenames:
      if not filename.endswith('.serp'): continue
      result_files.append( ResultFile( os.path.join( dirname, filename ) ) )
  return result_files

def parse_log_files():
  log_files = []
  opts = get_cli_options()
  log_dir = opts.logs
  if log_dir is None: error_exit()
  for dirname, dirnames, filenames in os.walk( log_dir ):
    for filename in filenames:
      if not filename.endswith('.log'): continue
      log_files.append( LogFile( os.path.join( dirname, filename ) ) )
  return log_files

def get_queries_file_name():
  opts = get_cli_options()
  if opts.queries is None: error_exit()
  return opts.queries

result_files = parse_result_files()
log_files = parse_log_files()
queries_file = QueriesSummaryFile( get_queries_file_name() )

print "Queries: %i" % len(Query.store)
print "Topics: %i: %s" % (len(Topic.store), Topic.store.keys())
print "Users: %i" % len(User.store)
print "Documents: %i" % len(Document.store)
print "Conditions: %i: %s" % (len(Condition.store), Condition.store.keys())
print "Action types: %i: %s" % (len(Action.type_dict.keys()), Action.type_dict.keys())

print "Mark-doc-as-viewed actions: %i" % (
  len(Action.by_type('DOC_MARKED_VIEWED')))
print "Mark-doc-as-viewed actions with a moderately relevant doc: %i" % (
  len(Action.filter_by_type('DOC_MARKED_VIEWED', Action.moderately_relevant_filter)))
print "Mark-doc-as-viewed actions with a highly relevant doc: %i" % (
  len(Action.filter_by_type('DOC_MARKED_VIEWED', Action.highly_relevant_filter)))

print "Mark-doc-as-relevant actions: %i" % (
  len(Action.by_type('DOC_MARKED_RELEVANT')))
print "Mark-doc-as-relevant actions with a moderately relevant doc: %i" % (
  len(Action.filter_by_type('DOC_MARKED_RELEVANT', Action.moderately_relevant_filter)))
print "Mark-doc-as-relevant actions with a highly relevant doc: %i" % (
  len(Action.filter_by_type('DOC_MARKED_RELEVANT', Action.highly_relevant_filter)))

print "Mark-doc-as-viewed actions with no delays: %i" % (
  len(Action.filter_by_type('DOC_MARKED_VIEWED', Action.no_delays_filter)))
print "Mark-doc-as-viewed actions with no delays and moderately relevant doc: %i" % (
  len(Action.filter_by_type('DOC_MARKED_VIEWED', Action.combine_filters( Action.no_delays_filter, Action.moderately_relevant_filter))))
print "Mark-doc-as-viewed actions with no delays and highly relevant doc: %i" % (
  len(Action.filter_by_type('DOC_MARKED_VIEWED', Action.combine_filters( Action.no_delays_filter, Action.highly_relevant_filter))))

print "Mark-doc-as-relevant actions with no delays: %i" % (
  len(Action.filter_by_type('DOC_MARKED_RELEVANT', Action.no_delays_filter)))
print "Mark-doc-as-relevant actions with no delays and moderately relevant doc: %i" % (
  len(Action.filter_by_type('DOC_MARKED_RELEVANT', Action.combine_filters( Action.no_delays_filter, Action.moderately_relevant_filter))))
print "Mark-doc-as-relevant actions with no delays and highly relevant doc: %i" % (
  len(Action.filter_by_type('DOC_MARKED_RELEVANT', Action.combine_filters( Action.no_delays_filter, Action.highly_relevant_filter))))

print "Mark-doc-as-viewed actions with query delays: %i" % (
  len(Action.filter_by_type('DOC_MARKED_VIEWED', Action.query_delay_filter)))
print "Mark-doc-as-viewed actions with query delays and moderately relevant doc: %i" % (
  len(Action.filter_by_type('DOC_MARKED_VIEWED', Action.combine_filters( Action.query_delay_filter, Action.moderately_relevant_filter))))
print "Mark-doc-as-viewed actions with query delays and highly relevant doc: %i" % (
  len(Action.filter_by_type('DOC_MARKED_VIEWED', Action.combine_filters( Action.query_delay_filter, Action.highly_relevant_filter))))

print "Mark-doc-as-relevant actions with query delays: %i" % (
  len(Action.filter_by_type('DOC_MARKED_RELEVANT', Action.query_delay_filter)))
print "Mark-doc-as-relevant actions with query delays and moderately relevant doc: %i" % (
  len(Action.filter_by_type('DOC_MARKED_RELEVANT', Action.combine_filters( Action.query_delay_filter, Action.moderately_relevant_filter))))
print "Mark-doc-as-relevant actions with query delays and highly relevant doc: %i" % (
  len(Action.filter_by_type('DOC_MARKED_RELEVANT', Action.combine_filters( Action.query_delay_filter, Action.highly_relevant_filter))))

print "Mark-doc-as-viewed actions with document delays: %i" % (
  len(Action.filter_by_type('DOC_MARKED_VIEWED', Action.document_delay_filter)))
print "Mark-doc-as-viewed actions with document delays and moderately relevant doc: %i" % (
  len(Action.filter_by_type('DOC_MARKED_VIEWED', Action.combine_filters( Action.document_delay_filter, Action.moderately_relevant_filter))))
print "Mark-doc-as-viewed actions with document delays and highly relevant doc: %i" % (
  len(Action.filter_by_type('DOC_MARKED_VIEWED', Action.combine_filters( Action.document_delay_filter, Action.highly_relevant_filter))))

print "Mark-doc-as-relevant actions with document delays: %i" % (
  len(Action.filter_by_type('DOC_MARKED_RELEVANT', Action.document_delay_filter)))
print "Mark-doc-as-relevant actions with document delays and moderately relevant doc: %i" % (
  len(Action.filter_by_type('DOC_MARKED_RELEVANT', Action.combine_filters( Action.document_delay_filter, Action.moderately_relevant_filter))))
print "Mark-doc-as-relevant actions with document delays and highly relevant doc: %i" % (
  len(Action.filter_by_type('DOC_MARKED_RELEVANT', Action.combine_filters( Action.document_delay_filter, Action.highly_relevant_filter))))

print "Mark-doc-as-viewed actions with both delays: %i" % (
  len(Action.filter_by_type('DOC_MARKED_VIEWED', Action.combined_delay_filter)))
print "Mark-doc-as-viewed actions with both delays and moderately relevant doc: %i" % (
  len(Action.filter_by_type('DOC_MARKED_VIEWED', Action.combine_filters( Action.combined_delay_filter, Action.moderately_relevant_filter))))
print "Mark-doc-as-viewed actions with both delays and highly relevant doc: %i" % (
  len(Action.filter_by_type('DOC_MARKED_VIEWED', Action.combine_filters( Action.combined_delay_filter, Action.highly_relevant_filter))))

print "Mark-doc-as-relevant actions with both delays: %i" % (
  len(Action.filter_by_type('DOC_MARKED_RELEVANT', Action.combined_delay_filter)))
print "Mark-doc-as-relevant actions with both delays and moderately relevant doc: %i" % (
  len(Action.filter_by_type('DOC_MARKED_RELEVANT', Action.combine_filters( Action.combined_delay_filter, Action.moderately_relevant_filter))))
print "Mark-doc-as-relevant actions with both delays and highly relevant doc: %i" % (
  len(Action.filter_by_type('DOC_MARKED_RELEVANT', Action.combine_filters( Action.combined_delay_filter, Action.highly_relevant_filter))))
<commit_msg>Add process indicator output to main parser script<commit_after>import sys
import optparse
import os

from result_file import ResultFile
from queries_summary_file import QueriesSummaryFile
from log_file import LogFile
from document import Document
from query import Query
from topic import Topic
from user import User
from condition import Condition
from action import Action

def error_exit():
  sys.stderr.write( 'ERROR: missing argument(s).\n' )
  CLI.parser.print_help()
  sys.exit(1)

class CLI: pass
CLI.parser = optparse.OptionParser()
CLI.parser.add_option( "-q", "--queries", dest = "queries", help = "Queries csv file", metavar="FILE")
CLI.parser.add_option( "-l", "--logs", dest = "logs", help = "Path to log files containing directory", metavar="DIR")
CLI.parser.add_option( "-r", "--results", dest = "results", help = "Path to result files containing directory", metavar="DIR")
CLI.parsedArgs = CLI.parser.parse_args()

def get_cli_options():
  (options, args) = CLI.parsedArgs
  return options

def parse_result_files():
  result_files = []
  opts = get_cli_options()
  results_dir = opts.results
  if results_dir is None: error_exit()
  sys.stdout.write( 'Parsing result files' )
  for dirname, dirnames, filenames in os.walk( results_dir ):
    for filename in filenames:
      if not filename.endswith('.serp'): continue
      sys.stdout.write( '.' )
      sys.stdout.flush()
      result_files.append( ResultFile( os.path.join( dirname, filename ) ) )
  sys.stdout.write( '\n' )
  return result_files

def parse_log_files():
  log_files = []
  opts = get_cli_options()
  log_dir = opts.logs
  if log_dir is None: error_exit()
  sys.stdout.write( 'Parsing log files' )
  for dirname, dirnames, filenames in os.walk( log_dir ):
    for filename in filenames:
      if not filename.endswith('.log'): continue
      sys.stdout.write( '.' )
      sys.stdout.flush()
      log_files.append( LogFile( os.path.join( dirname, filename ) ) )
  sys.stdout.write( '\n' )
  return log_files

def get_queries_file_name():
  opts = get_cli_options()
  if opts.queries is None: error_exit()
  return opts.queries

result_files = parse_result_files()
log_files = parse_log_files()
queries_file = QueriesSummaryFile( get_queries_file_name() )

print "Queries: %i" % len(Query.store)
print "Topics: %i: %s" % (len(Topic.store), Topic.store.keys())
print "Users: %i" % len(User.store)
print "Documents: %i" % len(Document.store)
print "Conditions: %i: %s" % (len(Condition.store), Condition.store.keys())
print "Action types: %i: %s" % (len(Action.type_dict.keys()), Action.type_dict.keys())

print "Mark-doc-as-viewed actions: %i" % (
  len(Action.by_type('DOC_MARKED_VIEWED')))
print "Mark-doc-as-viewed actions with a moderately relevant doc: %i" % (
  len(Action.filter_by_type('DOC_MARKED_VIEWED', Action.moderately_relevant_filter)))
print "Mark-doc-as-viewed actions with a highly relevant doc: %i" % (
  len(Action.filter_by_type('DOC_MARKED_VIEWED', Action.highly_relevant_filter)))

print "Mark-doc-as-relevant actions: %i" % (
  len(Action.by_type('DOC_MARKED_RELEVANT')))
print "Mark-doc-as-relevant actions with a moderately relevant doc: %i" % (
  len(Action.filter_by_type('DOC_MARKED_RELEVANT', Action.moderately_relevant_filter)))
print "Mark-doc-as-relevant actions with a highly relevant doc: %i" % (
  len(Action.filter_by_type('DOC_MARKED_RELEVANT', Action.highly_relevant_filter)))

print "Mark-doc-as-viewed actions with no delays: %i" % (
  len(Action.filter_by_type('DOC_MARKED_VIEWED', Action.no_delays_filter)))
print "Mark-doc-as-viewed actions with no delays and moderately relevant doc: %i" % (
  len(Action.filter_by_type('DOC_MARKED_VIEWED', Action.combine_filters( Action.no_delays_filter, Action.moderately_relevant_filter))))
print "Mark-doc-as-viewed actions with no delays and highly relevant doc: %i" % (
  len(Action.filter_by_type('DOC_MARKED_VIEWED', Action.combine_filters( Action.no_delays_filter, Action.highly_relevant_filter))))

print "Mark-doc-as-relevant actions with no delays: %i" % (
  len(Action.filter_by_type('DOC_MARKED_RELEVANT', Action.no_delays_filter)))
print "Mark-doc-as-relevant actions with no delays and moderately relevant doc: %i" % (
  len(Action.filter_by_type('DOC_MARKED_RELEVANT', Action.combine_filters( Action.no_delays_filter, Action.moderately_relevant_filter))))
print "Mark-doc-as-relevant actions with no delays and highly relevant doc: %i" % (
  len(Action.filter_by_type('DOC_MARKED_RELEVANT', Action.combine_filters( Action.no_delays_filter, Action.highly_relevant_filter))))

print "Mark-doc-as-viewed actions with query delays: %i" % (
  len(Action.filter_by_type('DOC_MARKED_VIEWED', Action.query_delay_filter)))
print "Mark-doc-as-viewed actions with query delays and moderately relevant doc: %i" % (
  len(Action.filter_by_type('DOC_MARKED_VIEWED', Action.combine_filters( Action.query_delay_filter, Action.moderately_relevant_filter))))
print "Mark-doc-as-viewed actions with query delays and highly relevant doc: %i" % (
  len(Action.filter_by_type('DOC_MARKED_VIEWED', Action.combine_filters( Action.query_delay_filter, Action.highly_relevant_filter))))

print "Mark-doc-as-relevant actions with query delays: %i" % (
  len(Action.filter_by_type('DOC_MARKED_RELEVANT', Action.query_delay_filter)))
print "Mark-doc-as-relevant actions with query delays and moderately relevant doc: %i" % (
  len(Action.filter_by_type('DOC_MARKED_RELEVANT', Action.combine_filters( Action.query_delay_filter, Action.moderately_relevant_filter))))
print "Mark-doc-as-relevant actions with query delays and highly relevant doc: %i" % (
  len(Action.filter_by_type('DOC_MARKED_RELEVANT', Action.combine_filters( Action.query_delay_filter, Action.highly_relevant_filter))))

print "Mark-doc-as-viewed actions with document delays: %i" % (
  len(Action.filter_by_type('DOC_MARKED_VIEWED', Action.document_delay_filter)))
print "Mark-doc-as-viewed actions with document delays and moderately relevant doc: %i" % (
  len(Action.filter_by_type('DOC_MARKED_VIEWED', Action.combine_filters( Action.document_delay_filter, Action.moderately_relevant_filter))))
print "Mark-doc-as-viewed actions with document delays and highly relevant doc: %i" % (
  len(Action.filter_by_type('DOC_MARKED_VIEWED', Action.combine_filters( Action.document_delay_filter, Action.highly_relevant_filter))))

print "Mark-doc-as-relevant actions with document delays: %i" % (
  len(Action.filter_by_type('DOC_MARKED_RELEVANT', Action.document_delay_filter)))
print "Mark-doc-as-relevant actions with document delays and moderately relevant doc: %i" % (
  len(Action.filter_by_type('DOC_MARKED_RELEVANT', Action.combine_filters( Action.document_delay_filter, Action.moderately_relevant_filter))))
print "Mark-doc-as-relevant actions with document delays and highly relevant doc: %i" % (
  len(Action.filter_by_type('DOC_MARKED_RELEVANT', Action.combine_filters( Action.document_delay_filter, Action.highly_relevant_filter))))

print "Mark-doc-as-viewed actions with both delays: %i" % (
  len(Action.filter_by_type('DOC_MARKED_VIEWED', Action.combined_delay_filter)))
print "Mark-doc-as-viewed actions with both delays and moderately relevant doc: %i" % (
  len(Action.filter_by_type('DOC_MARKED_VIEWED', Action.combine_filters( Action.combined_delay_filter, Action.moderately_relevant_filter))))
print "Mark-doc-as-viewed actions with both delays and highly relevant doc: %i" % (
  len(Action.filter_by_type('DOC_MARKED_VIEWED', Action.combine_filters( Action.combined_delay_filter, Action.highly_relevant_filter))))

print "Mark-doc-as-relevant actions with both delays: %i" % (
  len(Action.filter_by_type('DOC_MARKED_RELEVANT', Action.combined_delay_filter)))
print "Mark-doc-as-relevant actions with both delays and moderately relevant doc: %i" % (
  len(Action.filter_by_type('DOC_MARKED_RELEVANT', Action.combine_filters( Action.combined_delay_filter, Action.moderately_relevant_filter))))
print "Mark-doc-as-relevant actions with both delays and highly relevant doc: %i" % (
  len(Action.filter_by_type('DOC_MARKED_RELEVANT', Action.combine_filters( Action.combined_delay_filter, Action.highly_relevant_filter))))
<|endoftext|>


<commit_before>#!/usr/bin/python

import argparse

from pebble.PblCommand import PblCommand
from pebble.PblProjectCreator import PblProjectCreator
from pebble.PblBuildCommand import PblBuildCommand

class PbSDKShell:
  commands = []

  def __init__(self):
    self.commands.append(PblProjectCreator())
    self.commands.append(PblBuildCommand())

  def main(self):
    parser = argparse.ArgumentParser(description = 'Pebble SDK Shell')
    subparsers = parser.add_subparsers(dest="command", title="Command", description="Action to perform")
    for command in self.commands:
      subparser = subparsers.add_parser(command.name, help = command.help)
      command.configure_subparser(subparser)
    args = parser.parse_args()

    # Find the extension that was called
    command = [x for x in self.commands if x.name == args.command][0]
    command.run(args)

  def run_action(self, action):
    pass

if __name__ == '__main__':
  PbSDKShell().main()

<commit_msg>Change pb-sdk.py to exec env python<commit_after>#!/usr/bin/env python

import argparse

from pebble.PblCommand import PblCommand
from pebble.PblProjectCreator import PblProjectCreator
from pebble.PblBuildCommand import PblBuildCommand

class PbSDKShell:
  commands = []

  def __init__(self):
    self.commands.append(PblProjectCreator())
    self.commands.append(PblBuildCommand())

  def main(self):
    parser = argparse.ArgumentParser(description = 'Pebble SDK Shell')
    subparsers = parser.add_subparsers(dest="command", title="Command", description="Action to perform")
    for command in self.commands:
      subparser = subparsers.add_parser(command.name, help = command.help)
      command.configure_subparser(subparser)
    args = parser.parse_args()

    # Find the extension that was called
    command = [x for x in self.commands if x.name == args.command][0]
    command.run(args)

  def run_action(self, action):
    pass

if __name__ == '__main__':
  PbSDKShell().main()

<|endoftext|>