<commit_before>#!/usr/bin/env python

# Packrat PEG (parsing expression grammar) generator
# http://pdos.csail.mit.edu/~baford/packrat/

next_var = 0
def nextVar():
    global next_var;
    next_var += 1;
    return next_var

def newResult():
    return "result_%d" % nextVar()

def newOut():
    return "out_%d" % nextVar()

def indent(s):
    space = '    '
    return s.replace('\n', '\n%s' % space)

start_code = """
class Result{
public:
    Result(const int position):
    position(position),
    isError(false){
    }

    inline const int getPosition() const {
        return position;
    }

    inline bool error(){
        return isError;
    }

    inline void nextPosition(){
        position += 1;
    }

    void setError(){
        isError = true;
    }

    void setValue(void * value){
        this->values.push_back(value);
    }

    const int matches() const {
        return this->values.size();
    }

    const std::vector<void *> & getValues() const {
        return values;
    }

    void addResult(const Result & result){
        const std::vector<void *> & his = result.getValues();
        this->values.insert(this->values.end(), his.begin(), his.end());
        this->position = result.getPosition();
    }

private:
    int position;
    bool isError;
    std::vector<void *> values;
};

class Stream{
public:
    Stream(const std::string & filename){
        stream.open(filename.c_str());
    }

    char get(const int position){
        char z;
        stream.seekg(position, std::ios_base::beg);
        stream >> z;
        return z;
    }

    void update(const Result & result){
    }

    bool hasResult(const int position){
        return false;
    }

    Result result(const int position){
        return Result(-1);
    }

private:
    std::ifstream stream;
};

Result errorResult(-1);
"""

class Pattern:
    def __init__(self):
        pass

    def generate(self, result, stream, failure):
        pass

class PatternNot(Pattern):
    def __init__(self, next):
        Pattern.__init__(self)
        self.next = next

    def generate_python(self, result, stream, failure):
        my_result = newResult()
        my_fail = lambda : "raise PegError"
        data = """
%s = Result(%s);
try:
    %s
except PegError:
    %s
        """ % (my_result, result, indent(self.next.generate_python(my_result, stream, my_fail).strip()), failure())

        return data

    def generate(self, result, stream, failure):
        not_label = "not_%d" % nextVar()
        my_result = newResult()
        my_fail = lambda : "goto %s;" % not_label
        data = """
Result %s(%s);
%s
%s
%s:
        """ % (my_result, result, self.next.generate(my_result, stream, my_fail).strip(), failure(), not_label)

        return data

class PatternRule(Pattern):
    def __init__(self, rule):
        Pattern.__init__(self)
        self.rule = rule

    def generate_python(self, result, stream, failure):
        data = """
%s = rule_%s(%s, %s.getPosition())
if %s == None:
    %s
""" % (result, self.rule, stream, result, result, failure())

        return data

    def generate(self, result, stream, failure):
        data = """
%s = rule_%s(%s, %s.getPosition());
if (%s.error()){
    %s
}
""" % (result, self.rule, stream, result, result, failure())

        return data

class PatternSequence(Pattern):
    def __init__(self, patterns):
        Pattern.__init__(self)
        self.patterns = patterns

    def generate_python(self, result, stream, failure):
        data = ""
        for pattern in self.patterns:
            my_result = newResult()
            data += """
%s = Result(%s.getPosition())
%s
%s.addResult(%s);
""" % (my_result, result, pattern.generate_python(my_result, stream, failure), result, my_result)

        return data

    def generate(self, result, stream, failure):
        data = ""
        for pattern in self.patterns:
            my_result = newResult()
            data += """
{
Result %s(%s.getPosition());
%s
%s.addResult(%s);
}
""" % (my_result, result, pattern.generate(my_result, stream, failure), result, my_result)
        return data

class PatternRepeatOnce(Pattern):
    def __init__(self, next):
        Pattern.__init__(self)
        self.next = next

    def generate_python(self, result, stream, failure):
        loop_done = "loop_%d" % nextVar()
        my_fail = lambda : "raise PegError"
        my_result = newResult()
        data = """
try:
    while True:
        %s = Result(%s.getPosition());
        %s
        %s.addResult(%s);
except PegError:
    if %s.matches() == 0:
        %s
        """ % (my_result, result, indent(indent(self.next.generate_python(my_result, stream, my_fail).strip())), result, my_result, result, failure())

        return data

    def generate(self, result, stream, failure):
        loop_done = "loop_%d" % nextVar()
        my_fail = lambda : "goto %s;" % loop_done
        my_result = newResult()
        data = """
do{
    Result %s(%s.getPosition());
    %s
    %s.addResult(%s);
} while (true);
%s:
if (%s.matches() == 0){
    %s
}
        """ % (my_result, result, indent(self.next.generate(my_result, stream, my_fail).strip()), result, my_result, loop_done, result, failure())

        return data

class PatternAction(Pattern):
    def __init__(self, before, code):
        Pattern.__init__(self)
        self.before = before
        self.code = code

    def generate_python(self, result, stream, failure):
        data = """
%s
if True:
    value = None
    %s
    %s.setValue(value)
""" % (self.before.generate_python(result, stream, failure).strip(), indent(self.code.strip()), result)

        return data

    def generate(self, result, stream, failure):
        data = """
%s
{
    void * value = 0;
    %s
    %s.setValue(value);
}
        """ % (self.before.generate(result, stream, failure).strip(), indent(self.code.strip()), result)

        return data

class PatternRepeatMany(Pattern):
    def __init__(self, next):
        Pattern.__init__(self)
        self.next = next

    def generate_python(self, result, stream, failure):
        my_fail = lambda : "raise PegError"
        my_result = newResult()
        data = """
try:
    while True:
        %s = Result(%s.getPosition());
        %s
        %s.addResult(%s);
except PegError:
    pass
        """ % (my_result, result, indent(indent(self.next.generate_python(my_result, stream, my_fail).strip())), result, my_result)

        return data

    def generate(self, result, stream, failure):
        loop_done = "loop_%d" % nextVar()
        my_fail = lambda : "goto %s;" % loop_done
        my_result = newResult()
        data = """
do{
    Result %s(%s.getPosition());
    %s
    %s.addResult(%s);
} while (true);
%s:
        """ % (my_result, result, indent(self.next.generate(my_result, stream, my_fail).strip()), result, my_result, loop_done)
        return data

class PatternOr(Pattern):
    def __init__(self, patterns):
        Pattern.__init__(self)
        self.patterns = patterns

    def generate(self, result, stream, failure):
        data = ""
        success = "success_%d" % nextVar()
        for pattern in self.patterns:
            out = "or_%d" % nextVar()
            my_result = newResult()
            fail = lambda : "goto %s;" % out
            if pattern == self.patterns[-1]:
                fail = failure
            data += """
{
Result %s(%s.getPosition());
%s
%s = %s;
}
goto %s;
%s:
""" % (my_result, result, pattern.generate(my_result, stream, fail).strip(), result, my_result, success, out)
        data += "%s:\n" % success
        return data

class PatternVerbatim(Pattern):
    def __init__(self, letters):
        Pattern.__init__(self)
        self.letters = letters

    def generate_python(self, result, stream, failure):
        data = """
for letter in '%s':
    if letter == %s.get(%s.getPosition()):
        %s.nextPosition()
    else:
        %s
""" % (self.letters, stream, result, result, failure())
        return data

    def generate(self, result, stream, failure):
        data = """
%s = "%s";
        """ % (result, self.letters)

        data = ""

        for letter in self.letters[::-1]:
            newdata = """
if (%s.get(%s.getPosition()) == '%s'){
    %s.nextPosition();
    %s
} else {
    %s
}
""" % (stream, result, letter, result, indent(data), failure())
            data = newdata

        # return data

        data = """
for (int i = 0; i < %d; i++){
    if ("%s"[i] == %s.get(%s.getPosition())){
        %s.nextPosition();
    } else {
        %s
    }
}
""" % (len(self.letters), self.letters, stream, result, result, failure())
        return data

class Rule:
    def __init__(self, name, patterns):
        self.name = name
        self.patterns = patterns

    def generate_python(self):
        def newPattern(pattern, stream, position):
            result = newResult()

            def fail():
                return "raise PegError"
            data = """
try:
    %s = Result(%s)
    %s
    %s.update(%s)
    return %s
except PegError:
    pass
            """ % (result, position, indent(pattern.generate_python(result, stream, fail).strip()), stream, result, result)
            return data

        stream = "stream"
        position = "position"
        data = """
def rule_%s(%s, %s):
    %s
    return None
""" % (self.name, stream, position, indent('\n'.join([newPattern(pattern, stream, position).strip() for pattern in self.patterns])))

        return data

    def generate(self):
        def newPattern(pattern, stream, position):
            result = newResult()
            out = newOut()
            def failure():
                return "goto %s;" % out
            data = """
Result %s(%s);
%s
%s.update(%s);
return %s;
%s:
            """ % (result, position, pattern.generate(result, stream, failure).strip(), stream, result, result, out)
            return data

        stream = "stream"
        position = "position"
        data = """
Result rule_%s(Stream & %s, const int %s){
    if (%s.hasResult(%s)){
        return %s.result(%s);
    }
    %s
    return errorResult;
}
        """ % (self.name, stream, position, stream, position, stream, position, indent('\n'.join([newPattern(pattern, stream, position).strip() for pattern in self.patterns])))

        return data

class Peg:
    def __init__(self, namespace, start, rules):
        self.namespace = namespace
        self.start = start
        self.rules = rules

    def generate_python(self):
        data = """
import peg

%s

def parse(file):
    stream = Stream(file)
    done = rule_%s(stream, 0)
    if (done.error()):
        print "Error parsing " + file
    else:
        return done.getValues()
""" % ('\n'.join([rule.generate_python() for rule in self.rules]), self.start)

        return data

    def generate(self):
        def prototype(rule):
            return "Result rule_%s(Stream &, const int);" % rule.name

        data = """
#include <vector>
#include <string>
#include <fstream>
#include <iostream>

namespace %s{
    %s

    %s

    %s

std::vector<void *> main(const std::string & filename){
    Stream stream(filename);
    errorResult.setError();
    Result done = rule_%s(stream, 0);
    if (done.error()){
        std::cout << "Could not parse" << std::endl;
    }
    return done.getValues();
}

}
        """ % (self.namespace, start_code, indent('\n'.join([prototype(rule) for rule in self.rules])), '\n'.join([rule.generate() for rule in self.rules]), self.start)

        return data

def generate(peg):
    print peg.generate()

def test():
    s_code = """
printf("parsed cheese\\n");
value = (void *) 2;
"""
    rules = [
        Rule("s", [PatternNot(PatternVerbatim("hello")), PatternAction(PatternVerbatim("cheese"), s_code), PatternRepeatOnce(PatternVerbatim("once"))]),
        Rule("blah", [PatternRepeatMany(PatternRule("s"))]),
        Rule("or", [PatternOr([PatternVerbatim("joe"), PatternVerbatim("bob"), PatternVerbatim("sally")])]),
        Rule("all", [PatternSequence([PatternVerbatim("abc"), PatternVerbatim("def"), PatternVerbatim("ghi")])]),
    ]
    peg = Peg("Peg", "s", rules)
    generate(peg)

def create_peg(name, peg):
    import imp
    module = imp.new_module(name)
    exec peg.generate_python() in module.__dict__
    return module.parse

def test2():
    start_code_abc = """
std::cout << "Parsed abc!" << std::endl;
"""
    start_code_def = """
std::cout << "Parsed def!" << std::endl;
"""
    rules = [
        Rule("start", [
            PatternAction(PatternSequence([PatternRule("a"),PatternRule("b"), PatternRule("c")]), start_code_abc),
            PatternAction(PatternSequence([PatternRule("d"),PatternRule("e"), PatternRule("f")]), start_code_def),
        ]),
        Rule("a", [PatternVerbatim("a")]),
        Rule("b", [PatternVerbatim("b")]),
        Rule("c", [PatternVerbatim("c")]),

        Rule("d", [PatternVerbatim("d")]),
        Rule("e", [PatternVerbatim("e")]),
        Rule("f", [PatternVerbatim("f")]),
    ]

    peg = Peg("Peg", "start", rules)
    generate(peg)

def make_peg_parser():
    start_code_abc = """
print "parsed abc"
"""
    s_code = """
print "s code"
"""
    rules = [
        Rule("start", [
            PatternAction(PatternSequence([PatternRule("a"),PatternRule("b"), PatternRule("c")]), start_code_abc),
            ]),
        Rule("s", [
            PatternNot(PatternVerbatim("hello")), PatternAction(PatternVerbatim("cheese"), s_code),
            PatternRepeatOnce(PatternVerbatim("once"))]),
        Rule("blah", [PatternRepeatMany(PatternRule("s"))]),
        ]
    peg = Peg("*peg*", "start", rules)
    print peg.generate_python()

# test()
# test2()

make_peg_parser()
<commit_msg>generate python parser in python<commit_after>#!/usr/bin/env python

# Packrat PEG (parsing expression grammar) generator
# http://pdos.csail.mit.edu/~baford/packrat/

next_var = 0
def nextVar():
    global next_var;
    next_var += 1;
    return next_var

def newResult():
    return "result_%d" % nextVar()

def newOut():
    return "out_%d" % nextVar()

def indent(s):
    space = '    '
    return s.replace('\n', '\n%s' % space)

start_code = """
class Result{
public:
    Result(const int position):
    position(position),
    isError(false){
    }

    inline const int getPosition() const {
        return position;
    }

    inline bool error(){
        return isError;
    }

    inline void nextPosition(){
        position += 1;
    }

    void setError(){
        isError = true;
    }

    void setValue(void * value){
        this->values.push_back(value);
    }

    const int matches() const {
        return this->values.size();
    }

    const std::vector<void *> & getValues() const {
        return values;
    }

    void addResult(const Result & result){
        const std::vector<void *> & his = result.getValues();
        this->values.insert(this->values.end(), his.begin(), his.end());
        this->position = result.getPosition();
    }

private:
    int position;
    bool isError;
    std::vector<void *> values;
};

class Stream{
public:
    Stream(const std::string & filename){
        stream.open(filename.c_str());
    }

    char get(const int position){
        char z;
        stream.seekg(position, std::ios_base::beg);
        stream >> z;
        return z;
    }

    void update(const Result & result){
    }

    bool hasResult(const int position){
        return false;
    }

    Result result(const int position){
        return Result(-1);
    }

private:
    std::ifstream stream;
};

Result errorResult(-1);
"""

start_python = """
class PegError(Exception):
    def __init__(self):
        Exception.__init__(self)

class Result:
    def __init__(self, position):
        self.position = position
        self.values = []

    def getPosition(self):
        return self.position

    def nextPosition(self):
        self.position += 1

    def setValue(self, value):
        self.values = [value]

    def matches(self):
        return len(self.values)

    def getValues(self):
        return self.values

    def addResult(self, him):
        self.values.extend(him.values)
        self.position = him.position

class Stream:
    def __init__(self, filename):
        self.file = open(filename, 'r')

    def close(self):
        self.file.close()

    def get(self, position):
        self.file.seek(position)
        return self.file.read(1)

    def update(self, result):
        pass

    def hasResult(self, position):
        return false

    def result(self, position):
        return Result(-1)
"""

class Pattern:
    def __init__(self):
        pass

    def generate(self, result, stream, failure):
        pass

class PatternNot(Pattern):
    def __init__(self, next):
        Pattern.__init__(self)
        self.next = next

    def generate_python(self, result, stream, failure):
        my_result = newResult()
        my_fail = lambda : "raise PegError"
        data = """
%s = Result(%s);
try:
    %s
except PegError:
    %s
        """ % (my_result, result, indent(self.next.generate_python(my_result, stream, my_fail).strip()), failure())

        return data

    def generate(self, result, stream, failure):
        not_label = "not_%d" % nextVar()
        my_result = newResult()
        my_fail = lambda : "goto %s;" % not_label
        data = """
Result %s(%s);
%s
%s
%s:
        """ % (my_result, result, self.next.generate(my_result, stream, my_fail).strip(), failure(), not_label)

        return data

class PatternRule(Pattern):
    def __init__(self, rule):
        Pattern.__init__(self)
        self.rule = rule

    def generate_python(self, result, stream, failure):
        data = """
%s = rule_%s(%s, %s.getPosition())
if %s == None:
    %s
""" % (result, self.rule, stream, result, result, failure())

        return data

    def generate(self, result, stream, failure):
        data = """
%s = rule_%s(%s, %s.getPosition());
if (%s.error()){
    %s
}
""" % (result, self.rule, stream, result, result, failure())

        return data

class PatternSequence(Pattern):
    def __init__(self, patterns):
        Pattern.__init__(self)
        self.patterns = patterns

    def generate_python(self, result, stream, failure):
        data = ""
        for pattern in self.patterns:
            my_result = newResult()
            data += """
%s = Result(%s.getPosition())
%s
%s.addResult(%s);
""" % (my_result, result, pattern.generate_python(my_result, stream, failure), result, my_result)

        return data

    def generate(self, result, stream, failure):
        data = ""
        for pattern in self.patterns:
            my_result = newResult()
            data += """
{
Result %s(%s.getPosition());
%s
%s.addResult(%s);
}
""" % (my_result, result, pattern.generate(my_result, stream, failure), result, my_result)
        return data

class PatternRepeatOnce(Pattern):
    def __init__(self, next):
        Pattern.__init__(self)
        self.next = next

    def generate_python(self, result, stream, failure):
        loop_done = "loop_%d" % nextVar()
        my_fail = lambda : "raise PegError"
        my_result = newResult()
        data = """
try:
    while True:
        %s = Result(%s.getPosition());
        %s
        %s.addResult(%s);
except PegError:
    if %s.matches() == 0:
        %s
        """ % (my_result, result, indent(indent(self.next.generate_python(my_result, stream, my_fail).strip())), result, my_result, result, failure())

        return data

    def generate(self, result, stream, failure):
        loop_done = "loop_%d" % nextVar()
        my_fail = lambda : "goto %s;" % loop_done
        my_result = newResult()
        data = """
do{
    Result %s(%s.getPosition());
    %s
    %s.addResult(%s);
} while (true);
%s:
if (%s.matches() == 0){
    %s
}
        """ % (my_result, result, indent(self.next.generate(my_result, stream, my_fail).strip()), result, my_result, loop_done, result, failure())

        return data

class PatternAction(Pattern):
    def __init__(self, before, code):
        Pattern.__init__(self)
        self.before = before
        self.code = code

    def generate_python(self, result, stream, failure):
        data = """
%s
if True:
    value = None
    %s
    %s.setValue(value)
""" % (self.before.generate_python(result, stream, failure).strip(), indent(self.code.strip()), result)

        return data

    def generate(self, result, stream, failure):
        data = """
%s
{
    void * value = 0;
    %s
    %s.setValue(value);
}
        """ % (self.before.generate(result, stream, failure).strip(), indent(self.code.strip()), result)

        return data

class PatternRepeatMany(Pattern):
    def __init__(self, next):
        Pattern.__init__(self)
        self.next = next

    def generate_python(self, result, stream, failure):
        my_fail = lambda : "raise PegError"
        my_result = newResult()
        data = """
try:
    while True:
        %s = Result(%s.getPosition());
        %s
        %s.addResult(%s);
except PegError:
    pass
        """ % (my_result, result, indent(indent(self.next.generate_python(my_result, stream, my_fail).strip())), result, my_result)

        return data

    def generate(self, result, stream, failure):
        loop_done = "loop_%d" % nextVar()
        my_fail = lambda : "goto %s;" % loop_done
        my_result = newResult()
        data = """
do{
    Result %s(%s.getPosition());
    %s
    %s.addResult(%s);
} while (true);
%s:
        """ % (my_result, result, indent(self.next.generate(my_result, stream, my_fail).strip()), result, my_result, loop_done)
        return data

class PatternOr(Pattern):
    def __init__(self, patterns):
        Pattern.__init__(self)
        self.patterns = patterns

    def generate(self, result, stream, failure):
        data = ""
        success = "success_%d" % nextVar()
        for pattern in self.patterns:
            out = "or_%d" % nextVar()
            my_result = newResult()
            fail = lambda : "goto %s;" % out
            if pattern == self.patterns[-1]:
                fail = failure
            data += """
{
Result %s(%s.getPosition());
%s
%s = %s;
}
goto %s;
%s:
""" % (my_result, result, pattern.generate(my_result, stream, fail).strip(), result, my_result, success, out)
        data += "%s:\n" % success
        return data

class PatternVerbatim(Pattern):
    def __init__(self, letters):
        Pattern.__init__(self)
        self.letters = letters

    def generate_python(self, result, stream, failure):
        data = """
for letter in '%s':
    if letter == %s.get(%s.getPosition()):
        %s.nextPosition()
    else:
        %s
""" % (self.letters, stream, result, result, failure())
        return data

    def generate(self, result, stream, failure):
        data = """
%s = "%s";
        """ % (result, self.letters)

        data = ""

        for letter in self.letters[::-1]:
            newdata = """
if (%s.get(%s.getPosition()) == '%s'){
    %s.nextPosition();
    %s
} else {
    %s
}
""" % (stream, result, letter, result, indent(data), failure())
            data = newdata

        # return data

        data = """
for (int i = 0; i < %d; i++){
    if ("%s"[i] == %s.get(%s.getPosition())){
        %s.nextPosition();
    } else {
        %s
    }
}
""" % (len(self.letters), self.letters, stream, result, result, failure())
        return data

class Rule:
    def __init__(self, name, patterns):
        self.name = name
        self.patterns = patterns

    def generate_python(self):
        def newPattern(pattern, stream, position):
            result = newResult()

            def fail():
                return "raise PegError"
            data = """
try:
    %s = Result(%s)
    %s
    %s.update(%s)
    return %s
except PegError:
    pass
            """ % (result, position, indent(pattern.generate_python(result, stream, fail).strip()), stream, result, result)
            return data

        stream = "stream"
        position = "position"
        data = """
def rule_%s(%s, %s):
    %s
    return None
""" % (self.name, stream, position, indent('\n'.join([newPattern(pattern, stream, position).strip() for pattern in self.patterns])))

        return data

    def generate(self):
        def newPattern(pattern, stream, position):
            result = newResult()
            out = newOut()
            def failure():
                return "goto %s;" % out
            data = """
Result %s(%s);
%s
%s.update(%s);
return %s;
%s:
            """ % (result, position, pattern.generate(result, stream, failure).strip(), stream, result, result, out)
            return data

        stream = "stream"
        position = "position"
        data = """
Result rule_%s(Stream & %s, const int %s){
    if (%s.hasResult(%s)){
        return %s.result(%s);
    }
    %s
    return errorResult;
}
        """ % (self.name, stream, position, stream, position, stream, position, indent('\n'.join([newPattern(pattern, stream, position).strip() for pattern in self.patterns])))

        return data

class Peg:
    def __init__(self, namespace, start, rules):
        self.namespace = namespace
        self.start = start
        self.rules = rules

    def generate_python(self):
        data = """
import peg

%s

%s

def parse(file):
    # print "Parsing " + file
    stream = Stream(file)
    done = rule_%s(stream, 0)
    stream.close()
    if done == None:
        print "Error parsing " + file
        return []
    else:
        return done.getValues()
""" % (start_python, '\n'.join([rule.generate_python() for rule in self.rules]), self.start)

        return data

    def generate(self):
        def prototype(rule):
            return "Result rule_%s(Stream &, const int);" % rule.name

        data = """
#include <vector>
#include <string>
#include <fstream>
#include <iostream>

namespace %s{
    %s

    %s

    %s

std::vector<void *> main(const std::string & filename){
    Stream stream(filename);
    errorResult.setError();
    Result done = rule_%s(stream, 0);
    if (done.error()){
        std::cout << "Could not parse" << std::endl;
    }
    return done.getValues();
}

}
        """ % (self.namespace, start_code, indent('\n'.join([prototype(rule) for rule in self.rules])), '\n'.join([rule.generate() for rule in self.rules]), self.start)

        return data

def generate(peg):
    print peg.generate()

def test():
    s_code = """
printf("parsed cheese\\n");
value = (void *) 2;
"""
    rules = [
        Rule("s", [PatternNot(PatternVerbatim("hello")), PatternAction(PatternVerbatim("cheese"), s_code), PatternRepeatOnce(PatternVerbatim("once"))]),
        Rule("blah", [PatternRepeatMany(PatternRule("s"))]),
        Rule("or", [PatternOr([PatternVerbatim("joe"), PatternVerbatim("bob"), PatternVerbatim("sally")])]),
        Rule("all", [PatternSequence([PatternVerbatim("abc"), PatternVerbatim("def"), PatternVerbatim("ghi")])]),
    ]
    peg = Peg("Peg", "s", rules)
    generate(peg)

def create_peg(peg):
    # import imp
    # module = imp.new_module(peg.namespace)
    # exec peg.generate_python() in module.__dict__
    # return module.parse

    name = "peg_" + peg.namespace
    out = open(name + ".py", 'w')
    out.write(peg.generate_python())
    out.close()
    module = __import__(name, globals(), locals(), ['parse'])
    # print module
    # print dir(module)
    return module.parse

def test2():
    start_code_abc = """
std::cout << "Parsed abc!" << std::endl;
"""
    start_code_def = """
std::cout << "Parsed def!" << std::endl;
"""
    rules = [
        Rule("start", [
            PatternAction(PatternSequence([PatternRule("a"),PatternRule("b"), PatternRule("c")]), start_code_abc),
            PatternAction(PatternSequence([PatternRule("d"),PatternRule("e"), PatternRule("f")]), start_code_def),
        ]),
        Rule("a", [PatternVerbatim("a")]),
        Rule("b", [PatternVerbatim("b")]),
        Rule("c", [PatternVerbatim("c")]),

        Rule("d", [PatternVerbatim("d")]),
        Rule("e", [PatternVerbatim("e")]),
        Rule("f", [PatternVerbatim("f")]),
    ]

    peg = Peg("Peg", "start", rules)
    generate(peg)

def make_peg_parser():
    start_code_abc = """
print "parsed abc"
"""
    s_code = """
print "s code"
"""
    rules = [
        Rule("start", [
            PatternAction(PatternSequence([PatternRule("a"),PatternRule("b"), PatternRule("c")]), start_code_abc),
            ]),
        Rule("a", [PatternVerbatim("a")]),
        Rule("b", [PatternVerbatim("b")]),
        Rule("c", [PatternVerbatim("c")]),

        Rule("s", [
            PatternNot(PatternVerbatim("hello")), PatternAction(PatternVerbatim("cheese"), s_code),
            PatternRepeatOnce(PatternVerbatim("once"))]),
        Rule("blah", [PatternRepeatMany(PatternRule("s"))]),
        ]
    peg = Peg("peg", "start", rules)
    # print peg.generate_python()
    parser = create_peg(peg)
    # print parser
    print "Got " + str(parser('peg.in'))
    # module = compile(peg.generate_python(), peg.namespace, 'exec')
    # print module

# test()
# test2()

# make_peg_parser()
if __name__ == '__main__':
    make_peg_parser()
<|endoftext|>


<commit_before>import copy
import pytest
import mock
import six

import scrapi
from scrapi.linter.document import NormalizedDocument

from scrapi import tasks
from scrapi import registry
from scrapi.migrations import delete
from scrapi.migrations import rename
from scrapi.migrations import cross_db
from scrapi.migrations import renormalize
from scrapi.migrations import DocumentModelOld

# Need to force cassandra to ignore set keyspace
from scrapi.processing.postgres import PostgresProcessor, Document
from scrapi.processing.cassandra import CassandraProcessor, DocumentModel
from scrapi.processing.elasticsearch import ElasticsearchProcessor

from . import utils

test_cass = CassandraProcessor()
test_postgres = PostgresProcessor()
test_es = ElasticsearchProcessor()

test_harvester = utils.TestHarvester()

NORMALIZED = NormalizedDocument(utils.RECORD)
RAW = test_harvester.harvest()[0]

@pytest.fixture
def harvester():
    pass  # Need to override this

@pytest.mark.django_db
@pytest.mark.cassandra
def test_rename():
    real_es = scrapi.processing.elasticsearch.es
    scrapi.processing.elasticsearch.es = mock.MagicMock()
    test_cass.process_raw(RAW)
    test_cass.process_normalized(RAW, NORMALIZED)

    queryset = DocumentModel.objects(docID=RAW['docID'], source=RAW['source'])
    old_source = NORMALIZED['shareProperties']['source']

    assert queryset[0].source == utils.RECORD['shareProperties']['source']
    assert queryset[0].source == old_source

    new_record = copy.deepcopy(utils.RECORD)

    new_record['shareProperties']['source'] = 'wwe_news'

    test_harvester.short_name = 'wwe_news'

    registry['wwe_news'] = test_harvester

    tasks.migrate(rename, sources=[old_source], target='wwe_news', dry=False)

    queryset = DocumentModel.objects(docID=RAW['docID'], source='wwe_news')
    assert queryset[0].source == 'wwe_news'
    assert len(queryset) == 1
    scrapi.processing.elasticsearch.es = real_es
    test_harvester.short_name = RAW['source']
    registry['test'] = test_harvester
    del registry['wwe_news']

@pytest.mark.django_db
@pytest.mark.cassandra
def test_delete():
    real_es = scrapi.processing.elasticsearch.es
    scrapi.processing.elasticsearch.es = mock.MagicMock()
    test_cass.process_raw(RAW)
    test_cass.process_normalized(RAW, NORMALIZED)

    queryset = DocumentModel.objects(docID=RAW['docID'], source=RAW['source'])
    assert len(queryset) == 1

    tasks.migrate(delete, sources=[RAW['source']], dry=False)
    queryset = DocumentModel.objects(docID=RAW['docID'], source=RAW['source'])
    assert len(queryset) == 0
    scrapi.processing.elasticsearch.es = real_es

@pytest.mark.django_db
@pytest.mark.cassandra
def test_renormalize():
    # Set up
    real_es = scrapi.processing.elasticsearch.es
    scrapi.processing.elasticsearch.es = mock.MagicMock()

    # Process raw and normalized with fake docs
    test_cass.process_raw(RAW)
    test_cass.process_normalized(RAW, NORMALIZED)

    # Check to see those docs were processed
    queryset = DocumentModel.objects(docID=RAW['docID'], source=RAW['source'])
    assert len(queryset) == 1

    # Create a new doucment to be renormalized
    new_raw = copy.deepcopy(RAW)
    new_raw.attributes['docID'] = 'get_the_tables'
    new_raw.attributes['doc'] = new_raw.attributes['doc'].encode('utf-8')

    # This is basically like running the improved harvester right?
    DocumentModel.create(**new_raw.attributes).save()

    tasks.migrate(renormalize, sources=[RAW['source']], dry=False)

    queryset = DocumentModel.objects(docID='get_the_tables', source=RAW['source'])
    assert len(queryset) == 1
    scrapi.processing.elasticsearch.es = real_es

@pytest.mark.django_db
@pytest.mark.cassandra
def test_migrate_v2():
    try:
        RAW['doc'] = RAW['doc'].encode('utf-8')
    except AttributeError:
        RAW['doc'] = str(RAW['doc'])
    DocumentModelOld.create(**RAW.attributes).save()
    queryset = DocumentModel.objects(docID=RAW['docID'], source=RAW['source'])
    assert len(queryset) == 0
    tasks.migrate_to_source_partition(dry=False)
    queryset = DocumentModel.objects(docID=RAW['docID'], source=RAW['source'])
    assert len(queryset) == 1

@pytest.mark.django_db
@pytest.mark.cassandra
def test_cassandra_to_postgres():

    test_cass.process_raw(RAW)
    test_cass.process_normalized(RAW, NORMALIZED)

    cassandra_queryset = DocumentModel.objects(docID=RAW['docID'], source=RAW['source'])
    postgres_queryset = Document.objects.filter(source=RAW['source'], docID=RAW['docID'])
    assert len(cassandra_queryset) == 1
    assert len(postgres_queryset) == 0
    tasks.migrate(cross_db, target_db='postgres', dry=False, sources=['test'])

    postgres_queryset = Document.objects.filter(source=RAW['source'], docID=RAW['docID'])
    assert len(postgres_queryset) == 1

@pytest.mark.django_db
@pytest.mark.elasticsearch
def test_cassandra_to_elasticsearch():

    # Make sure es has nothing in it
    results = test_es.search(index='test', doc_type=RAW['source'])
    assert (len(results['hits']['hits']) == 0)

    # create the postgres documents
    test_cass.process_raw(RAW)
    test_cass.process_normalized(RAW, NORMALIZED)

    # run the migration
    tasks.migrate(cross_db, target_db='elasticsearch', dry=False)

    # check the elasticsearch results
    results = test_es.search(index='test', doc_type=RAW['source'])
    assert (len(results['hits']['hits']) == 1)
<commit_msg>Add a while loop to wait for the index to be created before querying<commit_after>import six
import copy
import mock
import pytest

from elasticsearch import TransportError

import scrapi
from scrapi.linter.document import NormalizedDocument

from scrapi import tasks
from scrapi import registry
from scrapi.migrations import delete
from scrapi.migrations import rename
from scrapi.migrations import cross_db
from scrapi.migrations import renormalize
from scrapi.migrations import DocumentModelOld

# Need to force cassandra to ignore set keyspace
from scrapi.processing.postgres import Document
from scrapi.processing.cassandra import CassandraProcessor, DocumentModel

from . import utils

test_cass = CassandraProcessor()
# test_es = ElasticsearchProcessor()

test_harvester = utils.TestHarvester()

NORMALIZED = NormalizedDocument(utils.RECORD)
RAW = test_harvester.harvest()[0]

@pytest.fixture
def harvester():
    pass  # Need to override this

@pytest.mark.django_db
@pytest.mark.cassandra
def test_rename():
    real_es = scrapi.processing.elasticsearch.es
    scrapi.processing.elasticsearch.es = mock.MagicMock()
    test_cass.process_raw(RAW)
    test_cass.process_normalized(RAW, NORMALIZED)

    queryset = DocumentModel.objects(docID=RAW['docID'], source=RAW['source'])
    old_source = NORMALIZED['shareProperties']['source']

    assert queryset[0].source == utils.RECORD['shareProperties']['source']
    assert queryset[0].source == old_source

    new_record = copy.deepcopy(utils.RECORD)

    new_record['shareProperties']['source'] = 'wwe_news'

    test_harvester.short_name = 'wwe_news'

    registry['wwe_news'] = test_harvester

    tasks.migrate(rename, sources=[old_source], target='wwe_news', dry=False)

    queryset = DocumentModel.objects(docID=RAW['docID'], source='wwe_news')
    assert queryset[0].source == 'wwe_news'
    assert len(queryset) == 1
    scrapi.processing.elasticsearch.es = real_es
    test_harvester.short_name = RAW['source']
    registry['test'] = test_harvester
    del registry['wwe_news']

@pytest.mark.django_db
@pytest.mark.cassandra
def test_delete():
    real_es = scrapi.processing.elasticsearch.es
    scrapi.processing.elasticsearch.es = mock.MagicMock()
    test_cass.process_raw(RAW)
    test_cass.process_normalized(RAW, NORMALIZED)

    queryset = DocumentModel.objects(docID=RAW['docID'], source=RAW['source'])
    assert len(queryset) == 1

    tasks.migrate(delete, sources=[RAW['source']], dry=False)
    queryset = DocumentModel.objects(docID=RAW['docID'], source=RAW['source'])
    assert len(queryset) == 0
    scrapi.processing.elasticsearch.es = real_es

@pytest.mark.django_db
@pytest.mark.cassandra
def test_renormalize():
    # Set up
    real_es = scrapi.processing.elasticsearch.es
    scrapi.processing.elasticsearch.es = mock.MagicMock()

    # Process raw and normalized with fake docs
    test_cass.process_raw(RAW)
    test_cass.process_normalized(RAW, NORMALIZED)

    # Check to see those docs were processed
    queryset = DocumentModel.objects(docID=RAW['docID'], source=RAW['source'])
    assert len(queryset) == 1

    # Create a new doucment to be renormalized
    new_raw = copy.deepcopy(RAW)
    new_raw.attributes['docID'] = 'get_the_tables'
    new_raw.attributes['doc'] = new_raw.attributes['doc'].encode('utf-8')

    # This is basically like running the improved harvester right?
    DocumentModel.create(**new_raw.attributes).save()

    tasks.migrate(renormalize, sources=[RAW['source']], dry=False)

    queryset = DocumentModel.objects(docID='get_the_tables', source=RAW['source'])
    assert len(queryset) == 1
    scrapi.processing.elasticsearch.es = real_es

@pytest.mark.django_db
@pytest.mark.cassandra
def test_migrate_v2():
    real_es = scrapi.processing.elasticsearch.es
    scrapi.processing.elasticsearch.es = mock.MagicMock()

    try:
        RAW['doc'] = RAW['doc'].encode('utf-8')
    except AttributeError:
        RAW['doc'] = str(RAW['doc'])
    DocumentModelOld.create(**RAW.attributes).save()
    queryset = DocumentModel.objects(docID=RAW['docID'], source=RAW['source'])
    assert len(queryset) == 0
    tasks.migrate_to_source_partition(dry=False)
    queryset = DocumentModel.objects(docID=RAW['docID'], source=RAW['source'])
    assert len(queryset) == 1
    scrapi.processing.elasticsearch.es = real_es

@pytest.mark.django_db
@pytest.mark.cassandra
def test_cassandra_to_postgres():
    real_es = scrapi.processing.elasticsearch.es
    scrapi.processing.elasticsearch.es = mock.MagicMock()

    test_cass.process_raw(RAW)
    test_cass.process_normalized(RAW, NORMALIZED)

    cassandra_queryset = DocumentModel.objects(docID=RAW['docID'], source=RAW['source'])
    postgres_queryset = Document.objects.filter(source=RAW['source'], docID=RAW['docID'])
    assert len(cassandra_queryset) == 1
    assert len(postgres_queryset) == 0
    tasks.migrate(cross_db, target_db='postgres', dry=False, sources=['test'])

    postgres_queryset = Document.objects.filter(source=RAW['source'], docID=RAW['docID'])
    assert len(postgres_queryset) == 1
    scrapi.processing.elasticsearch.es = real_es

@pytest.mark.cassandra
@pytest.mark.elasticsearch
def test_cassandra_to_elasticsearch():

    # This is done to let the test index finish being created before connecting to search
    while True:
        try:
            results = scrapi.processing.elasticsearch.es.search(index='test', doc_type=RAW['source'])
            break
        except TransportError:
            continue

    assert (len(results['hits']['hits']) == 0)

    # create the cassandra documents
    test_cass.process_raw(RAW)
    test_cass.process_normalized(RAW, NORMALIZED)

    # run the migration
    tasks.migrate(cross_db, target_db='elasticsearch', dry=False, sources=['test'], index='test')

    # check the elasticsearch results
    results = scrapi.processing.elasticsearch.es.search(index='test', doc_type=RAW['source'])
    # import ipdb; ipdb.set_trace()
    assert (len(results['hits']['hits']) == 1)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

#   Copyright (c) 2010-2016, MIT Probabilistic Computing Project
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

import pytest

from bayeslite import BQLError
from bayeslite import bayesdb_open
from bayeslite import bayesdb_register_metamodel
from bayeslite.metamodels.nig_normal import NIGNormalMetamodel

def test_nig_normal_smoke():
    with bayesdb_open(':memory:') as bdb:
        bayesdb_register_metamodel(bdb, NIGNormalMetamodel())
        bdb.sql_execute('create table t(x)')
        for x in xrange(100):
            bdb.sql_execute('insert into t(x) values(?)', (x,))
        bdb.execute('create population p for t(x numerical)')
        bdb.execute('create generator g for p using nig_normal')
        bdb.execute('initialize 1 model for g')
        bdb.execute('analyze g for 1 iteration wait')
        bdb.execute('estimate probability of x = 50 from p').fetchall()
        bdb.execute('simulate x from p limit 1').fetchall()
        bdb.execute('drop models from g')
        bdb.execute('drop generator g')
        bdb.execute('drop population p')
        bdb.execute('drop table t')

def test_nig_normal_latent_smoke():
    with bayesdb_open(':memory:') as bdb:
        bayesdb_register_metamodel(bdb, NIGNormalMetamodel())
        bdb.sql_execute('create table t(x)')
        for x in xrange(100):
            bdb.sql_execute('insert into t(x) values(?)', (x,))
        bdb.execute('create population p for t(x numerical)')
        bdb.execute('create generator g0 for p using nig_normal')
        bdb.execute('''
            create generator g1 for p using nig_normal(xe deviation(x))
        ''')
        bdb.execute('initialize 1 model for g0')
        bdb.execute('analyze g0 for 1 iteration wait')
        bdb.execute('initialize 1 model for g1')
        bdb.execute('analyze g1 for 1 iteration wait')

        # PROBABILITY OF x = v
        bdb.execute('estimate probability of x = 50 within p').fetchall()
        with pytest.raises(BQLError):
            bdb.execute('estimate probability of xe = 1 within p').fetchall()
        with pytest.raises(BQLError):
            bdb.execute('''
                estimate probability of xe = 1 within p modelled by g0
            ''').fetchall()
        bdb.execute('''
            estimate probability of xe = 1 within p modelled by g1
        ''').fetchall()

        # PREDICTIVE PROBABILITY OF x
        bdb.execute('estimate predictive probability of x from p').fetchall()
        with pytest.raises(ValueError):
            bdb.execute(
                'estimate predictive probability of xe from p').fetchall()
        with pytest.raises(ValueError):
            bdb.execute('''
                estimate predictive probability of xe from p modelled by g0
            ''').fetchall()
        for r, p_xe in bdb.execute('''
            estimate rowid, predictive probability of xe from p modelled by g1
        '''):
            assert p_xe is None, 'rowid %r p(xe) %r' % (r, p_xe)

        # SIMULATE x
        bdb.execute('simulate x from p limit 1').fetchall()
        with pytest.raises(BQLError):
            bdb.execute('simulate x, xe from p limit 1').fetchall()
        with pytest.raises(BQLError):
            bdb.execute(
                'simulate x, xe from p modelled by g0 limit 1').fetchall()
        bdb.execute('simulate x, xe from p modelled by g1 limit 1').fetchall()

        assert 100*100 == \
            len(bdb.execute('estimate similarity from pairwise p').fetchall())
        assert 1 == len(bdb.execute('''
            estimate similarity from pairwise p modelled by g0 limit 1
        ''').fetchall())
        assert 1 == len(bdb.execute('''
            estimate similarity with respect to (x)
                from pairwise p modelled by g0 limit 1
        ''').fetchall())
        with pytest.raises(BQLError):
            assert 1 == len(bdb.execute('''
                estimate similarity with respect to (xe)
                    from pairwise p modelled by g0 limit 1
            ''').fetchall())
        with pytest.raises(BQLError):
            assert 1 == len(bdb.execute('''
                estimate similarity with respect to (x, xe)
                    from pairwise p modelled by g0 limit 1
            ''').fetchall())
        assert 1 == len(bdb.execute('''
            estimate similarity from pairwise p modelled by g1 limit 1
        ''').fetchall())
        assert 1 == len(bdb.execute('''
            estimate similarity with respect to (xe)
                from pairwise p modelled by g1 limit 1
        ''').fetchall())
        assert 1 == len(bdb.execute('''
            estimate similarity with respect to (x, xe)
                from pairwise p modelled by g1 limit 1
        ''').fetchall())

        bdb.execute('drop models from g0')
        bdb.execute('drop generator g0')
        bdb.execute('drop models from g1')
        bdb.execute('drop generator g1')
        bdb.execute('drop population p')
        bdb.execute('drop table t')

def test_nig_normal_latent_2var_smoke():
    with bayesdb_open(':memory:') as bdb:
        bayesdb_register_metamodel(bdb, NIGNormalMetamodel())
        bdb.sql_execute('create table t(x, y)')
        for x in xrange(100):
            bdb.sql_execute('insert into t(x, y) values(?, ?)',
                (x, x*x - 100))
        bdb.execute('create population p for t(x numerical, y numerical)')

        # CORRELATION, CORRELATION PVALUE, without generators.
        assert 4 == len(bdb.execute('''
            estimate correlation, correlation pvalue
                from pairwise variables of p
        ''').fetchall())

        bdb.execute('create generator g0 for p using nig_normal')
        bdb.execute('''
            create generator g1 for p using nig_normal(xe deviation(x))
        ''')
        bdb.execute('initialize 1 model for g0')
        bdb.execute('analyze g0 for 1 iteration wait')
        bdb.execute('initialize 1 model for g1')
        bdb.execute('analyze g1 for 1 iteration wait')

        # CORRELATION, CORRELATION PVALUE, with generators.
        assert 4 == len(bdb.execute('''
            estimate correlation, correlation pvalue
                from pairwise variables of p
        ''').fetchall())
        assert 4 == len(bdb.execute('''
            estimate correlation, correlation pvalue
                from pairwise variables of p modelled by g0
        ''').fetchall())
        with pytest.raises(BQLError):
            assert 4 == len(bdb.execute('''
                estimate correlation, correlation pvalue
                    from pairwise variables of p modelled by g1
            ''').fetchall())

        # DEPENDENCE PROBABILITY, MUTUAL INFORMATION
        assert 4 == len(bdb.execute('''
            estimate dependence probability, mutual information
                from pairwise variables of p
        ''').fetchall())
        assert 4 == len(bdb.execute('''
            estimate dependence probability, mutual information
                from pairwise variables of p modelled by g0
        ''').fetchall())
        assert 9 == len(bdb.execute('''
            estimate 1
                from pairwise variables of p modelled by g1
        ''').fetchall())

        # SIMULATE LATENT VARIABLE
        assert 10 == len(bdb.execute('''
            simulate xe from p modeled by g1 limit 10;
        ''').fetchall())
        assert 10 == len(bdb.execute('''
            simulate y, xe from p modeled by g1 limit 10;
        ''').fetchall())
        # Cannot simulate the latent xe from the population p.
        with pytest.raises(BQLError):
            assert 10 == len(bdb.execute('''
                simulate xe from p limit 10;
            ''').fetchall())
        # Cannot simulate the latent xe from the generator g0.
        with pytest.raises(BQLError):
            assert 10 == len(bdb.execute('''
                simulate xe from p modeled by g0 limit 10;
            ''').fetchall())

        bdb.execute('drop models from g0')
        bdb.execute('drop generator g0')
        bdb.execute('drop models from g1')
        bdb.execute('drop generator g1')
        bdb.execute('drop population p')
        bdb.execute('drop table t')
<commit_msg>Test some more conditionals involving latents.<commit_after># -*- coding: utf-8 -*-

#   Copyright (c) 2010-2016, MIT Probabilistic Computing Project
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

import pytest

from bayeslite import BQLError
from bayeslite import bayesdb_open
from bayeslite import bayesdb_register_metamodel
from bayeslite.metamodels.nig_normal import NIGNormalMetamodel

def test_nig_normal_smoke():
    with bayesdb_open(':memory:') as bdb:
        bayesdb_register_metamodel(bdb, NIGNormalMetamodel())
        bdb.sql_execute('create table t(x)')
        for x in xrange(100):
            bdb.sql_execute('insert into t(x) values(?)', (x,))
        bdb.execute('create population p for t(x numerical)')
        bdb.execute('create generator g for p using nig_normal')
        bdb.execute('initialize 1 model for g')
        bdb.execute('analyze g for 1 iteration wait')
        bdb.execute('estimate probability of x = 50 from p').fetchall()
        bdb.execute('simulate x from p limit 1').fetchall()
        bdb.execute('drop models from g')
        bdb.execute('drop generator g')
        bdb.execute('drop population p')
        bdb.execute('drop table t')

def test_nig_normal_latent_smoke():
    with bayesdb_open(':memory:') as bdb:
        bayesdb_register_metamodel(bdb, NIGNormalMetamodel())
        bdb.sql_execute('create table t(x)')
        for x in xrange(100):
            bdb.sql_execute('insert into t(x) values(?)', (x,))
        bdb.execute('create population p for t(x numerical)')
        bdb.execute('create generator g0 for p using nig_normal')
        bdb.execute('''
            create generator g1 for p using nig_normal(xe deviation(x))
        ''')
        bdb.execute('initialize 1 model for g0')
        bdb.execute('analyze g0 for 1 iteration wait')
        bdb.execute('initialize 1 model for g1')
        bdb.execute('analyze g1 for 1 iteration wait')

        # PROBABILITY OF x = v
        bdb.execute('estimate probability of x = 50 within p').fetchall()
        with pytest.raises(BQLError):
            bdb.execute('estimate probability of xe = 1 within p').fetchall()
        with pytest.raises(BQLError):
            bdb.execute('''
                estimate probability of xe = 1 within p modelled by g0
            ''').fetchall()
        bdb.execute('''
            estimate probability of xe = 1 within p modelled by g1
        ''').fetchall()

        # PREDICTIVE PROBABILITY OF x
        bdb.execute('estimate predictive probability of x from p').fetchall()
        with pytest.raises(ValueError):
            bdb.execute(
                'estimate predictive probability of xe from p').fetchall()
        with pytest.raises(ValueError):
            bdb.execute('''
                estimate predictive probability of xe from p modelled by g0
            ''').fetchall()
        for r, p_xe in bdb.execute('''
            estimate rowid, predictive probability of xe from p modelled by g1
        '''):
            assert p_xe is None, 'rowid %r p(xe) %r' % (r, p_xe)

        # SIMULATE x
        bdb.execute('simulate x from p limit 1').fetchall()
        with pytest.raises(BQLError):
            bdb.execute('simulate x, xe from p limit 1').fetchall()
        with pytest.raises(BQLError):
            bdb.execute(
                'simulate x, xe from p modelled by g0 limit 1').fetchall()
        bdb.execute('simulate x, xe from p modelled by g1 limit 1').fetchall()

        assert 100*100 == \
            len(bdb.execute('estimate similarity from pairwise p').fetchall())
        assert 1 == len(bdb.execute('''
            estimate similarity from pairwise p modelled by g0 limit 1
        ''').fetchall())
        assert 1 == len(bdb.execute('''
            estimate similarity with respect to (x)
                from pairwise p modelled by g0 limit 1
        ''').fetchall())
        with pytest.raises(BQLError):
            assert 1 == len(bdb.execute('''
                estimate similarity with respect to (xe)
                    from pairwise p modelled by g0 limit 1
            ''').fetchall())
        with pytest.raises(BQLError):
            assert 1 == len(bdb.execute('''
                estimate similarity with respect to (x, xe)
                    from pairwise p modelled by g0 limit 1
            ''').fetchall())
        assert 1 == len(bdb.execute('''
            estimate similarity from pairwise p modelled by g1 limit 1
        ''').fetchall())
        assert 1 == len(bdb.execute('''
            estimate similarity with respect to (xe)
                from pairwise p modelled by g1 limit 1
        ''').fetchall())
        assert 1 == len(bdb.execute('''
            estimate similarity with respect to (x, xe)
                from pairwise p modelled by g1 limit 1
        ''').fetchall())

        bdb.execute('drop models from g0')
        bdb.execute('drop generator g0')
        bdb.execute('drop models from g1')
        bdb.execute('drop generator g1')
        bdb.execute('drop population p')
        bdb.execute('drop table t')

def test_nig_normal_latent_conditional_smoke():
    with bayesdb_open(':memory:') as bdb:
        bayesdb_register_metamodel(bdb, NIGNormalMetamodel())
        bdb.sql_execute('create table t(x)')
        for x in xrange(100):
            bdb.sql_execute('insert into t(x) values(?)', (x,))
        bdb.execute('create population p for t(x numerical)')
        bdb.execute('create generator g0 for p using nig_normal')
        bdb.execute('''
            create generator g1 for p using nig_normal(xe deviation(x))
        ''')
        bdb.execute('initialize 1 model for g0')
        bdb.execute('analyze g0 for 1 iteration wait')
        bdb.execute('initialize 1 model for g1')
        bdb.execute('analyze g1 for 1 iteration wait')

        # observed given observed
        bdb.execute('''
            estimate probability of x = 50 given (x = 50) within p
        ''').fetchall()
        bdb.execute('''
            estimate probability of x = 50 given (x = 50) within p
                modelled by g0
        ''').fetchall()
        bdb.execute('''
            estimate probability of x = 50 given (x = 50) within p
                modelled by g1
        ''').fetchall()

        # observed given latent
        with pytest.raises(BQLError):
            bdb.execute('''
                estimate probability of x = 50 given (xe = 50) within p
            ''').fetchall()
        with pytest.raises(BQLError):
            bdb.execute('''
                estimate probability of x = 50 given (xe = 50) within p
                    modelled by g0
            ''').fetchall()
        bdb.execute('''
            estimate probability of x = 50 given (xe = 50) within p
                modelled by g1
        ''').fetchall()

        # latent given observed
        with pytest.raises(BQLError):
            bdb.execute('''
                estimate probability of xe = 50 given (x = 50) within p
            ''').fetchall()
        with pytest.raises(BQLError):
            bdb.execute('''
                estimate probability of xe = 50 given (x = 50) within p
                    modelled by g0
            ''').fetchall()
        bdb.execute('''
            estimate probability of xe = 50 given (x = 50) within p
                modelled by g1
        ''').fetchall()

        bdb.execute('drop models from g0')
        bdb.execute('drop generator g0')
        bdb.execute('drop models from g1')
        bdb.execute('drop generator g1')
        bdb.execute('drop population p')
        bdb.execute('drop table t')

def test_nig_normal_latent_2var_smoke():
    with bayesdb_open(':memory:') as bdb:
        bayesdb_register_metamodel(bdb, NIGNormalMetamodel())
        bdb.sql_execute('create table t(x, y)')
        for x in xrange(100):
            bdb.sql_execute('insert into t(x, y) values(?, ?)',
                (x, x*x - 100))
        bdb.execute('create population p for t(x numerical, y numerical)')

        # CORRELATION, CORRELATION PVALUE, without generators.
        assert 4 == len(bdb.execute('''
            estimate correlation, correlation pvalue
                from pairwise variables of p
        ''').fetchall())

        bdb.execute('create generator g0 for p using nig_normal')
        bdb.execute('''
            create generator g1 for p using nig_normal(xe deviation(x))
        ''')
        bdb.execute('initialize 1 model for g0')
        bdb.execute('analyze g0 for 1 iteration wait')
        bdb.execute('initialize 1 model for g1')
        bdb.execute('analyze g1 for 1 iteration wait')

        # CORRELATION, CORRELATION PVALUE, with generators.
        assert 4 == len(bdb.execute('''
            estimate correlation, correlation pvalue
                from pairwise variables of p
        ''').fetchall())
        assert 4 == len(bdb.execute('''
            estimate correlation, correlation pvalue
                from pairwise variables of p modelled by g0
        ''').fetchall())
        with pytest.raises(BQLError):
            assert 4 == len(bdb.execute('''
                estimate correlation, correlation pvalue
                    from pairwise variables of p modelled by g1
            ''').fetchall())

        # DEPENDENCE PROBABILITY, MUTUAL INFORMATION
        assert 4 == len(bdb.execute('''
            estimate dependence probability, mutual information
                from pairwise variables of p
        ''').fetchall())
        assert 4 == len(bdb.execute('''
            estimate dependence probability, mutual information
                from pairwise variables of p modelled by g0
        ''').fetchall())
        assert 9 == len(bdb.execute('''
            estimate 1
                from pairwise variables of p modelled by g1
        ''').fetchall())

        # SIMULATE LATENT VARIABLE
        assert 10 == len(bdb.execute('''
            simulate xe from p modeled by g1 limit 10;
        ''').fetchall())
        assert 10 == len(bdb.execute('''
            simulate y, xe from p modeled by g1 limit 10;
        ''').fetchall())
        # Cannot simulate the latent xe from the population p.
        with pytest.raises(BQLError):
            assert 10 == len(bdb.execute('''
                simulate xe from p limit 10;
            ''').fetchall())
        # Cannot simulate the latent xe from the generator g0.
        with pytest.raises(BQLError):
            assert 10 == len(bdb.execute('''
                simulate xe from p modeled by g0 limit 10;
            ''').fetchall())

        bdb.execute('drop models from g0')
        bdb.execute('drop generator g0')
        bdb.execute('drop models from g1')
        bdb.execute('drop generator g1')
        bdb.execute('drop population p')
        bdb.execute('drop table t')

def test_nig_normal_latent_2var_conditional_smoke():
    with bayesdb_open(':memory:') as bdb:
        bayesdb_register_metamodel(bdb, NIGNormalMetamodel())
        bdb.sql_execute('create table t(x, y)')
        for x in xrange(100):
            bdb.sql_execute('insert into t(x, y) values(?, ?)',
                (x, x*x - 100))
        bdb.execute('create population p for t(x numerical, y numerical)')

        # CORRELATION, CORRELATION PVALUE, without generators.
        assert 4 == len(bdb.execute('''
            estimate correlation, correlation pvalue
                from pairwise variables of p
        ''').fetchall())

        bdb.execute('create generator g0 for p using nig_normal')
        bdb.execute('''
            create generator g1 for p using nig_normal(xe deviation(x))
        ''')
        bdb.execute('initialize 1 model for g0')
        bdb.execute('analyze g0 for 1 iteration wait')
        bdb.execute('initialize 1 model for g1')
        bdb.execute('analyze g1 for 1 iteration wait')

        # observed given other observed
        bdb.execute('''
            estimate probability of x = 50 given (y = 49) within p
        ''').fetchall()
        bdb.execute('''
            estimate probability of x = 50 given (y = 49) within p
                modelled by g0
        ''').fetchall()
        bdb.execute('''
            estimate probability of x = 50 given (y = 49) within p
                modelled by g1
        ''').fetchall()
        bdb.execute('simulate x from p given y = 49 limit 1').fetchall()
        bdb.execute('''
            simulate x from p modelled by g0 given y = 49 limit 1
        ''').fetchall()
        bdb.execute('''
            simulate x from p modelled by g1 given y = 49 limit 1
        ''').fetchall()

        # observed given related latent
        with pytest.raises(BQLError):
            bdb.execute('''
                estimate probability of x = 50 given (xe = 1) within p
            ''').fetchall()
        with pytest.raises(BQLError):
            bdb.execute('''
                estimate probability of x = 50 given (xe = 1) within p
                    modelled by g0
            ''').fetchall()
        bdb.execute('''
            estimate probability of x = 50 given (xe = 1) within p
                modelled by g1
        ''').fetchall()
        with pytest.raises(BQLError):
            bdb.execute('simulate x from p given xe = 1 limit 1').fetchall()
        with pytest.raises(BQLError):
            bdb.execute('''
                simulate x from p modelled by g0 given xe = 1 limit 1
            ''').fetchall()
        bdb.execute('''
            simulate x from p modelled by g1 given xe = 1 limit 1
        ''').fetchall()

        # observed given unrelated latent
        with pytest.raises(BQLError):
            bdb.execute('''
                estimate probability of y = 50 given (xe = 1) within p
            ''').fetchall()
        with pytest.raises(BQLError):
            bdb.execute('''
                estimate probability of y = 50 given (xe = 1) within p
                    modelled by g0
            ''').fetchall()
        bdb.execute('''
            estimate probability of y = 50 given (xe = 1) within p
                modelled by g1
        ''').fetchall()
        with pytest.raises(BQLError):
            bdb.execute('simulate y from p given xe = 1 limit 1').fetchall()
        with pytest.raises(BQLError):
            bdb.execute('''
                simulate y from p modelled by g0 given xe = 1 limit 1
            ''').fetchall()
        bdb.execute('''
            simulate y from p modelled by g1 given xe = 1 limit 1
        ''').fetchall()

        # latent given related observed
        with pytest.raises(BQLError):
            bdb.execute('''
                estimate probability of xe = 1 given (x = 50) within p
            ''').fetchall()
        with pytest.raises(BQLError):
            bdb.execute('''
                estimate probability of xe = 1 given (x = 50) within p
                    modelled by g0
            ''').fetchall()
        bdb.execute('''
            estimate probability of xe = 1 given (x = 50) within p
                modelled by g1
        ''').fetchall()
        with pytest.raises(BQLError):
            bdb.execute('simulate xe from p given x = 50 limit 1').fetchall()
        with pytest.raises(BQLError):
            bdb.execute('''
                simulate xe from p modelled by g0 given x = 50 limit 1
            ''').fetchall()
        bdb.execute('''
            simulate xe from p modelled by g1 given x = 50 limit 1
        ''').fetchall()

        # latent given unrelated observed
        with pytest.raises(BQLError):
            bdb.execute('''
                estimate probability of xe = 1 given (y = 50) within p
            ''').fetchall()
        with pytest.raises(BQLError):
            bdb.execute('''
                estimate probability of xe = 1 given (y = 50) within p
                    modelled by g0
            ''').fetchall()
        bdb.execute('''
            estimate probability of xe = 1 given (y = 50) within p
                modelled by g1
        ''').fetchall()
        with pytest.raises(BQLError):
            bdb.execute('simulate xe from p given y = 50 limit 1').fetchall()
        with pytest.raises(BQLError):
            bdb.execute('''
                simulate xe from p modelled by g0 given y = 50 limit 1
            ''').fetchall()
        bdb.execute('''
            simulate xe from p modelled by g1 given y = 50 limit 1
        ''').fetchall()

        bdb.execute('drop models from g0')
        bdb.execute('drop generator g0')
        bdb.execute('drop models from g1')
        bdb.execute('drop generator g1')
        bdb.execute('drop population p')
        bdb.execute('drop table t')
<|endoftext|>


<commit_before>#!/usr/bin/python
#coding: utf8

from argparse import ArgumentParser
from ast import literal_eval
from os import popen
from re import sub
from string import capitalize
from sys import argv, exit
from urllib import urlencode
from urllib2 import urlopen, URLError

def get_data(query, src_lan='en', tgt_lan='en', quiet=False):
    '''Return data in JSON from Google.'''
    # TODO This IP might not be static - add more IPs in case of errors.
    base_url = 'http://173.194.35.144/dictionary/json?callback=c&'
    options = urlencode({'q': query, 'sl': src_lan, 'tl': tgt_lan})
    try:
        return urlopen(base_url + options).read().decode('utf8')
    except URLError:
        if not quiet:
            print 'Error - could not connect to Google.com'
            print 'Checking internet connectivity.'
            if connected():
                print 'You appear to be connected to the internet. Try again.'
            else:
                print 'You appear to be disconnected from the internet.'
        exit()

def parse(data, query, quiet=False):
    '''Return a dictionary containing definitions and their categories parsed
    from data in JSON.'''
    try:
        dicts = literal_eval(data[2:-10])['primaries']
        categories = []
        definitions = []
        for i, _ in enumerate(dicts):
            category = dicts[i]['terms'][0]['labels'][0]['text']
            categories.append(category)
            definition = dicts[i]['entries']
            temp_defs = []
            for i, _ in enumerate(definition):
                if definition[i].has_key('type'):
                    if definition[i]['type'] == 'meaning':
                        temp_defs.append(definition[i]['terms'][0]['text'])
            definitions.append(temp_defs)
    except KeyError:
        if not quiet:
            print 'No definition found for "%s".' % query
        exit()
    return zip(categories, definitions)

def format_output(input, limit):
    '''Parse and print input list in a specific manner. Only return limit
    number of definitions.'''
    # cols = int(popen('stty size', 'r').read().split()[1])
    # NOTE Adjust text if longer than one line (number of columns) so that the
    # output is more readable.
    print '    %s' % capitalize(input[0])
    for category in input[1]:
        print category[0]
        for definition in category[1][:limit]:
            print '  • %s.' % untag(definition)

def untag(string):
    '''Remove XML tags from string.'''
    return sub('<[^<]+?>', '', string).strip()

def connected():
    '''Return True if connected to the internet, otherwise return False.'''
    try:
        urlopen('http://74.125.113.99', timeout=1)
        return True
    except URLError:
        return False

if __name__ == '__main__':
    parser = ArgumentParser()
    # TODO Fix indentation problems withouth metavar='' and add a long option.
    parser.add_argument('-sl', metavar='', action='append',
                        help='specify a source language, e.g. en')
    parser.add_argument('-tl', metavar='', action='append',
                        help='specify a target language, e.g. en')
    parser.add_argument('-n', metavar='', type=int, help='specify the number '
                        + 'of definitions to print for each word category')
    # TODO Add an option to print the definitions for each word in a file.
    parser.add_argument('-q', '--quiet', action='store_true',
                        help='do not print error messages')
    parser.add_argument('query', help='look up the meaning of the word or '
                        + 'concept; requires quotes if longer than one word')
    args = parser.parse_args()
    # TODO Only English seems to work at the moment - check later. Remove
    # language options if not relevant.
    langs = ['en', 'en']
    if args.sl != None:
        langs[0] = args.sl[0]
    if args.tl != None:
        langs[1] = args.tl[0]
    if args.n == None:
        # Show us 3 definitions of each category by default.
        args.n = 3
    try:
        # TODO Make this more readable.
        format_output([args.query, parse(get_data(args.query, langs[0],
                       langs[1], args.quiet), args.query, args.quiet)],
                      args.n)
    except KeyboardInterrupt:
        # Print a newline to start the prompt on an empty line and exit.
        # TODO Do not print "^C".
        print
        exit(130)
<commit_msg>exit codes<commit_after>#!/usr/bin/python
#coding: utf8

from argparse import ArgumentParser
from ast import literal_eval
from os import popen
from re import sub
from string import capitalize
from sys import argv, exit
from urllib import urlencode
from urllib2 import urlopen, URLError

def get_data(query, src_lan='en', tgt_lan='en', quiet=False):
    '''Return data in JSON from Google.'''
    # TODO This IP might not be static - add more IPs in case of errors.
    base_url = 'http://173.194.35.144/dictionary/json?callback=c&'
    options = urlencode({'q': query, 'sl': src_lan, 'tl': tgt_lan})
    try:
        return urlopen(base_url + options).read().decode('utf8')
    except URLError:
        if not quiet:
            print 'Error - could not connect to Google.com'
            print 'Checking internet connectivity.'
            if connected():
                print 'You appear to be connected to the internet. Try again.'
            else:
                print 'You appear to be disconnected from the internet.'
        exit(1)

def parse(data, query, quiet=False):
    '''Return a dictionary containing definitions and their categories parsed
    from data in JSON.'''
    try:
        dicts = literal_eval(data[2:-10])['primaries']
        categories = []
        definitions = []
        for i, _ in enumerate(dicts):
            category = dicts[i]['terms'][0]['labels'][0]['text']
            categories.append(category)
            definition = dicts[i]['entries']
            temp_defs = []
            for i, _ in enumerate(definition):
                if definition[i].has_key('type'):
                    if definition[i]['type'] == 'meaning':
                        temp_defs.append(definition[i]['terms'][0]['text'])
            definitions.append(temp_defs)
    except KeyError:
        if not quiet:
            print 'No definition found for "%s".' % query
        exit(1)
    return zip(categories, definitions)

def format_output(input, limit):
    '''Parse and print input list in a specific manner. Only return limit
    number of definitions.'''
    # cols = int(popen('stty size', 'r').read().split()[1])
    # NOTE Adjust text if longer than one line (number of columns) so that the
    # output is more readable.
    print '    %s' % capitalize(input[0])
    for category in input[1]:
        print category[0]
        for definition in category[1][:limit]:
            print '  • %s.' % untag(definition)

def untag(string):
    '''Remove XML tags from string.'''
    return sub('<[^<]+?>', '', string).strip()

def connected():
    '''Return True if connected to the internet, otherwise return False.'''
    try:
        urlopen('http://74.125.113.99', timeout=1)
        return True
    except URLError:
        return False

if __name__ == '__main__':
    parser = ArgumentParser()
    # TODO Fix indentation problems withouth metavar='' and add a long option.
    parser.add_argument('-sl', metavar='', action='append',
                        help='specify a source language, e.g. en')
    parser.add_argument('-tl', metavar='', action='append',
                        help='specify a target language, e.g. en')
    parser.add_argument('-n', metavar='', type=int, help='specify the number '
                        + 'of definitions to print for each word category')
    # TODO Add an option to print the definitions for each word in a file.
    parser.add_argument('-q', '--quiet', action='store_true',
                        help='do not print error messages')
    parser.add_argument('query', help='look up the meaning of the word or '
                        + 'concept; requires quotes if longer than one word')
    args = parser.parse_args()
    # TODO Only English seems to work at the moment - check later. Remove
    # language options if not relevant.
    langs = ['en', 'en']
    if args.sl != None:
        langs[0] = args.sl[0]
    if args.tl != None:
        langs[1] = args.tl[0]
    if args.n == None:
        # Show us 3 definitions of each category by default.
        args.n = 3
    try:
        # TODO Make this more readable.
        format_output([args.query, parse(get_data(args.query, langs[0],
                       langs[1], args.quiet), args.query, args.quiet)],
                      args.n)
    except KeyboardInterrupt:
        # Print a newline to start the prompt on an empty line and exit.
        # TODO Do not print "^C".
        print
        exit(130)
<|endoftext|>


<commit_before>from __future__ import division, print_function
import unittest
import warnings
import pickle

import numpy as np

from symfit import (
    Variable, Parameter, Eq, Ge, Le, Lt, Gt, Ne, parameters, ModelError, Fit,
    Model, FitResults, variables, CallableNumericalModel, Idx,
    IndexedBase, symbols, Sum, log, exp, cos, pi, besseli
)
from symfit.core.objectives import (
    VectorLeastSquares, LeastSquares, LogLikelihood, MinimizeModel,
    BaseIndependentObjective
)
from symfit.core.fit_results import FitResults
from symfit.distributions import Exp

# Overwrite the way Sum is printed by numpy just while testing. Is not
# general enough to be moved to symfit.core.printing, but has to be used
# in this test. This way of summing completely ignores the summation indices and
# the dimensions, and instead just flattens everything to a scalar. Only used
# in this test to build the analytical equivalents of our LeastSquares
# and LogLikelihood
class FlattenSum(Sum):
    """
    Just a sum which is printed differently: by flattening the whole array and
    summing it. Used in tests only.
    """
    def _numpycode(self, printer):
        return "%s(%s)" % (printer._module_format('numpy.sum'),
                           printer.doprint(self.function))

class TestObjectives(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        np.random.seed(0)

    def test_pickle(self):
        """
        Test the picklability of the built-in objectives.
        """
        # Create test data
        xdata = np.linspace(0, 100, 100)  # From 0 to 100 in 100 steps
        a_vec = np.random.normal(15.0, scale=2.0, size=xdata.shape)
        b_vec = np.random.normal(100, scale=2.0, size=xdata.shape)
        ydata = a_vec * xdata + b_vec  # Point scattered around the line 15 * x + 100

        # Normal symbolic fit
        a = Parameter('a', value=0, min=0.0, max=1000)
        b = Parameter('b', value=0, min=0.0, max=1000)
        x, y = variables('x, y')
        model = Model({y: a * x + b})

        for objective in [VectorLeastSquares, LeastSquares, LogLikelihood, MinimizeModel]:
            if issubclass(objective, BaseIndependentObjective):
                data = {x: xdata}
            else:
                data = {x: xdata, y: ydata,
                        model.sigmas[y]: np.ones_like(ydata)}
            obj = objective(model, data=data)
            new_obj = pickle.loads(pickle.dumps(obj))
            self.assertTrue(FitResults._array_safe_dict_eq(obj.__dict__,
                                                           new_obj.__dict__))

    def test_LeastSquares(self):
        """
        Tests if the LeastSquares objective gives the right shapes of output by
        comparing with its analytical equivalent.
        """
        i = Idx('i', 100)
        x, y = symbols('x, y', cls=Variable)
        X2 = symbols('X2', cls=Variable)
        a, b = parameters('a, b')

        model = Model({y: a * x**2 + b * x})
        xdata = np.linspace(0, 10, 100)
        ydata = model(x=xdata, a=5, b=2).y + np.random.normal(0, 5, xdata.shape)

        # Construct a LeastSquares objective and its analytical equivalent
        chi2_numerical = LeastSquares(model, data={
            x: xdata, y: ydata, model.sigmas[y]: np.ones_like(xdata)
        })
        chi2_exact = Model(
            {X2: FlattenSum(0.5 * ((a * x ** 2 + b * x) - y) ** 2, i)})

        eval_exact = chi2_exact(x=xdata, y=ydata, a=2, b=3)
        jac_exact = chi2_exact.eval_jacobian(x=xdata, y=ydata, a=2, b=3)
        hess_exact = chi2_exact.eval_hessian(x=xdata, y=ydata, a=2, b=3)
        eval_numerical = chi2_numerical(x=xdata, a=2, b=3)
        jac_numerical = chi2_numerical.eval_jacobian(x=xdata, a=2, b=3)
        hess_numerical = chi2_numerical.eval_hessian(x=xdata, a=2, b=3)

        # Test model jacobian and hessian shape
        self.assertEqual(model(x=xdata, a=2, b=3)[0].shape, ydata.shape)
        self.assertEqual(model.eval_jacobian(x=xdata, a=2, b=3)[0].shape,
                         (2, 100))
        self.assertEqual(model.eval_hessian(x=xdata, a=2, b=3)[0].shape,
                         (2, 2, 100))
        # Test exact chi2 shape
        self.assertEqual(eval_exact[0].shape, (1,))
        self.assertEqual(jac_exact[0].shape, (2, 1))
        self.assertEqual(hess_exact[0].shape, (2, 2, 1))

        # Test if these two models have the same call, jacobian, and hessian
        self.assertAlmostEqual(eval_exact[0], eval_numerical)
        self.assertIsInstance(eval_numerical, float)
        self.assertIsInstance(eval_exact[0][0], float)
        np.testing.assert_almost_equal(np.squeeze(jac_exact[0], axis=-1),
                                       jac_numerical)
        self.assertIsInstance(jac_numerical, np.ndarray)
        np.testing.assert_almost_equal(np.squeeze(hess_exact[0], axis=-1),
                                       hess_numerical)
        self.assertIsInstance(hess_numerical, np.ndarray)

        fit = Fit(chi2_exact, x=xdata, y=ydata, objective=MinimizeModel)
        fit_exact_result = fit.execute()
        fit = Fit(model, x=xdata, y=ydata, absolute_sigma=True)
        fit_num_result = fit.execute()
        self.assertEqual(fit_exact_result.value(a), fit_num_result.value(a))
        self.assertEqual(fit_exact_result.value(b), fit_num_result.value(b))
        self.assertAlmostEqual(fit_exact_result.stdev(a),
                               fit_num_result.stdev(a))
        self.assertAlmostEqual(fit_exact_result.stdev(b),
                               fit_num_result.stdev(b))

    def test_LogLikelihood(self):
        """
        Tests if the LeastSquares objective gives the right shapes of output by
        comparing with its analytical equivalent.
        """
        # TODO: update these tests to use indexed variables in the future
        a, b = parameters('a, b')
        i = Idx('i', 100)
        x, y = variables('x, y')
        pdf = Exp(x, 1 / a) * Exp(x, b)

        np.random.seed(10)
        xdata = np.random.exponential(3.5, 100)

        # We use minus loglikelihood for the model, because the objective was
        # designed to find the maximum when used with a *minimizer*, so it has
        # opposite sign. Also test MinimizeModel at the same time.
        logL_model = Model({y: pdf})
        logL_exact = Model({y: - FlattenSum(log(pdf), i)})
        logL_numerical = LogLikelihood(logL_model, {x: xdata, y: None})
        logL_minmodel = MinimizeModel(logL_exact, data={x: xdata, y: None})

        # Test model jacobian and hessian shape
        eval_exact = logL_exact(x=xdata, a=2, b=3)
        jac_exact = logL_exact.eval_jacobian(x=xdata, a=2, b=3)
        hess_exact = logL_exact.eval_hessian(x=xdata, a=2, b=3)
        eval_minimizemodel = logL_minmodel(a=2, b=3)
        jac_minimizemodel = logL_minmodel.eval_jacobian(a=2, b=3)
        hess_minimizemodel = logL_minmodel.eval_hessian(a=2, b=3)
        eval_numerical = logL_numerical(a=2, b=3)
        jac_numerical = logL_numerical.eval_jacobian(a=2, b=3)
        hess_numerical = logL_numerical.eval_hessian(a=2, b=3)

        # TODO: These shapes should not have the ones! This is due to the current
        # convention that scalars should be returned as a 1d array by Model's.
        self.assertEqual(eval_exact[0].shape, (1,))
        self.assertEqual(jac_exact[0].shape, (2, 1))
        self.assertEqual(hess_exact[0].shape, (2, 2, 1))
        # Test if identical to MinimizeModel
        np.testing.assert_almost_equal(eval_exact[0], eval_minimizemodel)
        np.testing.assert_almost_equal(jac_exact[0], jac_minimizemodel)
        np.testing.assert_almost_equal(hess_exact[0], hess_minimizemodel)

        # Test if these two models have the same call, jacobian, and hessian.
        # Since models always have components as their first dimension, we have
        # to slice that away.
        self.assertAlmostEqual(eval_exact.y, eval_numerical)
        self.assertIsInstance(eval_numerical, float)
        self.assertIsInstance(eval_exact.y[0], float)
        np.testing.assert_almost_equal(np.squeeze(jac_exact[0], axis=-1),
                                       jac_numerical)
        self.assertIsInstance(jac_numerical, np.ndarray)
        np.testing.assert_almost_equal(np.squeeze(hess_exact[0], axis=-1),
                                       hess_numerical)
        self.assertIsInstance(hess_numerical, np.ndarray)

        fit = Fit(logL_exact, x=xdata, objective=MinimizeModel)
        fit_exact_result = fit.execute()
        fit = Fit(logL_model, x=xdata, objective=LogLikelihood)
        fit_num_result = fit.execute()
        self.assertAlmostEqual(fit_exact_result.value(a), fit_num_result.value(a))
        self.assertAlmostEqual(fit_exact_result.value(b), fit_num_result.value(b))
        self.assertAlmostEqual(fit_exact_result.stdev(a), fit_num_result.stdev(a))
        self.assertAlmostEqual(fit_exact_result.stdev(b), fit_num_result.stdev(b))

    def test_data_sanity(self):
        """
        Tests very basicly the data sanity for different objective types.
        :return:
        """
        # Create test data
        xdata = np.linspace(0, 100, 25)  # From 0 to 100 in 100 steps
        a_vec = np.random.normal(15.0, scale=2.0, size=xdata.shape)
        b_vec = np.random.normal(100, scale=2.0, size=xdata.shape)
        ydata = a_vec * xdata + b_vec  # Point scattered around the line 5 * x + 105

        # Normal symbolic fit
        a = Parameter('a', value=0, min=0.0, max=1000)
        b = Parameter('b', value=0, min=0.0, max=1000)
        x, y, z = variables('x, y, z')
        model = Model({y: a * x + b})

        for objective in [VectorLeastSquares, LeastSquares, LogLikelihood,
                          MinimizeModel]:
            if issubclass(objective, BaseIndependentObjective):
                incomplete_data = {}
                data = {x: xdata}
                overcomplete_data = {x: xdata, z: ydata}
            else:
                incomplete_data = {x: xdata, y: ydata}
                data = {x: xdata, y: ydata,
                        model.sigmas[y]: np.ones_like(ydata)}
                overcomplete_data = {x: xdata, y: ydata, z: ydata,
                        model.sigmas[y]: np.ones_like(ydata)}
            with self.assertRaises(KeyError):
                obj = objective(model, data=incomplete_data)
            obj = objective(model, data=data)
            # Overcomplete data has to be allowed, since constraints share their
            # data with models.
            obj = objective(model, data=overcomplete_data)

    def test_LogLikelihood_global(self):
        """
        This is a test for global likelihood fitting to multiple data sets.
        Based on SO question 56006357.
        """
        # creating the data
        mu1, mu2 = .05, -.05
        sigma1, sigma2 = 3.5, 2.5
        n1, n2 = 80, 90
        np.random.seed(42)
        x1 = np.random.vonmises(mu1, sigma1, n1)
        x2 = np.random.vonmises(mu2, sigma2, n2)

        n = 2  # number of components
        xs = variables(
            'x,' + ','.join('x_{}'.format(i) for i in range(1, n + 1)))
        x, xs = xs[0], xs[1:]
        ys = variables(','.join('y_{}'.format(i) for i in range(1, n + 1)))
        mu, kappa = parameters('mu, kappa')
        kappas = parameters(','.join('k_{}'.format(i) for i in range(1, n + 1)),
                            min=0, max=10)
        mu.min, mu.max = - np.pi, np.pi

        template = exp(kappa * cos(x - mu)) / (2 * pi * besseli(0, kappa))

        model = Model(
            {y_i: template.subs({kappa: k_i, x: x_i}) for y_i, x_i, k_i in
             zip(ys, xs, kappas)}
        )

        all_data = {xs[0]: x1, xs[1]: x2, ys[0]: None, ys[1]: None}
        all_params = {'mu': 1}
        all_params.update({k_i.name: 1 for k_i in kappas})

        # Evaluate the loglikelihood and its jacobian and hessian
        logL = LogLikelihood(model, data=all_data)
        eval_numerical = logL(**all_params)
        jac_numerical = logL.eval_jacobian(**all_params)
        hess_numerical = logL.eval_hessian(**all_params)

        # Test the types and shapes of the components.
        self.assertIsInstance(eval_numerical, float)
        self.assertIsInstance(jac_numerical, np.ndarray)
        self.assertIsInstance(hess_numerical, np.ndarray)

        self.assertEqual(eval_numerical.shape, tuple())  # Empty tuple -> scalar
        self.assertEqual(jac_numerical.shape, (3,))
        self.assertEqual(hess_numerical.shape, (3, 3,))

if __name__ == '__main__':
    try:
        unittest.main(warnings='ignore')
        # Note that unittest will catch and handle exceptions raised by tests.
        # So this line will *only* deal with exceptions raised by the line
        # above.
    except TypeError:
        # In Py2, unittest.main doesn't take a warnings argument
        warnings.simplefilter('ignore')
        unittest.main()
<commit_msg>Convert test_objectives<commit_after>from __future__ import division, print_function
import pytest
import warnings
import pickle

import numpy as np

from symfit import (
    Variable, Parameter, Eq, Ge, Le, Lt, Gt, Ne, parameters, ModelError, Fit,
    Model, FitResults, variables, CallableNumericalModel, Idx,
    IndexedBase, symbols, Sum, log, exp, cos, pi, besseli
)
from symfit.core.objectives import (
    VectorLeastSquares, LeastSquares, LogLikelihood, MinimizeModel,
    BaseIndependentObjective
)
from symfit.core.fit_results import FitResults
from symfit.distributions import Exp

# Overwrite the way Sum is printed by numpy just while testing. Is not
# general enough to be moved to symfit.core.printing, but has to be used
# in this test. This way of summing completely ignores the summation indices and
# the dimensions, and instead just flattens everything to a scalar. Only used
# in this test to build the analytical equivalents of our LeastSquares
# and LogLikelihood

class FlattenSum(Sum):
    """
    Just a sum which is printed differently: by flattening the whole array and
    summing it. Used in tests only.
    """

    def _numpycode(self, printer):
        return "%s(%s)" % (printer._module_format('numpy.sum'),
                           printer.doprint(self.function))

def setup_method():
    np.random.seed(0)

def test_pickle():
    """
    Test the picklability of the built-in objectives.
    """
    # Create test data
    xdata = np.linspace(0, 100, 100)  # From 0 to 100 in 100 steps
    a_vec = np.random.normal(15.0, scale=2.0, size=xdata.shape)
    b_vec = np.random.normal(100, scale=2.0, size=xdata.shape)
    ydata = a_vec * xdata + b_vec  # Point scattered around the line 15 * x + 100

    # Normal symbolic fit
    a = Parameter('a', value=0, min=0.0, max=1000)
    b = Parameter('b', value=0, min=0.0, max=1000)
    x, y = variables('x, y')
    model = Model({y: a * x + b})

    for objective in [VectorLeastSquares, LeastSquares, LogLikelihood, MinimizeModel]:
        if issubclass(objective, BaseIndependentObjective):
            data = {x: xdata}
        else:
            data = {x: xdata, y: ydata,
                    model.sigmas[y]: np.ones_like(ydata)}
        obj = objective(model, data=data)
        new_obj = pickle.loads(pickle.dumps(obj))
        assert FitResults._array_safe_dict_eq(obj.__dict__, new_obj.__dict__)

def test_LeastSquares():
    """
    Tests if the LeastSquares objective gives the right shapes of output by
    comparing with its analytical equivalent.
    """
    i = Idx('i', 100)
    x, y = symbols('x, y', cls=Variable)
    X2 = symbols('X2', cls=Variable)
    a, b = parameters('a, b')

    model = Model({y: a * x**2 + b * x})
    xdata = np.linspace(0, 10, 100)
    ydata = model(x=xdata, a=5, b=2).y + np.random.normal(0, 5, xdata.shape)

    # Construct a LeastSquares objective and its analytical equivalent
    chi2_numerical = LeastSquares(model, data={
        x: xdata, y: ydata, model.sigmas[y]: np.ones_like(xdata)
    })
    chi2_exact = Model(
        {X2: FlattenSum(0.5 * ((a * x ** 2 + b * x) - y) ** 2, i)})

    eval_exact = chi2_exact(x=xdata, y=ydata, a=2, b=3)
    jac_exact = chi2_exact.eval_jacobian(x=xdata, y=ydata, a=2, b=3)
    hess_exact = chi2_exact.eval_hessian(x=xdata, y=ydata, a=2, b=3)
    eval_numerical = chi2_numerical(x=xdata, a=2, b=3)
    jac_numerical = chi2_numerical.eval_jacobian(x=xdata, a=2, b=3)
    hess_numerical = chi2_numerical.eval_hessian(x=xdata, a=2, b=3)

    # Test model jacobian and hessian shape
    assert model(x=xdata, a=2, b=3)[0].shape == ydata.shape
    assert model.eval_jacobian(x=xdata, a=2, b=3)[0].shape == (2, 100)
    assert model.eval_hessian(x=xdata, a=2, b=3)[0].shape == (2, 2, 100)
    # Test exact chi2 shape
    assert eval_exact[0].shape, (1,)
    assert jac_exact[0].shape, (2, 1)
    assert hess_exact[0].shape, (2, 2, 1)

    # Test if these two models have the same call, jacobian, and hessian
    assert eval_exact[0] == pytest.approx(eval_numerical)
    assert isinstance(eval_numerical, float)
    assert isinstance(eval_exact[0][0], float)
    assert np.squeeze(jac_exact[0], axis=-1) == pytest.approx(jac_numerical)
    assert isinstance(jac_numerical, np.ndarray)
    assert np.squeeze(hess_exact[0], axis=-1) == pytest.approx(hess_numerical)
    assert isinstance(hess_numerical, np.ndarray)

    fit = Fit(chi2_exact, x=xdata, y=ydata, objective=MinimizeModel)
    fit_exact_result = fit.execute()
    fit = Fit(model, x=xdata, y=ydata, absolute_sigma=True)
    fit_num_result = fit.execute()
    assert fit_exact_result.value(a) == fit_num_result.value(a)
    assert fit_exact_result.value(b) == fit_num_result.value(b)
    assert fit_exact_result.stdev(a) == pytest.approx(fit_num_result.stdev(a))
    assert fit_exact_result.stdev(b) == pytest.approx(fit_num_result.stdev(b))

def test_LogLikelihood():
    """
    Tests if the LeastSquares objective gives the right shapes of output by
    comparing with its analytical equivalent.
    """
    # TODO: update these tests to use indexed variables in the future
    a, b = parameters('a, b')
    i = Idx('i', 100)
    x, y = variables('x, y')
    pdf = Exp(x, 1 / a) * Exp(x, b)

    np.random.seed(10)
    xdata = np.random.exponential(3.5, 100)

    # We use minus loglikelihood for the model, because the objective was
    # designed to find the maximum when used with a *minimizer*, so it has
    # opposite sign. Also test MinimizeModel at the same time.
    logL_model = Model({y: pdf})
    logL_exact = Model({y: - FlattenSum(log(pdf), i)})
    logL_numerical = LogLikelihood(logL_model, {x: xdata, y: None})
    logL_minmodel = MinimizeModel(logL_exact, data={x: xdata, y: None})

    # Test model jacobian and hessian shape
    eval_exact = logL_exact(x=xdata, a=2, b=3)
    jac_exact = logL_exact.eval_jacobian(x=xdata, a=2, b=3)
    hess_exact = logL_exact.eval_hessian(x=xdata, a=2, b=3)
    eval_minimizemodel = logL_minmodel(a=2, b=3)
    jac_minimizemodel = logL_minmodel.eval_jacobian(a=2, b=3)
    hess_minimizemodel = logL_minmodel.eval_hessian(a=2, b=3)
    eval_numerical = logL_numerical(a=2, b=3)
    jac_numerical = logL_numerical.eval_jacobian(a=2, b=3)
    hess_numerical = logL_numerical.eval_hessian(a=2, b=3)

    # TODO: These shapes should not have the ones! This is due to the current
    # convention that scalars should be returned as a 1d array by Model's.
    assert eval_exact[0].shape == (1,)
    assert jac_exact[0].shape == (2, 1)
    assert hess_exact[0].shape == (2, 2, 1)
    # Test if identical to MinimizeModel
    assert eval_exact[0] == pytest.approx(eval_minimizemodel)
    assert jac_exact[0] == pytest.approx(jac_minimizemodel)
    assert hess_exact[0] == pytest.approx(hess_minimizemodel)

    # Test if these two models have the same call, jacobian, and hessian.
    # Since models always have components as their first dimension, we have
    # to slice that away.
    assert eval_exact.y == pytest.approx(eval_numerical)
    assert isinstance(eval_numerical, float)
    assert isinstance(eval_exact.y[0], float)
    assert np.squeeze(jac_exact[0], axis=-1) == pytest.approx(jac_numerical)
    assert isinstance(jac_numerical, np.ndarray)
    assert np.squeeze(hess_exact[0], axis=-1) == pytest.approx(hess_numerical)
    assert isinstance(hess_numerical, np.ndarray)

    fit = Fit(logL_exact, x=xdata, objective=MinimizeModel)
    fit_exact_result = fit.execute()
    fit = Fit(logL_model, x=xdata, objective=LogLikelihood)
    fit_num_result = fit.execute()
    assert fit_exact_result.value(a) == pytest.approx(fit_num_result.value(a))
    assert fit_exact_result.value(b) == pytest.approx(fit_num_result.value(b))
    assert fit_exact_result.stdev(a) == pytest.approx(fit_num_result.stdev(a))
    assert fit_exact_result.stdev(b) == pytest.approx(fit_num_result.stdev(b))

def test_data_sanity():
    """
    Tests very basicly the data sanity for different objective types.
    :return:
    """
    # Create test data
    xdata = np.linspace(0, 100, 25)  # From 0 to 100 in 100 steps
    a_vec = np.random.normal(15.0, scale=2.0, size=xdata.shape)
    b_vec = np.random.normal(100, scale=2.0, size=xdata.shape)
    ydata = a_vec * xdata + b_vec  # Point scattered around the line 5 * x + 105

    # Normal symbolic fit
    a = Parameter('a', value=0, min=0.0, max=1000)
    b = Parameter('b', value=0, min=0.0, max=1000)
    x, y, z = variables('x, y, z')
    model = Model({y: a * x + b})

    for objective in [VectorLeastSquares, LeastSquares, LogLikelihood,
                      MinimizeModel]:
        if issubclass(objective, BaseIndependentObjective):
            incomplete_data = {}
            data = {x: xdata}
            overcomplete_data = {x: xdata, z: ydata}
        else:
            incomplete_data = {x: xdata, y: ydata}
            data = {x: xdata, y: ydata,
                    model.sigmas[y]: np.ones_like(ydata)}
            overcomplete_data = {x: xdata, y: ydata, z: ydata,
                                 model.sigmas[y]: np.ones_like(ydata)}
        with pytest.raises(KeyError):
            obj = objective(model, data=incomplete_data)

        obj = objective(model, data=data)
        # Overcomplete data has to be allowed, since constraints share their
        # data with models.
        obj = objective(model, data=overcomplete_data)

def test_LogLikelihood_global():
    """
    This is a test for global likelihood fitting to multiple data sets.
    Based on SO question 56006357.
    """
    # creating the data
    mu1, mu2 = .05, -.05
    sigma1, sigma2 = 3.5, 2.5
    n1, n2 = 80, 90
    np.random.seed(42)
    x1 = np.random.vonmises(mu1, sigma1, n1)
    x2 = np.random.vonmises(mu2, sigma2, n2)

    n = 2  # number of components
    xs = variables(
        'x,' + ','.join('x_{}'.format(i) for i in range(1, n + 1)))
    x, xs = xs[0], xs[1:]
    ys = variables(','.join('y_{}'.format(i) for i in range(1, n + 1)))
    mu, kappa = parameters('mu, kappa')
    kappas = parameters(','.join('k_{}'.format(i) for i in range(1, n + 1)),
                        min=0, max=10)
    mu.min, mu.max = - np.pi, np.pi

    template = exp(kappa * cos(x - mu)) / (2 * pi * besseli(0, kappa))

    model = Model(
        {y_i: template.subs({kappa: k_i, x: x_i}) for y_i, x_i, k_i in
            zip(ys, xs, kappas)}
    )

    all_data = {xs[0]: x1, xs[1]: x2, ys[0]: None, ys[1]: None}
    all_params = {'mu': 1}
    all_params.update({k_i.name: 1 for k_i in kappas})

    # Evaluate the loglikelihood and its jacobian and hessian
    logL = LogLikelihood(model, data=all_data)
    eval_numerical = logL(**all_params)
    jac_numerical = logL.eval_jacobian(**all_params)
    hess_numerical = logL.eval_hessian(**all_params)

    # Test the types and shapes of the components.
    assert isinstance(eval_numerical, float)
    assert isinstance(jac_numerical, np.ndarray)
    assert isinstance(hess_numerical, np.ndarray)

    assert eval_numerical.shape == tuple()  # Empty tuple -> scalar
    assert jac_numerical.shape == (3,)
    assert hess_numerical.shape == (3, 3,)
<|endoftext|>