<commit_before>"Converts Nearley grammars to Lark"

import os.path
import sys
import codecs
import argparse

from lark import Lark, InlineTransformer

nearley_grammar = r"""
    start: (ruledef|directive)+

    directive: "@" NAME (STRING|NAME)
             | "@" JS  -> js_code
    ruledef: NAME "->" expansions
           | NAME REGEXP "->" expansions -> macro
    expansions: expansion ("|" expansion)*

    expansion: expr+ js

    ?expr: item (":" /[+*?]/)?

    ?item: rule|string|regexp|null
         | "(" expansions ")"

    rule: NAME
    string: STRING
    regexp: REGEXP
    null: "null"
    JS: /{%.*?%}/s
    js: JS?

    NAME: /[a-zA-Z_$]\w*/
    COMMENT: /#[^\n]*/
    REGEXP: /\[.*?\]/

    %import common.ESCAPED_STRING -> STRING
    %import common.WS
    %ignore WS
    %ignore COMMENT

    """

nearley_grammar_parser = Lark(nearley_grammar, parser='earley', lexer='standard')

def _get_rulename(name):
    name = {'_': '_ws_maybe', '__':'_ws'}.get(name, name)
    return 'n_' + name.replace('$', '__DOLLAR__').lower()

class NearleyToLark(InlineTransformer):
    def __init__(self):
        self._count = 0
        self.extra_rules = {}
        self.extra_rules_rev = {}
        self.alias_js_code = {}

    def _new_function(self, code):
        name = 'alias_%d' % self._count
        self._count += 1

        self.alias_js_code[name] = code
        return name

    def _extra_rule(self, rule):
        if rule in self.extra_rules_rev:
            return self.extra_rules_rev[rule]

        name = 'xrule_%d' % len(self.extra_rules)
        assert name not in self.extra_rules
        self.extra_rules[name] = rule
        self.extra_rules_rev[rule] = name
        return name

    def rule(self, name):
        return _get_rulename(name)

    def ruledef(self, name, exps):
        return '!%s: %s' % (_get_rulename(name), exps)

    def expr(self, item, op):
        rule = '(%s)%s' % (item, op)
        return self._extra_rule(rule)

    def regexp(self, r):
        return '/%s/' % r

    def null(self):
        return ''

    def string(self, s):
        return self._extra_rule(s)

    def expansion(self, *x):
        x, js = x[:-1], x[-1]
        if js.children:
            js_code ,= js.children
            js_code = js_code[2:-2]
            alias = '-> ' + self._new_function(js_code)
        else:
            alias = ''
        return ' '.join(x) + alias

    def expansions(self, *x):
        return '%s' % ('\n    |'.join(x))

    def start(self, *rules):
        return '\n'.join(filter(None, rules))

def _nearley_to_lark(g, builtin_path, n2l, js_code, folder_path, includes):
    rule_defs = []

    tree = nearley_grammar_parser.parse(g)
    for statement in tree.children:
        if statement.data == 'directive':
            directive, arg = statement.children
            if directive in ('builtin', 'include'):
                folder = builtin_path if directive == 'builtin' else folder_path
                path = os.path.join(folder, arg[1:-1])
                if path not in includes:
                    includes.add(path)
                    with codecs.open(path, encoding='utf8') as f:
                        text = f.read()
                    rule_defs += _nearley_to_lark(text, builtin_path, n2l, js_code, os.path.abspath(os.path.dirname(path)), includes)
            else:
                assert False, directive
        elif statement.data == 'js_code':
            code ,= statement.children
            code = code[2:-2]
            js_code.append(code)
        elif statement.data == 'macro':
            pass    # TODO Add support for macros!
        elif statement.data == 'ruledef':
            rule_defs.append( n2l.transform(statement) )
        else:
            raise Exception("Unknown statement: %s" % statement)

    return rule_defs

def create_code_for_nearley_grammar(g, start, builtin_path, folder_path, es6=False):
    import js2py

    emit_code = []
    def emit(x=None):
        if x:
            emit_code.append(x)
        emit_code.append('\n')

    js_code = ['function id(x) {return x[0];}']
    n2l = NearleyToLark()
    rule_defs = _nearley_to_lark(g, builtin_path, n2l, js_code, folder_path, set())
    lark_g = '\n'.join(rule_defs)
    lark_g += '\n'+'\n'.join('!%s: %s' % item for item in n2l.extra_rules.items())

    emit('from lark import Lark, Transformer')
    emit()
    emit('grammar = ' + repr(lark_g))
    emit()

    for alias, code in n2l.alias_js_code.items():
        js_code.append('%s = (%s);' % (alias, code))

    if es6:
        emit(js2py.translate_js6('\n'.join(js_code)))
    else:
        emit(js2py.translate_js('\n'.join(js_code)))
    emit('class TransformNearley(Transformer):')
    for alias in n2l.alias_js_code:
        emit("    %s = var.get('%s').to_python()" % (alias, alias))
    emit("    __default__ = lambda self, n, c, m: c if c else None")

    emit()
    emit('parser = Lark(grammar, start="n_%s", maybe_placeholders=False)' % start)
    emit('def parse(text):')
    emit('    return TransformNearley().transform(parser.parse(text))')

    return ''.join(emit_code)

def main(fn, start, nearley_lib, es6=False):
    with codecs.open(fn, encoding='utf8') as f:
        grammar = f.read()
    return create_code_for_nearley_grammar(grammar, start, os.path.join(nearley_lib, 'builtin'), os.path.abspath(os.path.dirname(fn)), es6=es6)

def get_arg_parser():
    parser = argparse.ArgumentParser('Reads Nearley grammar (with js functions) outputs an equivalent lark parser.')
    parser.add_argument('nearley_grammar', help='Path to the file containing the nearley grammar')
    parser.add_argument('start_rule', help='Rule within the nearley grammar to make the base rule')
    parser.add_argument('nearley_lib', help='Path to root directory of nearley codebase (used for including builtins)')
    parser.add_argument('--es6', help='Enable experimental ES6 support', action='store_true')
    return parser

if __name__ == '__main__':
    parser = get_arg_parser()
    args = parser.parse_args()
    print(main(fn=args.nearley_grammar, start=args.start_rule, nearley_lib=args.nearley_lib, es6=args.es6))
<commit_msg>tools.nearley: Add support for case-insensitive strings (Issue #753)<commit_after>"Converts Nearley grammars to Lark"

import os.path
import sys
import codecs
import argparse

from lark import Lark, InlineTransformer

nearley_grammar = r"""
    start: (ruledef|directive)+

    directive: "@" NAME (STRING|NAME)
             | "@" JS  -> js_code
    ruledef: NAME "->" expansions
           | NAME REGEXP "->" expansions -> macro
    expansions: expansion ("|" expansion)*

    expansion: expr+ js

    ?expr: item (":" /[+*?]/)?

    ?item: rule|string|regexp|null
         | "(" expansions ")"

    rule: NAME
    string: STRING
    regexp: REGEXP
    null: "null"
    JS: /{%.*?%}/s
    js: JS?

    NAME: /[a-zA-Z_$]\w*/
    COMMENT: /#[^\n]*/
    REGEXP: /\[.*?\]/

    STRING: _STRING "i"?

    %import common.ESCAPED_STRING -> _STRING
    %import common.WS
    %ignore WS
    %ignore COMMENT

    """

nearley_grammar_parser = Lark(nearley_grammar, parser='earley', lexer='standard')

def _get_rulename(name):
    name = {'_': '_ws_maybe', '__':'_ws'}.get(name, name)
    return 'n_' + name.replace('$', '__DOLLAR__').lower()

class NearleyToLark(InlineTransformer):
    def __init__(self):
        self._count = 0
        self.extra_rules = {}
        self.extra_rules_rev = {}
        self.alias_js_code = {}

    def _new_function(self, code):
        name = 'alias_%d' % self._count
        self._count += 1

        self.alias_js_code[name] = code
        return name

    def _extra_rule(self, rule):
        if rule in self.extra_rules_rev:
            return self.extra_rules_rev[rule]

        name = 'xrule_%d' % len(self.extra_rules)
        assert name not in self.extra_rules
        self.extra_rules[name] = rule
        self.extra_rules_rev[rule] = name
        return name

    def rule(self, name):
        return _get_rulename(name)

    def ruledef(self, name, exps):
        return '!%s: %s' % (_get_rulename(name), exps)

    def expr(self, item, op):
        rule = '(%s)%s' % (item, op)
        return self._extra_rule(rule)

    def regexp(self, r):
        return '/%s/' % r

    def null(self):
        return ''

    def string(self, s):
        return self._extra_rule(s)

    def expansion(self, *x):
        x, js = x[:-1], x[-1]
        if js.children:
            js_code ,= js.children
            js_code = js_code[2:-2]
            alias = '-> ' + self._new_function(js_code)
        else:
            alias = ''
        return ' '.join(x) + alias

    def expansions(self, *x):
        return '%s' % ('\n    |'.join(x))

    def start(self, *rules):
        return '\n'.join(filter(None, rules))

def _nearley_to_lark(g, builtin_path, n2l, js_code, folder_path, includes):
    rule_defs = []

    tree = nearley_grammar_parser.parse(g)
    for statement in tree.children:
        if statement.data == 'directive':
            directive, arg = statement.children
            if directive in ('builtin', 'include'):
                folder = builtin_path if directive == 'builtin' else folder_path
                path = os.path.join(folder, arg[1:-1])
                if path not in includes:
                    includes.add(path)
                    with codecs.open(path, encoding='utf8') as f:
                        text = f.read()
                    rule_defs += _nearley_to_lark(text, builtin_path, n2l, js_code, os.path.abspath(os.path.dirname(path)), includes)
            else:
                assert False, directive
        elif statement.data == 'js_code':
            code ,= statement.children
            code = code[2:-2]
            js_code.append(code)
        elif statement.data == 'macro':
            pass    # TODO Add support for macros!
        elif statement.data == 'ruledef':
            rule_defs.append( n2l.transform(statement) )
        else:
            raise Exception("Unknown statement: %s" % statement)

    return rule_defs

def create_code_for_nearley_grammar(g, start, builtin_path, folder_path, es6=False):
    import js2py

    emit_code = []
    def emit(x=None):
        if x:
            emit_code.append(x)
        emit_code.append('\n')

    js_code = ['function id(x) {return x[0];}']
    n2l = NearleyToLark()
    rule_defs = _nearley_to_lark(g, builtin_path, n2l, js_code, folder_path, set())
    lark_g = '\n'.join(rule_defs)
    lark_g += '\n'+'\n'.join('!%s: %s' % item for item in n2l.extra_rules.items())

    emit('from lark import Lark, Transformer')
    emit()
    emit('grammar = ' + repr(lark_g))
    emit()

    for alias, code in n2l.alias_js_code.items():
        js_code.append('%s = (%s);' % (alias, code))

    if es6:
        emit(js2py.translate_js6('\n'.join(js_code)))
    else:
        emit(js2py.translate_js('\n'.join(js_code)))
    emit('class TransformNearley(Transformer):')
    for alias in n2l.alias_js_code:
        emit("    %s = var.get('%s').to_python()" % (alias, alias))
    emit("    __default__ = lambda self, n, c, m: c if c else None")

    emit()
    emit('parser = Lark(grammar, start="n_%s", maybe_placeholders=False)' % start)
    emit('def parse(text):')
    emit('    return TransformNearley().transform(parser.parse(text))')

    return ''.join(emit_code)

def main(fn, start, nearley_lib, es6=False):
    with codecs.open(fn, encoding='utf8') as f:
        grammar = f.read()
    return create_code_for_nearley_grammar(grammar, start, os.path.join(nearley_lib, 'builtin'), os.path.abspath(os.path.dirname(fn)), es6=es6)

def get_arg_parser():
    parser = argparse.ArgumentParser(description='Reads a Nearley grammar (with js functions), and outputs an equivalent lark parser.')
    parser.add_argument('nearley_grammar', help='Path to the file containing the nearley grammar')
    parser.add_argument('start_rule', help='Rule within the nearley grammar to make the base rule')
    parser.add_argument('nearley_lib', help='Path to root directory of nearley codebase (used for including builtins)')
    parser.add_argument('--es6', help='Enable experimental ES6 support', action='store_true')
    return parser

if __name__ == '__main__':
    parser = get_arg_parser()
    args = parser.parse_args()
    print(main(fn=args.nearley_grammar, start=args.start_rule, nearley_lib=args.nearley_lib, es6=args.es6))
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
'''
    syncing app
'''
import os

from sync_app.gdrive_instance import GdriveInstance
from sync_app.file_cache import FileListCache
from sync_app.file_list_gdrive import FileListGdrive

from sync_app.file_list_s3 import FileListS3

def parse_gdrive():
    flist = FileListGdrive()
    flist.fill_file_list_gdrive()
    fcache = FileListCache(pickle_file='%s/.gdrive_file_list_cache.pkl.gz' % os.getenv('HOME'))
    fcache.write_cache_file_list(flist.filelist)

def print_gdrive():
    #gdrive = GdriveInstance()
    fcache = FileListCache(pickle_file='%s/.gdrive_file_list_cache.pkl.gz' % os.getenv('HOME'))
    flist = fcache.get_cache_file_list(file_list_class=FileListGdrive)
    n = 0
    for fn, val in flist.filelist_name_dict.items():
        if n > 10:
            return
        finfo = val[0]
        fullname = '%s/%s' % (finfo.exportpath, finfo.filename)
        print fullname, finfo.urlname
        #gdrive.download(finfo.urlname, fullname)
        n += 1

def parse_s3():
    flist = FileListS3()
    flist.fill_file_list_gdrive()
    fcache = FileListCache(pickle_file='%s/.s3_file_list_cache.pkl.gz' % os.getenv('HOME'))
    fcache.write_cache_file_list(flist.filelist)

def print_s3():
    fcache = FileListCache(pickle_file='%s/.s3_file_list_cache.pkl.gz' % os.getenv('HOME'))
    flist = fcache.get_cache_file_list(file_list_class=FileListS3)
    for finfo in flist.filelist[:10]:
        print finfo

if __name__ == '__main__':
    parse_gdrive()
    print_gdrive()
    parse_s3()
    print_s3()<commit_msg>test local<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
'''
    syncing app
'''
import os

from sync_app.file_list_local import FileListLocal
from sync_app.gdrive_instance import GdriveInstance
from sync_app.file_cache import FileListCache
from sync_app.file_list_gdrive import FileListGdrive

from sync_app.file_list_s3 import FileListS3

def parse_gdrive():
    flist = FileListGdrive()
    flist.fill_file_list_gdrive()
    fcache = FileListCache(pickle_file='%s/.gdrive_file_list_cache.pkl.gz' % os.getenv('HOME'))
    fcache.write_cache_file_list(flist.filelist)

def print_gdrive():
    #gdrive = GdriveInstance()
    fcache = FileListCache(pickle_file='%s/.gdrive_file_list_cache.pkl.gz' % os.getenv('HOME'))
    flist = fcache.get_cache_file_list(file_list_class=FileListGdrive)
    n = 0
    for fn, val in flist.filelist_name_dict.items():
        if n > 10:
            return
        finfo = val[0]
        fullname = '%s/%s' % (finfo.exportpath, finfo.filename)
        print fullname, finfo.urlname
        #gdrive.download(finfo.urlname, fullname)
        n += 1

def parse_s3():
    flist = FileListS3()
    flist.fill_file_list_gdrive()
    fcache = FileListCache(pickle_file='%s/.s3_file_list_cache.pkl.gz' % os.getenv('HOME'))
    fcache.write_cache_file_list(flist.filelist)

def print_s3():
    fcache = FileListCache(pickle_file='%s/.s3_file_list_cache.pkl.gz' % os.getenv('HOME'))
    flist = fcache.get_cache_file_list(file_list_class=FileListS3)
    for finfo in flist.filelist[:10]:
        print finfo

def parse_mp3():
    flist = FileListLocal()
    flist.fill_file_list_local(directory='%s/Documents/mp3' % os.getenv('HOME'))
    fcache = FileListCache(pickle_file='%s/.local_file_list_cache.pkl.gz' % os.getenv('HOME'))
    fcache.write_cache_file_list(flist.filelist)

def print_mp3():
    fcache = FileListCache(pickle_file='%s/.local_file_list_cache.pkl.gz' % os.getenv('HOME'))
    flist = fcache.get_cache_file_list(file_list_class=FileListLocal)
    for finfo in flist.filelist[:10]:
        print finfo

if __name__ == '__main__':
    #parse_gdrive()
    #print_gdrive()
    #parse_s3()
    #print_s3()
    parse_mp3()
    print_mp3()

    <|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Created on Sat Jan 23 13:03:00 2016

@author: p
"""

import wellapplication as wa
import pandas as pd
import matplotlib

m = wa.Meso(token='demotoken')

def test_getelev():
    x = [-111.21, 41.4]
    m = wa.getelev(x)
    assert m > 100.0

def test_gethuc():
    x = [-111.21, 41.4]
    huc_data = wa.get_huc(x)
    assert len(huc_data[0])>0

def test_USGSID():
    x = [-111.21, 41.4]
    usgs_id = wa.USGSID(x)
    assert usgs_id == '412400111123601'

def test_nwis():
    nw = wa.nwis('dv', '01585200', 'sites')
    assert len(nw.sites) == 1

def test_nwis_gw():
    nw = wa.nwis('gwlevels','16010204','huc',siteStatus='all')
    df = nw.cleanGWL(nw.data)
    assert len(df) > 5

def test_mktest():
    x = range(0,100)
    trend = wa.MannKendall.mk_test(x,0.05)
    assert trend.trend == 'increasing'

def test_pipe():
    pipr = wa.piper()
    Chem =  {'Type':[1,2,2,3], 'Cl':[1.72,0.90,4.09,1.52], 'HCO3':[4.02,1.28,4.29,3.04],
             'SO4':[0.58,0.54,0.38,0.46], 'NaK':[1.40,0.90,3.38,2.86], 'Ca':[4.53,None,4.74,1.90],
             'Mg':[0.79,0.74,0.72,0.66], 'EC':[672.0,308.0,884.0,542.0], 'NO3':[0.4,0.36,0.08,0.40],
             'Sicc':[0.21,0.56,None,-0.41]}
    chem = pd.DataFrame(Chem)
    pipr.piperplot(chem)
    assert type(pipr.plot) == matplotlib.figure.Figure

def test_new_xle_imp():
    xle = 'docs/20160919_LittleHobble.xle'
    xle_df = wa.new_xle_imp(xle)
    assert len(xle_df) > 0

def test_xle_head_table():
    xle_dir = 'docs/'
    dir_df = wa.xle_head_table(xle_dir)
    assert len(xle_dir) > 0

def test_dataendclean():
    xle = 'docs/20160919_LittleHobble.xle'
    df = wa.new_xle_imp(xle)
    x = 'Level'
    xle1 = wa.dataendclean(df, x)
    assert len(xle1) > 1

def test_smoother():
    xle = 'docs/20160919_LittleHobble.xle'
    df = wa.new_xle_imp(xle)
    x = 'Level'
    xle1 = wa.smoother(df, x, sd=1)
    assert len(xle1) > 1

def test_hourly_resample():
    xle = 'docs/20160919_LittleHobble.xle'
    df = wa.new_xle_imp(xle)
    xle1 = wa.hourly_resample(df, minutes=30)

# Basic Function Tests
def testvars():
    var_list = m.variables()
    ok_(var_list)

def testmetadata():
    stations = m.metadata(radius=['wbb', 5])
    ok_(stations)

def testtimeseries():
    timeseries = m.timeseries(stid='kfnl', start='201504261800', end='201504262300')

def testclimatology():
    climatology = m.climatology(stid='kden', startclim='04260000', endclim='04270000', units='precip|in')

def testprecip():
    precip = m.precip(stid=['kfnl', 'ksdf'], start='201504261800', end='201504271200', units='precip|in')

def testclimatestats():
    climate_stats = m.climate_stats(stid='mtmet', startclim='03240000', endclim='03280000', type='all')

def testtimestats():
    stats = m.time_stats(stid='mtmet', start='201403240000', end='201403280000', type='all')

def testlatency():
    latency = m.latency(stid='mtmet', start='201403240000', end='201403280000')

def testnettypes():
    nettypes = m.networktypes()

def testnetworks():
    networks = m.networks()

def testattime():
    attime = m.attime(stid='kfnl', attime='201504261800', within='30')

<commit_msg>Update test.py<commit_after># -*- coding: utf-8 -*-
"""
Created on Sat Jan 23 13:03:00 2016

@author: p
"""

import wellapplication as wa
import pandas as pd
import matplotlib

m = wa.Meso(token='demotoken')

def test_getelev():
    x = [-111.21, 41.4]
    m = wa.getelev(x)
    assert m > 100.0

def test_gethuc():
    x = [-111.21, 41.4]
    huc_data = wa.get_huc(x)
    assert len(huc_data[0])>0

def test_USGSID():
    x = [-111.21, 41.4]
    usgs_id = wa.USGSID(x)
    assert usgs_id == '412400111123601'

def test_nwis():
    nw = wa.nwis('dv', '01585200', 'sites')
    assert len(nw.sites) == 1

def test_nwis_gw():
    nw = wa.nwis('gwlevels','16010204','huc',siteStatus='all')
    df = nw.cleanGWL(nw.data)
    assert len(df) > 5

def test_mktest():
    x = range(0,100)
    trend = wa.MannKendall.mk_test(x,0.05)
    assert trend.trend == 'increasing'

def test_pipe():
    pipr = wa.piper()
    Chem =  {'Type':[1,2,2,3], 'Cl':[1.72,0.90,4.09,1.52], 'HCO3':[4.02,1.28,4.29,3.04],
             'SO4':[0.58,0.54,0.38,0.46], 'NaK':[1.40,0.90,3.38,2.86], 'Ca':[4.53,None,4.74,1.90],
             'Mg':[0.79,0.74,0.72,0.66], 'EC':[672.0,308.0,884.0,542.0], 'NO3':[0.4,0.36,0.08,0.40],
             'Sicc':[0.21,0.56,None,-0.41]}
    chem = pd.DataFrame(Chem)
    pipr.piperplot(chem)
    assert type(pipr.plot) == matplotlib.figure.Figure

def test_new_xle_imp():
    xle = 'docs/20160919_LittleHobble.xle'
    xle_df = wa.new_xle_imp(xle)
    assert len(xle_df) > 0

def test_xle_head_table():
    xle_dir = 'docs/'
    dir_df = wa.xle_head_table(xle_dir)
    assert len(xle_dir) > 0

def test_dataendclean():
    xle = 'docs/20160919_LittleHobble.xle'
    df = wa.new_xle_imp(xle)
    x = 'Level'
    xle1 = wa.dataendclean(df, x)
    assert len(xle1) > 1

def test_smoother():
    xle = 'docs/20160919_LittleHobble.xle'
    df = wa.new_xle_imp(xle)
    x = 'Level'
    xle1 = wa.smoother(df, x, sd=1)
    assert len(xle1) > 1

def test_hourly_resample():
    xle = 'docs/20160919_LittleHobble.xle'
    df = wa.new_xle_imp(xle)
    xle1 = wa.hourly_resample(df, minutes=30)

# Basic Function Tests
def testvars():
    var_list = m.variables()

def testmetadata():
    stations = m.metadata(radius=['wbb', 5])

def testtimeseries():
    timeseries = m.timeseries(stid='kfnl', start='201504261800', end='201504262300')

def testclimatology():
    climatology = m.climatology(stid='kden', startclim='04260000', endclim='04270000', units='precip|in')

def testprecip():
    precip = m.precip(stid=['kfnl', 'ksdf'], start='201504261800', end='201504271200', units='precip|in')

def testclimatestats():
    climate_stats = m.climate_stats(stid='mtmet', startclim='03240000', endclim='03280000', type='all')

def testtimestats():
    stats = m.time_stats(stid='mtmet', start='201403240000', end='201403280000', type='all')

def testlatency():
    latency = m.latency(stid='mtmet', start='201403240000', end='201403280000')

def testnettypes():
    nettypes = m.networktypes()

def testnetworks():
    networks = m.networks()

def testattime():
    attime = m.attime(stid='kfnl', attime='201504261800', within='30')

<|endoftext|>


<commit_before>import os
import re
import sys
import json
import time
import shutil
import socket
import tempfile
import unittest
import subprocess

class TestUnit(unittest.TestCase):

    def setUp(self):
        self.testdir = tempfile.mkdtemp(prefix='unit-test-')

        os.mkdir(self.testdir + '/state')

        pardir = os.path.abspath(os.path.join(os.path.dirname(__file__),
            os.pardir))

        print()

        subprocess.call([pardir + '/build/unitd',
        # TODO       '--no-daemon',
            '--modules', pardir + '/build',
            '--state', self.testdir + '/state',
            '--pid', self.testdir + '/unit.pid',
            '--log', self.testdir + '/unit.log',
            '--control', 'unix:' + self.testdir + '/control.unit.sock'])

        if not self._waitforfiles([self.testdir + '/unit.pid',
            self.testdir + '/unit.log', self.testdir + '/control.unit.sock']):
            exit("Could not start unit")

    # TODO dependency check

    def tearDown(self):
        with open(self.testdir + '/unit.pid', 'r') as f:
            pid = f.read().rstrip()

        subprocess.call(['kill', pid])

        for i in range(1, 50):
            if not os.path.exists(self.testdir + '/unit.pid'):
                break
            time.sleep(0.1)

        if os.path.exists(self.testdir + '/unit.pid'):
            exit("Could not terminate unit")

        if '--log' in sys.argv:
            with open(self.testdir + '/unit.log', 'r') as f:
                print(f.read())

        if '--leave' not in sys.argv:
            shutil.rmtree(self.testdir)

    def _waitforfiles(self, files):
        if isinstance(files, str):
            files = [files]

        count = 0
        for i in range(1, 50):
            for f in files:
                if os.path.exists(f):
                   count += 1

            if count == len(files):
                return 1

            count = 0
            time.sleep(0.1)

        return 0

class TestUnitControl(TestUnit):

    # TODO socket reuse
    # TODO http client

    def get(self, path='/'):

        with self._control_sock() as sock:
            req = ('GET ' + path
                + ' HTTP/1.1\r\nHost: localhost\r\n\r\n').encode()

            sock.sendall(req)

            if '--verbose' in sys.argv:
                print('>>>\n', req)

            resp = self._recvall(sock)

            if '--verbose' in sys.argv:
                print('<<<\n', resp)

        return self._body_json(resp)

    def delete(self, path='/'):

        with self._control_sock() as sock:
            req = ('DELETE ' + path
                + ' HTTP/1.1\r\nHost: localhost\r\n\r\n').encode()

            sock.sendall(req)

            if '--verbose' in sys.argv:
                print('>>>\n', req)

            resp = self._recvall(sock)

            if '--verbose' in sys.argv:
                print('<<<\n', resp)

        return self._body_json(resp)

    def put(self, path='/', data=''):

        if isinstance(data, str):
            data = data.encode()

        with self._control_sock() as sock:
            req = ('PUT ' + path + (' HTTP/1.1\nHost: localhost\n'
                'Content-Length: ') + str(len(data)) + '\r\n\r\n').encode() + data

            sock.sendall(req)

            if '--verbose' in sys.argv:
                print('>>>\n', req)

            resp = self._recvall(sock)

            if '--verbose' in sys.argv:
                print('<<<\n', resp)

        return self._body_json(resp)

    def _control_sock(self):
        sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
        sock.connect(self.testdir + '/control.unit.sock')
        return sock

    def _recvall(self, sock, buff_size=4096):
        data = ''
        while True:
            part = sock.recv(buff_size).decode()
            data += part
            if len(part) < buff_size:
                break

        return data

    def _body_json(self, resp):
        m = re.search('.*?\x0d\x0a?\x0d\x0a?(.*)', resp, re.M | re.S)
        return json.loads(m.group(1))
<commit_msg>Tests: don't exceed 79 characters.<commit_after>import os
import re
import sys
import json
import time
import shutil
import socket
import tempfile
import unittest
import subprocess

class TestUnit(unittest.TestCase):

    def setUp(self):
        self.testdir = tempfile.mkdtemp(prefix='unit-test-')

        os.mkdir(self.testdir + '/state')

        pardir = os.path.abspath(os.path.join(os.path.dirname(__file__),
            os.pardir))

        print()

        subprocess.call([pardir + '/build/unitd',
        # TODO       '--no-daemon',
            '--modules', pardir + '/build',
            '--state', self.testdir + '/state',
            '--pid', self.testdir + '/unit.pid',
            '--log', self.testdir + '/unit.log',
            '--control', 'unix:' + self.testdir + '/control.unit.sock'])

        if not self._waitforfiles([self.testdir + '/unit.pid',
            self.testdir + '/unit.log', self.testdir + '/control.unit.sock']):
            exit("Could not start unit")

    # TODO dependency check

    def tearDown(self):
        with open(self.testdir + '/unit.pid', 'r') as f:
            pid = f.read().rstrip()

        subprocess.call(['kill', pid])

        for i in range(1, 50):
            if not os.path.exists(self.testdir + '/unit.pid'):
                break
            time.sleep(0.1)

        if os.path.exists(self.testdir + '/unit.pid'):
            exit("Could not terminate unit")

        if '--log' in sys.argv:
            with open(self.testdir + '/unit.log', 'r') as f:
                print(f.read())

        if '--leave' not in sys.argv:
            shutil.rmtree(self.testdir)

    def _waitforfiles(self, files):
        if isinstance(files, str):
            files = [files]

        count = 0
        for i in range(1, 50):
            for f in files:
                if os.path.exists(f):
                   count += 1

            if count == len(files):
                return 1

            count = 0
            time.sleep(0.1)

        return 0

class TestUnitControl(TestUnit):

    # TODO socket reuse
    # TODO http client

    def get(self, path='/'):

        with self._control_sock() as sock:
            req = ('GET ' + path
                + ' HTTP/1.1\r\nHost: localhost\r\n\r\n').encode()

            sock.sendall(req)

            if '--verbose' in sys.argv:
                print('>>>\n', req)

            resp = self._recvall(sock)

            if '--verbose' in sys.argv:
                print('<<<\n', resp)

        return self._body_json(resp)

    def delete(self, path='/'):

        with self._control_sock() as sock:
            req = ('DELETE ' + path
                + ' HTTP/1.1\r\nHost: localhost\r\n\r\n').encode()

            sock.sendall(req)

            if '--verbose' in sys.argv:
                print('>>>\n', req)

            resp = self._recvall(sock)

            if '--verbose' in sys.argv:
                print('<<<\n', resp)

        return self._body_json(resp)

    def put(self, path='/', data=''):

        if isinstance(data, str):
            data = data.encode()

        with self._control_sock() as sock:
            req = ('PUT ' + path + ' HTTP/1.1\nHost: localhost\n'
                + 'Content-Length: ' + str(len(data))
                + '\r\n\r\n').encode() + data

            sock.sendall(req)

            if '--verbose' in sys.argv:
                print('>>>\n', req)

            resp = self._recvall(sock)

            if '--verbose' in sys.argv:
                print('<<<\n', resp)

        return self._body_json(resp)

    def _control_sock(self):
        sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
        sock.connect(self.testdir + '/control.unit.sock')
        return sock

    def _recvall(self, sock, buff_size=4096):
        data = ''
        while True:
            part = sock.recv(buff_size).decode()
            data += part
            if len(part) < buff_size:
                break

        return data

    def _body_json(self, resp):
        m = re.search('.*?\x0d\x0a?\x0d\x0a?(.*)', resp, re.M | re.S)
        return json.loads(m.group(1))
<|endoftext|>


<commit_before># No shebang line, this module is meant to be imported
#
# Copyright 2013 Oliver Palmer
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from textwrap import dedent
from sqlalchemy import event
from pyfarm.flaskapp import db
from pyfarm.config.enum import WorkState
from pyfarm.models.core import (
    TABLE_JOB, TABLE_TASK, TABLE_AGENT, WorkColumns, modelfor)
from pyfarm.models.mixins import WorkValidationMixin, StateChangedMixin

class TaskModel(db.Model, WorkValidationMixin, StateChangedMixin):
    """
    Defines a task which a child of a :class:`Job`.  This table represents
    rows which contain the individual work unit(s) for a job.
    """
    __tablename__ = TABLE_TASK
    STATE_ENUM = WorkState()

    # shared work columns
    id, state, priority, time_submitted, time_started, time_finished = \
        WorkColumns(STATE_ENUM.QUEUED, "job.priority")

    attempts = db.Column(db.Integer, default=0,
                         doc=dedent("""
                         The number attempts which have been made on this
                         task. This value is auto incremented when
                         :attr:`state` changes to a value synonyms with a
                         running state."""))
    frame = db.Column(db.Float, nullable=False,
                      doc=dedent("""
                      The frame the :class:`TaskModel` will be executing."""))

    # relationships
    _agentid = db.Column(db.Integer, db.ForeignKey("%s.id" % TABLE_AGENT),
                         doc=dedent("""
                         Foreign key which stores :attr:`JobModel.id`"""))
    _jobid = db.Column(db.Integer, db.ForeignKey("%s.id" % TABLE_JOB),
                       doc=dedent("""
                       Foreign key which stores :attr:`JobModel.id`"""))
    _parenttask = db.Column(db.Integer, db.ForeignKey("%s.id" % TABLE_TASK),
                            doc=dedent("""
                            The foreign key which stores :attr:`TaskModel.id`
                            """))
    siblings = db.relationship("TaskModel",
                               backref=db.backref("task", remote_side=[id]),
                               doc=dedent("""
                               Relationship to other tasks which have the same
                               parent"""))

    @staticmethod
    def agentChangedEvent(target, new_value, old_value, initiator):
        """set the state to ASSIGN whenever the agent is changed"""
        if new_value is not None:
            target.state = target.STATE_ENUM.ASSIGN

event.listen(TaskModel._agentid, "set", TaskModel.agentChangedEvent)
event.listen(TaskModel.state, "set", TaskModel.stateChangedEvent)

class Task(TaskModel):
    """
    Provides :meth:`__init__` for :class:`TaskModel` so the model can
    be instanced with initial values.
    """
    def __init__(self, job, frame, parent_task=None, state=None,
                 priority=None, attempts=None, agent=None):
        # build parent job id
        if modelfor(job, TABLE_JOB):
            jobid = job.jobid
            if jobid is None:
                raise ValueError("`job` with null id provided")
        elif isinstance(job, int):
            jobid = job
        else:
            raise ValueError("failed to determine job id")

        # build parent task id
        if parent_task is None:
            parent_taskid = None
        elif modelfor(parent_task, TABLE_TASK):
            parent_taskid = parent_task.id
            if parent_taskid is None:
                raise ValueError("`parent_task` with null id provided")
        elif isinstance(parent_task, int):
            parent_taskid = parent_task
        else:
            raise ValueError("failed to determine parent task id")

        # build agent id
        if agent is None:
            agentid = None
        elif modelfor(agent, TABLE_AGENT):
            agentid = agent.id
            if agentid is None:
                raise ValueError("`agent` with null id provided")
        elif isinstance(agent, int):
            agentid = agent
        else:
            raise ValueError("failed to determine agent id")

        self._jobid = jobid
        self.frame = frame

        if parent_taskid is not None:
            self._parenttask = parent_taskid

        if agentid is not None:
            self._agentid = agentid

        if state is not None:
            self.state = state

        if priority is not None:
            self.priority = priority

        if attempts is not None:
            self.attempts = attempts
<commit_msg>needed for tests<commit_after># No shebang line, this module is meant to be imported
#
# Copyright 2013 Oliver Palmer
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from textwrap import dedent
from sqlalchemy import event
from pyfarm.flaskapp import db
from pyfarm.config.enum import WorkState
from pyfarm.models.core import (
    TABLE_JOB, TABLE_TASK, TABLE_AGENT, WorkColumns, modelfor)
from pyfarm.models.mixins import WorkValidationMixin, StateChangedMixin

class TaskModel(db.Model, WorkValidationMixin, StateChangedMixin):
    """
    Defines a task which a child of a :class:`Job`.  This table represents
    rows which contain the individual work unit(s) for a job.
    """
    __tablename__ = TABLE_TASK
    STATE_ENUM = WorkState()
    STATE_DEFAULT = STATE_ENUM.QUEUED

    # shared work columns
    id, state, priority, time_submitted, time_started, time_finished = \
        WorkColumns(STATE_DEFAULT, "job.priority")

    attempts = db.Column(db.Integer, default=0,
                         doc=dedent("""
                         The number attempts which have been made on this
                         task. This value is auto incremented when
                         :attr:`state` changes to a value synonyms with a
                         running state."""))
    frame = db.Column(db.Float, nullable=False,
                      doc=dedent("""
                      The frame the :class:`TaskModel` will be executing."""))

    # relationships
    _agentid = db.Column(db.Integer, db.ForeignKey("%s.id" % TABLE_AGENT),
                         doc=dedent("""
                         Foreign key which stores :attr:`JobModel.id`"""))
    _jobid = db.Column(db.Integer, db.ForeignKey("%s.id" % TABLE_JOB),
                       doc=dedent("""
                       Foreign key which stores :attr:`JobModel.id`"""))
    _parenttask = db.Column(db.Integer, db.ForeignKey("%s.id" % TABLE_TASK),
                            doc=dedent("""
                            The foreign key which stores :attr:`TaskModel.id`
                            """))
    siblings = db.relationship("TaskModel",
                               backref=db.backref("task", remote_side=[id]),
                               doc=dedent("""
                               Relationship to other tasks which have the same
                               parent"""))

    @staticmethod
    def agentChangedEvent(target, new_value, old_value, initiator):
        """set the state to ASSIGN whenever the agent is changed"""
        if new_value is not None:
            target.state = target.STATE_ENUM.ASSIGN

event.listen(TaskModel._agentid, "set", TaskModel.agentChangedEvent)
event.listen(TaskModel.state, "set", TaskModel.stateChangedEvent)

class Task(TaskModel):
    """
    Provides :meth:`__init__` for :class:`TaskModel` so the model can
    be instanced with initial values.
    """
    def __init__(self, job, frame, parent_task=None, state=None,
                 priority=None, attempts=None, agent=None):
        # build parent job id
        if modelfor(job, TABLE_JOB):
            jobid = job.jobid
            if jobid is None:
                raise ValueError("`job` with null id provided")
        elif isinstance(job, int):
            jobid = job
        else:
            raise ValueError("failed to determine job id")

        # build parent task id
        if parent_task is None:
            parent_taskid = None
        elif modelfor(parent_task, TABLE_TASK):
            parent_taskid = parent_task.id
            if parent_taskid is None:
                raise ValueError("`parent_task` with null id provided")
        elif isinstance(parent_task, int):
            parent_taskid = parent_task
        else:
            raise ValueError("failed to determine parent task id")

        # build agent id
        if agent is None:
            agentid = None
        elif modelfor(agent, TABLE_AGENT):
            agentid = agent.id
            if agentid is None:
                raise ValueError("`agent` with null id provided")
        elif isinstance(agent, int):
            agentid = agent
        else:
            raise ValueError("failed to determine agent id")

        self._jobid = jobid
        self.frame = frame

        if parent_taskid is not None:
            self._parenttask = parent_taskid

        if agentid is not None:
            self._agentid = agentid

        if state is not None:
            self.state = state

        if priority is not None:
            self.priority = priority

        if attempts is not None:
            self.attempts = attempts
<|endoftext|>


<commit_before>import subprocess

class TestResult:
    def __init__(self, result):
        self.raw_result = result
        self.stdout = result.stdout.decode()
        self.stderr = result.stderr.decode()

def run(args, should_fail=False):
    result = subprocess.run(args, capture_output=True)
    if should_fail:
        assert result.returncode > 0
    else:
        assert result.returncode == 0
    return TestResult(result)

def fails(args):
    return run(args, should_fail=True)

def prints_help(args):
    stderr = run(args, should_fail=True).stderr.decode()
    print(stderr)
    return stderr.startswith("Usage: ")
<commit_msg>Remove unnecessary prints_help() helper.<commit_after>import subprocess

class TestResult:
    def __init__(self, result):
        self.raw_result = result
        self.stdout = result.stdout.decode()
        self.stderr = result.stderr.decode()

def run(args, should_fail=False):
    result = subprocess.run(args, capture_output=True)
    if should_fail:
        assert result.returncode > 0
    else:
        assert result.returncode == 0
    return TestResult(result)

def fails(args):
    return run(args, should_fail=True)
<|endoftext|>


<commit_before># Copyright (c) 2015 Intracom S.A. Telecom Solutions. All rights reserved.
#
# This program and the accompanying materials are made available under the
# terms of the Eclipse Public License v1.0 which accompanies this distribution,
# and is available at http://www.eclipse.org/legal/epl-v10.html

""" Reusable functions for processes that are cbench related """

import collections
import common
import logging
import subprocess

def cbench_thread(cbench_run_handler, cbench_cpus, controller_ip,
                  controller_port, threads, sw_per_thread, switches,
                  thr_delay_ms, traf_delay_ms, ms_per_test, internal_repeats,
                  hosts, warmup, mode, cbench_node, succ_msg='',
                  fail_msg='', data_queue=None):

    """ Function executed by cbench thread.

    :param cbench_run_handler: cbench run handler
    :param cbench_cpus: number of cpus allocated by create_cpu_shares
    :param controller_ip: controller IP
    :param controller_port: controller port
    :param threads: number of cbench threads
    :param sw_per_thread: number of switches per thread
    :param switches: number of total switches
    :param thr_delay_ms: delay between thread creation
    :param traf_delay_ms: delay between last thread creation and traffic
    transmission
    :param ms_per_test: test duration of a single cbench loop
    :param internal_repeats: number of cbench loops
    :param hosts: number of simulated hosts
    :param warmup: initial loops to be considered as 'warmup'
    :param mode: cbench mode
    :param cbench_node: named tuple containing 1) cbench_node_ip
    2) cbench_node_ssh_port 3) cbench_node_username 4) cbench_node_password
    :parar succ_msg: message written to data queue when cbench_thread
    succeeds
    :parar fail_msg: message written to data queue when cbench_thread fails
    :param data_queue: data queue where cbench output is posted line by line
    :type cbench_run_handler: str
    :type cbench_cpus: int
    :type controller_ip: str
    :type controller_port: int
    :type threads: int
    :type sw_per_thread: int
    :type switches: int
    :type thr_delay_ms: int
    :type traf_delay_ms: int
    :type ms_per_test: int
    :type internal_repeats: int
    :type hosts: int
    :type warmup: int
    :type mode: str
    :type cbench_node: namedtuple<str,int,str,str>
    :type succ_msg: str
    :type fail_msg: str
    :type data_queue: multiprocessing.Queue
    """

    logging.info('[cbench_thread] cbench thread started')

    try:
        # Opening connection with cbench_node_ip and returning cbench_ssh_client
        cbench_ssh_client =  common.open_ssh_connections([cbench_node])[0]

        run_cbench(cbench_run_handler, cbench_cpus, controller_ip,
                   controller_port, threads.value, sw_per_thread.value,
                   switches.value, thr_delay_ms.value, traf_delay_ms.value,
                   ms_per_test, internal_repeats, hosts.value, warmup, mode,
                   data_queue, cbench_ssh_client)

        # cbench ended, enqueue termination message
        if data_queue is not None:
            data_queue.put(succ_msg, block=True)
        logging.info('[cbench_thread] cbench thread ended successfully')
    except subprocess.CalledProcessError as err:
        if data_queue is not None:
            data_queue.put(fail_msg, block=True)
        logging.error('[cbench_thread] Exception:{0}'.format(str(err)))

    return

def cleanup_cbench(cbench_clean_handler, ssh_client=None):
    """Kills and cleans up cbench built files.

    :param cbench_clean_handler: cleanup handler filepath for cbench.
    :param ssh_client : SSH client provided by paramiko to run the command
    :type cbench_clean_handler: str
    :type ssh_client: paramiko.SSHClient
    """

    common.command_exec_wrapper([cbench_clean_handler],
                         '[cbench_clean_handler]', ssh_client)

def rebuild_cbench(cbench_build_handler, ssh_client=None):
    """Rebuilds cbench.

    :param cbench_build_handler: filepath to the handler that builds cbench
    :param ssh_client : SSH client provided by paramiko to run the command
    :type cbench_build_handler: str
    :type ssh_client: paramiko.SSHClient
    """

    common.command_exec_wrapper([cbench_build_handler],
                         '[cbench_build_handler]', ssh_client)

def run_cbench(cbench_run_handler, cbench_cpus, controller_ip, controller_port,
               threads, sw_per_thread, switches, thr_delay_ms, traf_delay_ms,
               ms_per_test, internal_repeats, hosts, warmup, mode,
               data_queue=None, ssh_client=None):
    """Runs a cbench instance

    :param cbench_run_handler: cbench run handler
    :param cbench_cpus: number of cpus allocated by create_cpu_shares
    :param controller_ip: controller IP for OpenFlow connection
    :param controller_port: controller port for OpenFlow connection
    :param threads: number of cbench threads
    :param sw_per_thread: number of switches per thread
    :param switches: number of total switches
    :param thr_delay_ms: delay between thread creation (in milliseconds)
    :param traf_delay_ms: delay between last thread creation and traffic
    transmission (in milliseconds)
    :param ms_per_test: test duration of a single cbench loop
    :param internal_repeats: number of cbench loops
    :param hosts: number of simulated hosts
    :param warmup: initial loops to be considered as 'warmup'
    :param mode: cbench mode
    :param data_queue: data queue where cbench output is posted line by line
    the cbench process will run.
    :param ssh_client : SSH client provided by paramiko to run the command
    :type cbench_run_handler: str
    :type cbench_cpus: int
    :type controller_ip: str
    :type controller_port: int
    :type threads: int
    :type sw_per_thread: int
    :type switches: int
    :type thr_delay_ms: int
    :type traf_delay_ms: int
    :type ms_per_test: int
    :type internal_repeats: int
    :type hosts: int
    :type warmup: int
    :type mode: int
    :type data_queue: multiprocessing.Queue
    :type ssh_client: paramiko.SSHClient
    """

    cmd_list = ['taskset', '-c', '{0}'.format(cbench_cpus),
                cbench_run_handler, controller_ip, str(controller_port),
                str(threads), str(sw_per_thread), str(switches),
                str(thr_delay_ms), str(traf_delay_ms), str(ms_per_test),
                str(internal_repeats), str(hosts), str(warmup), mode]
    common.command_exec_wrapper(cmd_list, '[cbench_run_handler]', ssh_client,
                         data_queue)

<commit_msg>adding debug messages<commit_after># Copyright (c) 2015 Intracom S.A. Telecom Solutions. All rights reserved.
#
# This program and the accompanying materials are made available under the
# terms of the Eclipse Public License v1.0 which accompanies this distribution,
# and is available at http://www.eclipse.org/legal/epl-v10.html

""" Reusable functions for processes that are cbench related """

import common
import logging
import subprocess

def cbench_thread(cbench_run_handler, cbench_cpus, controller_ip,
                  controller_port, threads, sw_per_thread, switches,
                  thr_delay_ms, traf_delay_ms, ms_per_test, internal_repeats,
                  hosts, warmup, mode, cbench_node, succ_msg='',
                  fail_msg='', data_queue=None):

    """ Function executed by cbench thread.

    :param cbench_run_handler: cbench run handler
    :param cbench_cpus: number of cpus allocated by create_cpu_shares
    :param controller_ip: controller IP
    :param controller_port: controller port
    :param threads: number of cbench threads
    :param sw_per_thread: number of switches per thread
    :param switches: number of total switches
    :param thr_delay_ms: delay between thread creation
    :param traf_delay_ms: delay between last thread creation and traffic
    transmission
    :param ms_per_test: test duration of a single cbench loop
    :param internal_repeats: number of cbench loops
    :param hosts: number of simulated hosts
    :param warmup: initial loops to be considered as 'warmup'
    :param mode: cbench mode
    :param cbench_node: named tuple containing 1) cbench_node_ip
    2) cbench_node_ssh_port 3) cbench_node_username 4) cbench_node_password
    :parar succ_msg: message written to data queue when cbench_thread
    succeeds
    :parar fail_msg: message written to data queue when cbench_thread fails
    :param data_queue: data queue where cbench output is posted line by line
    :type cbench_run_handler: str
    :type cbench_cpus: int
    :type controller_ip: str
    :type controller_port: int
    :type threads: int
    :type sw_per_thread: int
    :type switches: int
    :type thr_delay_ms: int
    :type traf_delay_ms: int
    :type ms_per_test: int
    :type internal_repeats: int
    :type hosts: int
    :type warmup: int
    :type mode: str
    :type cbench_node: namedtuple<str,int,str,str>
    :type succ_msg: str
    :type fail_msg: str
    :type data_queue: multiprocessing.Queue
    """

    logging.info('[cbench_thread] cbench thread started')

    try:
        # Opening connection with cbench_node_ip and returning cbench_ssh_client
        cbench_ssh_client =  common.open_ssh_connections([cbench_node])[0]

        run_cbench(cbench_run_handler, cbench_cpus, controller_ip,
                   controller_port, threads.value, sw_per_thread.value,
                   switches.value, thr_delay_ms.value, traf_delay_ms.value,
                   ms_per_test, internal_repeats, hosts.value, warmup, mode,
                   data_queue, cbench_ssh_client)

        # cbench ended, enqueue termination message
        if data_queue is not None:
            data_queue.put(succ_msg, block=True)
        logging.info('[cbench_thread] cbench thread ended successfully')
    except subprocess.CalledProcessError as err:
        if data_queue is not None:
            data_queue.put(fail_msg, block=True)
        logging.error('[cbench_thread] Exception:{0}'.format(str(err)))

    return

def cleanup_cbench(cbench_clean_handler, ssh_client=None):
    """Kills and cleans up cbench built files.

    :param cbench_clean_handler: cleanup handler filepath for cbench.
    :param ssh_client : SSH client provided by paramiko to run the command
    :type cbench_clean_handler: str
    :type ssh_client: paramiko.SSHClient
    """

    common.command_exec_wrapper([cbench_clean_handler],
                         '[cbench_clean_handler]', ssh_client)

def rebuild_cbench(cbench_build_handler, ssh_client=None):
    """Rebuilds cbench.

    :param cbench_build_handler: filepath to the handler that builds cbench
    :param ssh_client : SSH client provided by paramiko to run the command
    :type cbench_build_handler: str
    :type ssh_client: paramiko.SSHClient
    """

    common.command_exec_wrapper([cbench_build_handler],
                         '[cbench_build_handler]', ssh_client)

def run_cbench(cbench_run_handler, cbench_cpus, controller_ip, controller_port,
               threads, sw_per_thread, switches, thr_delay_ms, traf_delay_ms,
               ms_per_test, internal_repeats, hosts, warmup, mode,
               data_queue=None, ssh_client=None):
    """Runs a cbench instance

    :param cbench_run_handler: cbench run handler
    :param cbench_cpus: number of cpus allocated by create_cpu_shares
    :param controller_ip: controller IP for OpenFlow connection
    :param controller_port: controller port for OpenFlow connection
    :param threads: number of cbench threads
    :param sw_per_thread: number of switches per thread
    :param switches: number of total switches
    :param thr_delay_ms: delay between thread creation (in milliseconds)
    :param traf_delay_ms: delay between last thread creation and traffic
    transmission (in milliseconds)
    :param ms_per_test: test duration of a single cbench loop
    :param internal_repeats: number of cbench loops
    :param hosts: number of simulated hosts
    :param warmup: initial loops to be considered as 'warmup'
    :param mode: cbench mode
    :param data_queue: data queue where cbench output is posted line by line
    the cbench process will run.
    :param ssh_client : SSH client provided by paramiko to run the command
    :type cbench_run_handler: str
    :type cbench_cpus: int
    :type controller_ip: str
    :type controller_port: int
    :type threads: int
    :type sw_per_thread: int
    :type switches: int
    :type thr_delay_ms: int
    :type traf_delay_ms: int
    :type ms_per_test: int
    :type internal_repeats: int
    :type hosts: int
    :type warmup: int
    :type mode: int
    :type data_queue: multiprocessing.Queue
    :type ssh_client: paramiko.SSHClient
    """

    cmd_list = ['taskset', '-c', '{0}'.format(cbench_cpus),
                cbench_run_handler, controller_ip, str(controller_port),
                str(threads), str(sw_per_thread), str(switches),
                str(thr_delay_ms), str(traf_delay_ms), str(ms_per_test),
                str(internal_repeats), str(hosts), str(warmup), mode]
    logging.debug(' '.join(cmd_list))
    common.command_exec_wrapper(cmd_list, '[cbench_run_handler]', ssh_client,
                         data_queue)

<|endoftext|>


<commit_before>import os
import errno
import subprocess
import sys

class InTeXrationTask:

    def __init__(self, url, repository, commit):
        self._url = url
        self._repository = self._convert(repository)
        self._commit = commit
        self._build_dir = self._create_dir('build', commit)
        self._output_dir = self._create_dir('out')

    def _create_dir(self, prefix, suffix=''):
        path = os.path.join(os.getcwd(), prefix, self._repository, suffix)
        try:
            os.makedirs(path)
        except OSError as e:
            if e.errno != errno.EEXIST:
                raise
        return path

    @staticmethod
    def _convert(url):
        if url.startswith('https://'):
            return url + '.git'
        return url

    def _clone(self):
        if subprocess.call(['git', 'clone', self._url, self._build_dir]) != 0:
            raise RuntimeError('git clone failed!')

    def _compile(self, file):
        input_path = os.path.join(self._build_dir, file)
        output_path = self._build_dir
        if subprocess.call(['pdflatex', '-interaction=batchmode', '-aux-directory=' + output_path,
                            '-output-directory=' + output_path, input_path]) != 0:
            raise RuntimeError('pdflatex compilation failed!')

    def run(self):
        self._clone()
        self._compile('main.tex')

def error(*objs):
    print("ERROR: ", *objs, end='\n', file=sys.stderr)
<commit_msg>Bugfixes.<commit_after>import os
import errno
import subprocess
import sys

class InTeXrationTask:

    def __init__(self, url, repository, commit):
        self._url = url
        self._repository = self._convert(repository)
        self._commit = commit
        self._build_dir = self._create_dir('build', commit)
        self._output_dir = self._create_dir('out')

    def _create_dir(self, prefix, suffix=''):
        path = os.path.join(os.getcwd(), prefix, self._repository, suffix)
        try:
            os.makedirs(path)
        except OSError as e:
            if e.errno != errno.EEXIST:
                raise
        return path

    @staticmethod
    def _convert(url):
        if url.startswith('https://'):
            return url + '.git'
        return url

    def _clone(self):
        if subprocess.call(['git', 'clone', self._url, self._build_dir]) != 0:
            raise RuntimeError('git clone failed!')

    def _compile(self, file):
        input_path = os.path.join(self._build_dir, file)
        output_path = self._output_dir
        if subprocess.call(['pdflatex', '-interaction=nonstopmode', '-aux-directory=' + output_path,
                            '-output-directory=' + output_path, input_path]) != 0:
            raise RuntimeError('pdflatex compilation failed!')

    def run(self):
        self._clone()
        self._compile('main.tex')

def error(*objs):
    print("ERROR: ", *objs, end='\n', file=sys.stderr)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import unittest
from mock import MagicMock
from knot import Container, factory, service, provider

class TestContainer(unittest.TestCase):
    def test_returns_return_value_provider(self):
        c = Container()

        def foo(container):
            return 'bar'

        c.add_provider(foo, False)

        self.assertEqual(c.provide('foo'), 'bar')

    def test_returns_value(self):
        c = Container({'value': 'foobar'})

        self.assertEqual(c.provide('value'), 'foobar')

    def test_returns_default(self):
        c = Container()

        self.assertEqual(c.provide('foo', 'bar'), 'bar')

    def test_caches_return_value_provider(self):
        c = Container()

        def foobar(container):
            return {}

        c.add_provider(foobar, True)

        self.assertFalse(c.is_cached('foobar'))
        dict1 = c.provide('foobar')
        dict2 = c.provide('foobar')
        self.assertTrue(c.is_cached('foobar'))

        assert isinstance(dict1, dict)
        assert isinstance(dict2, dict)
        assert dict1 is dict2

    def test_uses_alternative_name(self):
        c = Container()

        def foobar(container):
            return 'foobar'

        c.add_provider(foobar, False, 'alternative')

        self.assertEqual(c.provide('alternative'), 'foobar')

    def test_registers_factory(self):
        c = Container()
        c.add_provider = MagicMock()

        def foo(container):
            pass

        c.add_factory(foo)
        c.add_provider.assert_called_with(foo, False, None)

        c.add_factory(foo, 'alternative')
        c.add_provider.assert_called_with(foo, False, 'alternative')

    def test_registers_factory_with_decorator(self):
        c = Container()
        c.add_factory = MagicMock()

        @factory(c)
        def foo(container):
            pass

        c.add_factory.assert_called_with(foo, None)

        @factory(c, 'alternative')
        def bar(container):
            pass

        c.add_factory.assert_called_with(bar, 'alternative')

    def test_registers_service_with_decorator(self):
        c = Container()
        c.add_service = MagicMock()

        @service(c)
        def foo(container):
            pass

        c.add_service.assert_called_with(foo, None)

        @service(c, 'alternative')
        def bar(container):
            pass

        c.add_service.assert_called_with(bar, 'alternative')

    def test_registers_provider_with_decorator(self):
        c = Container()
        c.add_provider = MagicMock()

        @provider(c, False)
        def foo(container):
            pass

        c.add_provider.assert_called_with(foo, False, None)

        @provider(c, True, 'alternative')
        def bar(container):
            pass

        c.add_provider.assert_called_with(bar, True, 'alternative')

if __name__ == '__main__':
    unittest.main()
<commit_msg>Add test for method to register a service.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import unittest
from mock import MagicMock
from knot import Container, factory, service, provider

class TestContainer(unittest.TestCase):
    def test_returns_return_value_provider(self):
        c = Container()

        def foo(container):
            return 'bar'

        c.add_provider(foo, False)

        self.assertEqual(c.provide('foo'), 'bar')

    def test_returns_value(self):
        c = Container({'value': 'foobar'})

        self.assertEqual(c.provide('value'), 'foobar')

    def test_returns_default(self):
        c = Container()

        self.assertEqual(c.provide('foo', 'bar'), 'bar')

    def test_caches_return_value_provider(self):
        c = Container()

        def foobar(container):
            return {}

        c.add_provider(foobar, True)

        self.assertFalse(c.is_cached('foobar'))
        dict1 = c.provide('foobar')
        dict2 = c.provide('foobar')
        self.assertTrue(c.is_cached('foobar'))

        assert isinstance(dict1, dict)
        assert isinstance(dict2, dict)
        assert dict1 is dict2

    def test_uses_alternative_name(self):
        c = Container()

        def foobar(container):
            return 'foobar'

        c.add_provider(foobar, False, 'alternative')

        self.assertEqual(c.provide('alternative'), 'foobar')

    def test_registers_factory(self):
        c = Container()
        c.add_provider = MagicMock()

        def foo(container):
            pass

        c.add_factory(foo)
        c.add_provider.assert_called_with(foo, False, None)

        c.add_factory(foo, 'alternative')
        c.add_provider.assert_called_with(foo, False, 'alternative')

    def test_registers_service(self):
        c = Container()
        c.add_provider = MagicMock()

        def foo(container):
            pass

        c.add_service(foo)
        c.add_provider.assert_called_with(foo, True, None)

        c.add_service(foo, 'alternative')
        c.add_provider.assert_called_with(foo, True, 'alternative')

    def test_registers_factory_with_decorator(self):
        c = Container()
        c.add_factory = MagicMock()

        @factory(c)
        def foo(container):
            pass

        c.add_factory.assert_called_with(foo, None)

        @factory(c, 'alternative')
        def bar(container):
            pass

        c.add_factory.assert_called_with(bar, 'alternative')

    def test_registers_service_with_decorator(self):
        c = Container()
        c.add_service = MagicMock()

        @service(c)
        def foo(container):
            pass

        c.add_service.assert_called_with(foo, None)

        @service(c, 'alternative')
        def bar(container):
            pass

        c.add_service.assert_called_with(bar, 'alternative')

    def test_registers_provider_with_decorator(self):
        c = Container()
        c.add_provider = MagicMock()

        @provider(c, False)
        def foo(container):
            pass

        c.add_provider.assert_called_with(foo, False, None)

        @provider(c, True, 'alternative')
        def bar(container):
            pass

        c.add_provider.assert_called_with(bar, True, 'alternative')

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Test of "new style" network inference
# Daniel Klein, 5/16/2012

import numpy as np

from Network import Network
from Models import StationaryLogistic, NonstationaryLogistic
from Models import FixedMargins
from Models import alpha_zero
from Experiment import RandomSubnetworks, Results, add_array_stats, rel_mse
from Utility import l2, logit

# Parameters
params = { 'N': 300,
           'B': 1,
           'theta_sd': 1.0,
           'theta_fixed': { 'x_0': 2.0, 'x_1': -1.0 },
           'cov_unif_sd': 1.0,
           'cov_norm_sd': 0.0,
           'cov_disc_sd': 0.0,
           'fisher_information': False,
           'baseline': False,
           'fit_nonstationary': True,
           'fit_method': 'saddlepoint',
           'num_reps': 15,
           'sub_sizes': np.floor(np.logspace(1.0, 2.1, 20)),
           'verbose': False,
           'plot_mse': True,
           'plot_network': False,
           'plot_fit_info': True }

# Set random seed for reproducible output
np.random.seed(137)

# Initialize full network
net = Network(params['N'])
alpha_zero(net)

# Generate covariates and associated coefficients
data_model = NonstationaryLogistic()
for b in range(params['B']):
    name = 'x_%d' % b

    if name in params['theta_fixed']:
        data_model.beta[name] = params['theta_fixed'][name]
    else:
        data_model.beta[name] = np.random.normal(0, params['theta_sd'])

    if params['cov_unif_sd'] > 0.0:
        c = np.sqrt(12) / 2
        def f_x(i_1, i_2):
            return np.random.uniform(-c * params['cov_unif_sd'],
                                     c * params['cov_unif_sd'])
    elif params['cov_norm_sd'] > 0.0:
        def f_x(i_1, i_2):
            return np.random.normal(0, params['cov_norm_sd'])
    elif params['cov_disc_sd'] > 0.0:
        def f_x(i_1, i_2):
            return (params['cov_disc_sd'] *
                    (np.sign(np.random.random() - 0.5)))
    else:
        print 'Error: no covariate distribution specified.'
        sys.exit()

    net.new_edge_covariate(name).from_binary_function_ind(f_x)

# Specify data model as generation permuation networks
net.new_node_covariate_int('r')[:] = 1
net.new_node_covariate_int('c')[:] = 1
data_model = FixedMargins(data_model, 'r', 'c', coverage = 2.0)

if params['fit_nonstationary']:
    fit_model = NonstationaryLogistic()
else:
    fit_model = StationaryLogistic()
for b in data_model.base_model.beta:
    fit_model.beta[b] = 0.0

# Set up recording of results from experiment
results = Results(params['sub_sizes'], params['sub_sizes'], params['num_reps'])
add_array_stats(results)
def true_est_theta_b(b):
    return (lambda d, f: d.base_model.beta[b]), (lambda d, f: f.beta[b])
for b in fit_model.beta:
    # Need to do this hackily to avoid for-loop/lambda-binding weirdness.
    f_true, f_est = true_est_theta_b(b)
    results.new('True theta_{%s}' % b, 'm', f_true)
    results.new('Est. theta_{%s}' % b, 'm', f_est)
results.new('# Active', 'n', lambda n: n.N ** 2)
results.new('Separated', 'm', lambda d, f: f.fit_info['separated'])
if params['fisher_information']:
    def info_theta_b(b):
        def f_info_theta_b(d, f):
            return d.base_model.I_inv['theta_{%s}' % b]
        return f_info_theta_b
    for b in fit_model.beta:
        results.new('Info theta_{%s}' % b, 'm', info_theta_b(b))
if params['baseline']:
    def rel_mse_p_ij(n, d, f):
        P = d.edge_probabilities(n)
        return rel_mse(f.edge_probabilities(n), f.baseline(n), P)
    results.new('Rel. MSE(P_ij)', 'nm', rel_mse_p_ij)
    def rel_mse_logit_p_ij(n, d, f):
        logit_P = logit(d.edge_probabilities(n))
        logit_Q = f.baseline_logit(n)
        return rel_mse(logit(f.edge_probabilities(n)), logit_Q, logit_P)
    results.new('Rel. MSE(logit P_ij)', 'nm', rel_mse_logit_p_ij)

if params['fit_method'] in ['convex_opt', 'conditional',
                            'irls', 'conditional_is']:
    results.new('Wall time (sec.)', 'm', lambda d, f: f.fit_info['wall_time'])
if params['fit_method'] in ['convex_opt', 'conditional', 'conditional_is']:
    def work(f):
        w = 0
        for work_type in ['nll_evals', 'grad_nll_evals', 'cnll_evals']:
            if work_type in f.fit_info:
                w += f.fit_info[work_type]
        return w
    results.new('Work', 'm', lambda d, f: work(f))
    results.new('||ET_final - T||_2', 'm',
                lambda d, f: l2(f.fit_info['grad_nll_final']))

for sub_size in params['sub_sizes']:
    size = (sub_size, sub_size)
    print 'subnetwork size = %s' % str(size)

    gen = RandomSubnetworks(net, size)

    for rep in range(params['num_reps']):
        subnet = gen.sample()
        subnet.generate(data_model)

        if params['fisher_information']:
            data_model.base_model.fisher_information(subnet)

        if params['fit_method'] in ('conditional', 'conditional_is',
                                    'brazzale', 'saddlepoint'):
            fixed_model = FixedMargins(base_model = fit_model, coverage = 0.5)
            fixed_model.check_separated(subnet, samples = 100)
        else:
            fit_model.check_separated(subnet)

        if fit_model.fit_info['separated']:
            print 'Separated, defaulting to theta = 0.'
            for b in data_model.base_model.beta:
                fit_model.beta[b] = 0.0
            fit_model.fit_convex_opt(subnet, fix_beta = True)
        elif params['fit_method'] == 'convex_opt':
            if params['verbose']:
                fit_model.fit_convex_opt(subnet, verbose = True)
                print
            else:
                fit_model.fit_convex_opt(subnet)
        elif params['fit_method'] == 'irls':
            fit_model.fit_irls(subnet)
        elif params['fit_method'] == 'logistic':
            fit_model.fit_logistic(subnet)
        elif params['fit_method'] == 'logistic_l2':
            fit_model.fit_logistic_l2(subnet, prior_precision = 1.0)
        elif params['fit_method'] == 'mh':
            fit_model.fit_mh(subnet)
        elif params['fit_method'] == 'conditional':
            fit_model.fit_conditional(subnet, verbose = params['verbose'])
        elif params['fit_method'] == 'conditional_is':
            fit_model.fit_conditional(subnet, T = 50, verbose = True)
        elif params['fit_method'] == 'composite':
            fit_model.fit_composite(subnet, T = 100, verbose = True)
        elif params['fit_method'] == 'brazzale':
            fit_model.fit_brazzale(subnet, 'x_0')
        elif params['fit_method'] == 'saddlepoint':
            fit_model.fit_saddlepoint(subnet, verbose = params['verbose'])
            fit_model.fit_convex_opt(subnet, fix_beta = True)
        elif params['fit_method'] == 'none':
            pass

        results.record(size, rep, subnet, data_model, fit_model)

# Compute beta MSEs
covariate_mses = []
for b in fit_model.beta:
    name = 'MSE(theta_{%s})' % b
    covariate_mses.append(name)
    results.estimate_mse(name, 'True theta_{%s}' % b, 'Est. theta_{%s}' % b)
results.summary()

# Plot inference performace, in terms of MSE(theta) and MSE(P_ij)
if params['plot_mse']:
    to_plot = []
    if not params['fit_method'] == 'none':
        to_plot.append((['MSE(theta_i)'] + covariate_mses,
                        {'ymin': 0, 'ymax': 0.5, 'plot_mean': True}))
    if params['baseline']:
        to_plot.append(('Rel. MSE(P_ij)',
                        {'ymin': 0, 'ymax': 2, 'baseline': 1}))
        to_plot.append(('Rel. MSE(logit P_ij)',
                        {'ymin':0, 'ymax': 2, 'baseline': 1}))
    to_plot.append(('# Active', {'ymin': 0}))
    to_plot.append(('Separated', {'ymin': 0, 'ymax': 1, 'plot_mean': True}))
    if params['fisher_information']:
        to_plot.append((['Info theta_i'] + \
                        ['Info theta_{%s}' % b for b in fit_model.beta],
                        {'ymin': 0, 'plot_mean': True}))
    results.plot(to_plot)

    to_plot = []
    to_plot.append((['MSE(theta_i)'] + covariate_mses,
                    {'plot_mean': True, 'loglog': True}))
    if params['fisher_information']:
        to_plot.append((['Info theta_i'] + \
                        ['Info theta_{%s}' % b for b in fit_model.beta],
                        {'plot_mean': True, 'loglog': True}))
    results.plot(to_plot)

# Plot network statistics
if params['plot_network']:
    to_plot = [('Density', {'ymin': 0, 'plot_mean': True}),
               (['Row-sum', 'Max row-sum', 'Min row-sum'],
                {'ymin': 0, 'plot_mean': True}),
               (['Col-sum', 'Max col-sum', 'Min col-sum'],
                {'ymin': 0, 'plot_mean': True})]
    results.plot(to_plot)

# Plot convex optimization fitting internal details
if (params['plot_fit_info'] and params['fit_method'] == 'irls'):
    results.plot([('Wall time (sec.)', {'ymin': 0})])
if (params['plot_fit_info'] and
    params['fit_method'] in ['convex_opt', 'conditional', 'conditional_is']):
    results.plot([('Work', {'ymin': 0}),
                  ('Wall time (sec.)', {'ymin': 0}),
                  ('||ET_final - T||_2', {'ymin': 0})])

# Report parameters for the run
print 'Parameters:'
for field in params:
    print '%s: %s' % (field, repr(params[field]))
<commit_msg>Add some code to ignore separation test results<commit_after>#!/usr/bin/env python

# Test of "new style" network inference
# Daniel Klein, 5/16/2012

import numpy as np

from Network import Network
from Models import StationaryLogistic, NonstationaryLogistic
from Models import FixedMargins
from Models import alpha_zero
from Experiment import RandomSubnetworks, Results, add_array_stats, rel_mse
from Utility import l2, logit

# Parameters
params = { 'N': 300,
           'B': 1,
           'theta_sd': 1.0,
           'theta_fixed': { 'x_0': 2.0, 'x_1': -1.0 },
           'cov_unif_sd': 0.0,
           'cov_norm_sd': 1.0,
           'cov_disc_sd': 0.0,
           'fisher_information': False,
           'baseline': False,
           'fit_nonstationary': True,
           'fit_method': 'brazzale',
           'ignore_separation': True,
           'separation_samples': 1,
           'num_reps': 15,
           'sub_sizes': np.floor(np.logspace(1.0, 2.1, 20)),
           'verbose': True,
           'plot_mse': True,
           'plot_network': False,
           'plot_fit_info': True }

# Set random seed for reproducible output
np.random.seed(137)

# Initialize full network
net = Network(params['N'])
alpha_zero(net)

# Generate covariates and associated coefficients
data_model = NonstationaryLogistic()
for b in range(params['B']):
    name = 'x_%d' % b

    if name in params['theta_fixed']:
        data_model.beta[name] = params['theta_fixed'][name]
    else:
        data_model.beta[name] = np.random.normal(0, params['theta_sd'])

    if params['cov_unif_sd'] > 0.0:
        c = np.sqrt(12) / 2
        def f_x(i_1, i_2):
            return np.random.uniform(-c * params['cov_unif_sd'],
                                     c * params['cov_unif_sd'])
    elif params['cov_norm_sd'] > 0.0:
        def f_x(i_1, i_2):
            return np.random.normal(0, params['cov_norm_sd'])
    elif params['cov_disc_sd'] > 0.0:
        def f_x(i_1, i_2):
            return (params['cov_disc_sd'] *
                    (np.sign(np.random.random() - 0.5)))
    else:
        print 'Error: no covariate distribution specified.'
        sys.exit()

    net.new_edge_covariate(name).from_binary_function_ind(f_x)

# Specify data model as generation permuation networks
net.new_node_covariate_int('r')[:] = 1
net.new_node_covariate_int('c')[:] = 1
data_model = FixedMargins(data_model, 'r', 'c', coverage = 2.0)

if params['fit_nonstationary']:
    fit_model = NonstationaryLogistic()
else:
    fit_model = StationaryLogistic()
for b in data_model.base_model.beta:
    fit_model.beta[b] = 0.0

# Set up recording of results from experiment
results = Results(params['sub_sizes'], params['sub_sizes'], params['num_reps'])
add_array_stats(results)
def true_est_theta_b(b):
    return (lambda d, f: d.base_model.beta[b]), (lambda d, f: f.beta[b])
for b in fit_model.beta:
    # Need to do this hackily to avoid for-loop/lambda-binding weirdness.
    f_true, f_est = true_est_theta_b(b)
    results.new('True theta_{%s}' % b, 'm', f_true)
    results.new('Est. theta_{%s}' % b, 'm', f_est)
results.new('# Active', 'n', lambda n: n.N ** 2)
results.new('Separated', 'm', lambda d, f: f.fit_info['separated'])
if params['fisher_information']:
    def info_theta_b(b):
        def f_info_theta_b(d, f):
            return d.base_model.I_inv['theta_{%s}' % b]
        return f_info_theta_b
    for b in fit_model.beta:
        results.new('Info theta_{%s}' % b, 'm', info_theta_b(b))
if params['baseline']:
    def rel_mse_p_ij(n, d, f):
        P = d.edge_probabilities(n)
        return rel_mse(f.edge_probabilities(n), f.baseline(n), P)
    results.new('Rel. MSE(P_ij)', 'nm', rel_mse_p_ij)
    def rel_mse_logit_p_ij(n, d, f):
        logit_P = logit(d.edge_probabilities(n))
        logit_Q = f.baseline_logit(n)
        return rel_mse(logit(f.edge_probabilities(n)), logit_Q, logit_P)
    results.new('Rel. MSE(logit P_ij)', 'nm', rel_mse_logit_p_ij)

if params['fit_method'] in ['convex_opt', 'conditional',
                            'irls', 'conditional_is']:
    results.new('Wall time (sec.)', 'm', lambda d, f: f.fit_info['wall_time'])
if params['fit_method'] in ['convex_opt', 'conditional', 'conditional_is']:
    def work(f):
        w = 0
        for work_type in ['nll_evals', 'grad_nll_evals', 'cnll_evals']:
            if work_type in f.fit_info:
                w += f.fit_info[work_type]
        return w
    results.new('Work', 'm', lambda d, f: work(f))
    results.new('||ET_final - T||_2', 'm',
                lambda d, f: l2(f.fit_info['grad_nll_final']))

for sub_size in params['sub_sizes']:
    size = (sub_size, sub_size)
    print 'subnetwork size = %s' % str(size)

    gen = RandomSubnetworks(net, size)

    for rep in range(params['num_reps']):
        subnet = gen.sample()
        subnet.generate(data_model)

        if params['fisher_information']:
            data_model.base_model.fisher_information(subnet)

        if params['fit_method'] in ('conditional', 'conditional_is',
                                    'brazzale', 'saddlepoint'):
            fixed_model = FixedMargins(base_model = fit_model, coverage = 0.5)
            fixed_model.check_separated(subnet,
                                        samples = params['separation_samples'])
        else:
            fit_model.check_separated(subnet)

        if not params['ignore_separation'] and fit_model.fit_info['separated']:
            print 'Separated, defaulting to theta = 0.'
            for b in data_model.base_model.beta:
                fit_model.beta[b] = 0.0
            fit_model.fit_convex_opt(subnet, fix_beta = True)
        elif params['fit_method'] == 'convex_opt':
            if params['verbose']:
                fit_model.fit_convex_opt(subnet, verbose = True)
                print
            else:
                fit_model.fit_convex_opt(subnet)
        elif params['fit_method'] == 'irls':
            fit_model.fit_irls(subnet)
        elif params['fit_method'] == 'logistic':
            fit_model.fit_logistic(subnet)
        elif params['fit_method'] == 'logistic_l2':
            fit_model.fit_logistic_l2(subnet, prior_precision = 1.0)
        elif params['fit_method'] == 'mh':
            fit_model.fit_mh(subnet)
        elif params['fit_method'] == 'conditional':
            fit_model.fit_conditional(subnet, verbose = params['verbose'])
        elif params['fit_method'] == 'conditional_is':
            fit_model.fit_conditional(subnet, T = 50, verbose = True)
        elif params['fit_method'] == 'composite':
            fit_model.fit_composite(subnet, T = 100, verbose = True)
        elif params['fit_method'] == 'brazzale':
            fit_model.fit_brazzale(subnet, 'x_0', verbose = params['verbose'])
        elif params['fit_method'] == 'saddlepoint':
            fit_model.fit_saddlepoint(subnet, verbose = params['verbose'])
            fit_model.fit_convex_opt(subnet, fix_beta = True)
        elif params['fit_method'] == 'none':
            pass

        results.record(size, rep, subnet, data_model, fit_model)

# Compute beta MSEs
covariate_mses = []
for b in fit_model.beta:
    name = 'MSE(theta_{%s})' % b
    covariate_mses.append(name)
    results.estimate_mse(name, 'True theta_{%s}' % b, 'Est. theta_{%s}' % b)
results.summary()

# Plot inference performace, in terms of MSE(theta) and MSE(P_ij)
if params['plot_mse']:
    to_plot = []
    if not params['fit_method'] == 'none':
        to_plot.append((['MSE(theta_i)'] + covariate_mses,
                        {'ymin': 0, 'ymax': 0.5, 'plot_mean': True}))
    if params['baseline']:
        to_plot.append(('Rel. MSE(P_ij)',
                        {'ymin': 0, 'ymax': 2, 'baseline': 1}))
        to_plot.append(('Rel. MSE(logit P_ij)',
                        {'ymin':0, 'ymax': 2, 'baseline': 1}))
    to_plot.append(('# Active', {'ymin': 0}))
    to_plot.append(('Separated', {'ymin': 0, 'ymax': 1, 'plot_mean': True}))
    if params['fisher_information']:
        to_plot.append((['Info theta_i'] + \
                        ['Info theta_{%s}' % b for b in fit_model.beta],
                        {'ymin': 0, 'plot_mean': True}))
    results.plot(to_plot)

    to_plot = []
    to_plot.append((['MSE(theta_i)'] + covariate_mses,
                    {'plot_mean': True, 'loglog': True}))
    if params['fisher_information']:
        to_plot.append((['Info theta_i'] + \
                        ['Info theta_{%s}' % b for b in fit_model.beta],
                        {'plot_mean': True, 'loglog': True}))
    results.plot(to_plot)

# Plot network statistics
if params['plot_network']:
    to_plot = [('Density', {'ymin': 0, 'plot_mean': True}),
               (['Row-sum', 'Max row-sum', 'Min row-sum'],
                {'ymin': 0, 'plot_mean': True}),
               (['Col-sum', 'Max col-sum', 'Min col-sum'],
                {'ymin': 0, 'plot_mean': True})]
    results.plot(to_plot)

# Plot convex optimization fitting internal details
if (params['plot_fit_info'] and params['fit_method'] == 'irls'):
    results.plot([('Wall time (sec.)', {'ymin': 0})])
if (params['plot_fit_info'] and
    params['fit_method'] in ['convex_opt', 'conditional', 'conditional_is']):
    results.plot([('Work', {'ymin': 0}),
                  ('Wall time (sec.)', {'ymin': 0}),
                  ('||ET_final - T||_2', {'ymin': 0})])

# Report parameters for the run
print 'Parameters:'
for field in params:
    print '%s: %s' % (field, repr(params[field]))
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import sys
import argparse

from syntcomp.syntcomp_constants import REALIZABLE_RC, UNKNOWN_RC
from tests.common import run_benchmark

realizable = [
    # "ctl/prolonged_full_arb.py --direct --size 7",
    "ctl/resettable_arbiter.py --direct --size 3",
    "ctl/self_loop_arbiter.py --direct --size 4",
    "ctl/full_arb.py --direct --size 4",
    "ctl/non_det_arb.py --direct --size 4",
    "ctl/delayed_full_arb.py --direct --size 5",
]

unknown = [  # no model should be found for these
    "ctl/resettable_arbiter.py --direct --size 2",
    "ctl/self_loop_arbiter.py --direct --size 3",
    "ctl/full_arb.py --direct --size 3",
    "ctl/non_det_arb.py --direct --size 3",
    "ctl/delayed_full_arb.py --direct --size 4",
]

def _get_status(b):
    if b in realizable:
        return REALIZABLE_RC
    if b in unknown:
        return UNKNOWN_RC
    assert 0

def main():
    parser = argparse.ArgumentParser(description='_Functional_ tests runner for star.py '
                                                 'For _unit_ tests -- run with nosetests3')
    parser.add_argument('--nonstop', action='store_true', required=False, default=False,
                        help="don't stop on error")

    args = parser.parse_args(sys.argv[1:])
    print(args)

    all_passed = True
    for benchmark in realizable + unknown:
        # TODO: should be separated into two separate loops,
        #       for realizable benchmarks: set size_expected
        #       for unknown: use None
        result = run_benchmark('star.py', benchmark, _get_status(benchmark), None)
        all_passed &= result

        if not args.nonstop and result is False:
            exit(1)

    print('-' * 80)
    print(['SOME TESTS FAILED', 'ALL TESTS PASSED'][all_passed])

if __name__ == '__main__':
    exit(main())
<commit_msg>minor<commit_after>#!/usr/bin/env python3

import sys
import argparse

from syntcomp.syntcomp_constants import REALIZABLE_RC, UNKNOWN_RC
from tests.common import run_benchmark

# TODO: test the "--aht" version

realizable = [
    # "ctl/prolonged_full_arb.py --direct --size 7",
    "ctl/resettable_arbiter.py --direct --size 3",
    "ctl/self_loop_arbiter.py --direct --size 4",
    "ctl/full_arb.py --direct --size 4",
    "ctl/non_det_arb.py --direct --size 4",
    "ctl/delayed_full_arb.py --direct --size 5",
]

unknown = [  # no model should be found for these
    "ctl/resettable_arbiter.py --direct --size 2",
    "ctl/self_loop_arbiter.py --direct --size 3",
    "ctl/full_arb.py --direct --size 3",
    "ctl/non_det_arb.py --direct --size 3",
    "ctl/delayed_full_arb.py --direct --size 4",
]

def _get_status(b):
    if b in realizable:
        return REALIZABLE_RC
    if b in unknown:
        return UNKNOWN_RC
    assert 0

def main():
    parser = argparse.ArgumentParser(description='_Functional_ tests runner for star.py '
                                                 'For _unit_ tests -- run with nosetests3')
    parser.add_argument('--nonstop', action='store_true', required=False, default=False,
                        help="don't stop on error")

    args = parser.parse_args(sys.argv[1:])
    print(args)

    all_passed = True
    for benchmark in realizable + unknown:
        # TODO: should be separated into two separate loops,
        #       for realizable benchmarks: set size_expected
        #       for unknown: use None
        result = run_benchmark('star.py', benchmark, _get_status(benchmark), None)
        all_passed &= result

        if not args.nonstop and result is False:
            exit(1)

    print('-' * 80)
    print(['SOME TESTS FAILED', 'ALL TESTS PASSED'][all_passed])

if __name__ == '__main__':
    exit(main())
<|endoftext|>


<commit_before>__author__ = 'Stephanie Dolbec'

import sys
import time
from pymuse.ios import OpenBCIIO, MuseIO, MuseIOError
from pymuse.signals import MultiChannelSignal
from pymuse.pipeline import Analyzer

def main():
    # initialization of variables
    signals, viewers = dict(), dict()

    # EEG signal
    signal_eeg = MultiChannelSignal(length=2000,
                                    estimated_acquisition_freq=250.0,
                                    label_channels=['P7', 'PO7', 'O1', 'O2', 'PO8', 'P8'])

    signals['eeg'] = signal_eeg

    # Initializing the analyzer
    """
    list_process=['ButterFilter',
                                      'WriteToFile',
                                      'FFT'],
                        list_params=[{'filter_type': 'bandpass', 'order': 1, 'cutoff_frequency': '0.25,35', 'acquisition_freq': 220.0},
                                     {'file_name': '/Users/benjamindeleener/data/test_muse/test.csv', 'save_delay': 180.0},
                                     None],
                                     {'filter_type': 'highpass', 'order': 1, 'cutoff_frequency': '0.5', 'acquisition_freq': 250.0},
    """
    pipeline = Analyzer(signal=signals['eeg'],
                        window_duration=500,
                        analysis_frequency=20.0,
                        list_process=['ButterFilter',
                                      'ButterFilter',
                                      'ButterFilter',
                                      'WriteToFile'],
                        list_params=[{'filter_type': 'bandstop', 'order': 3, 'cutoff_frequency': '40.0,80.0',
                                      'acquisition_freq': 250.0},
                                     {'filter_type': 'lowpass', 'order': 3, 'cutoff_frequency': '50.0',
                                      'acquisition_freq': 250.0},
                                     {'filter_type': 'highpass', 'order': 3, 'cutoff_frequency': '1.0',
                                      'acquisition_freq': 250.0},
                                     {'file_name': '/Users/stephaniedolbec/data/test_ttih/acquisitions/'
                                                   'no_contact/20170327/data_7.csv', 'save_delay': 10.0}],
                        processes_to_visualize=[])

    # Initializing the server
    try:
        #server = MuseIO(port=5001, signal=signals)
        #'/dev/tty.usbserial-DB00MF30'
        server = OpenBCIIO(port_name='/dev/tty.usbserial-DB00MHSH', baud=115200, signal=signals, index_channels=[0, 1, 2, 3, 4, 5])
    except MuseIOError, err:
        print str(err)
        sys.exit(1)

    pipeline.start()

    # Starting the server
    try:
        server.start()
        while 1:
            time.sleep(0.01)
    except KeyboardInterrupt:
        print "\nEnd of program: Caught KeyboardInterrupt"
        sys.exit(0)

if __name__ == "__main__":
    main()
<commit_msg>improvements on ttih<commit_after>__author__ = 'Stephanie Dolbec'

import sys
import time
from pymuse.ios import OpenBCIIO, MuseIO, MuseIOError
from pymuse.signals import MultiChannelSignal
from pymuse.pipeline import Analyzer
import easygui

def main():
    # initialization of variables
    signals, viewers = dict(), dict()

    # EEG signal
    signal_eeg = MultiChannelSignal(length=2000,
                                    estimated_acquisition_freq=250.0,
                                    label_channels=['P7', 'PO7', 'O1', 'O2', 'PO8', 'P8'])

    signals['eeg'] = signal_eeg

    # Initializing the analyzer
    """
    list_process=['ButterFilter',
                                      'WriteToFile',
                                      'FFT'],
                        list_params=[{'filter_type': 'bandpass', 'order': 1, 'cutoff_frequency': '0.25,35', 'acquisition_freq': 220.0},
                                     {'file_name': '/Users/benjamindeleener/data/test_muse/test.csv', 'save_delay': 180.0},
                                     None],
                                     {'filter_type': 'highpass', 'order': 1, 'cutoff_frequency': '0.5', 'acquisition_freq': 250.0},
    """

    path = '/Users/stephaniedolbec/Desktop'
    test_date = '20170419' # YYYYMMDD

    # TODO : write function
    msg = "Please fill in the fields below"
    title = "Intialization of data acquisition(" + test_date + ")"
    fieldNames = ["Initials", "Frequency (Hz)", "Test number"]
    fieldValues = easygui.multenterbox(msg, title, fieldNames)
    if fieldValues is None:
        exit(0)
    # make sure that none of the fields were left blank
    while 1:
        errmsg = ""
        for i, name in enumerate(fieldNames):
            if fieldValues[i].strip() == "":
                errmsg += "{} is a required field.\n\n".format(name)
        if errmsg == "":
            break  # no problems found
        fieldValues = easygui.multenterbox(errmsg, title, fieldNames, fieldValues)
        if fieldValues is None:
            break
    name = format(fieldValues[0])
    freq = format(fieldValues[1])
    test_number = format(fieldValues[2])
    file_name = '/data_' + test_date + '_' + name + '_' + test_number + '_' + freq + 'Hz.csv'

    pipeline = Analyzer(signal=signals['eeg'],
                        window_duration=2500,
                        analysis_frequency=10.0,
                        list_process=['ButterFilter',
                                      'ButterFilter',
                                      'ButterFilter',
                                      'WriteToFile'],
                        list_params=[{'filter_type': 'bandstop', 'order': 3, 'cutoff_frequency': '40.0,80.0',
                                      'acquisition_freq': 250.0},
                                     {'filter_type': 'lowpass', 'order': 3, 'cutoff_frequency': '50.0',
                                      'acquisition_freq': 250.0},
                                     {'filter_type': 'highpass', 'order': 3, 'cutoff_frequency': '1.0',
                                      'acquisition_freq': 250.0},
                                     {'file_name': path + file_name, 'save_delay': 10.0}],
                        processes_to_visualize=[])

    # Initializing the server
    try:
        #server = MuseIO(port=5001, signal=signals)
        #'/dev/tty.usbserial-DB00MF30'
        server = OpenBCIIO(port_name='/dev/tty.usbserial-DB00MHSH', baud=115200, signal=signals, index_channels=[0, 1, 2, 3, 4, 5])
    except MuseIOError, err:
        print str(err)
        sys.exit(1)

    pipeline.start()

    # Starting the server
    try:
        server.start()
        while 1:
            time.sleep(0.01)
    except KeyboardInterrupt:
        print "\nEnd of program: Caught KeyboardInterrupt"
        sys.exit(0)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import unittest
import itertools

import ttt
from ttt import GameStates

"""
give assingment
after 15 minutes pause, discussion time
* how many possible game states (9!)
* what possible states can board be in
* introduce GameStates
* groups must pull and merge
give another 15 minutes
"""

def all_boards():
    for a in itertools.product('xo.', repeat=3):
        for b in itertools.product('xo.', repeat=3):
            for c in itertools.product('xo.', repeat=3):
                yield a+b+c

class UberTestTTT(unittest.TestCase):

    def test_bad_boards(self):

        # or something
        self.assertEqual(GameStates.invalid, ttt.game_state(None))
        self.assertEqual(GameStates.invalid, ttt.game_state("xo.ox.ox."))
        self.assertEqual(GameStates.invalid, ttt.game_state([]))
        self.assertEqual(GameStates.invalid, ttt.game_state(['.']*8))
        self.assertEqual(GameStates.invalid, ttt.game_state(['.']*10))
        self.assertEqual(GameStates.invalid, ttt.game_state(['x']*2 + ['.']*7))
        self.assertEqual(GameStates.invalid, ttt.game_state(['o']*2 + ['.']*7))
        # x goes first
        self.assertEqual(GameStates.invalid, ttt.game_state(['o']*1 + ['.']*8))

    def test_all_perms(self):
        results = {}

        for board in all_boards():
            # if board == list("xoxxoxoxo"):
            #     print board
            e = ttt.game_state(list(board))
            results[e] = results.get(e, 0) + 1

        # {0: 13815, 2: 920, 1: 4520, 3: 412, 4: 16}
        print results
        self.assertEqual(3**9, sum(results.values()))

        self.assertEqual(13815, results[GameStates.invalid])
        self.assertEqual(4520, results[GameStates.unfinished])
        self.assertEqual(920, results[GameStates.x_wins])
        self.assertEqual(412, results[GameStates.o_wins])
        self.assertEqual(16, results[GameStates.draw])
<commit_msg>whitespace<commit_after>import unittest
import itertools

import ttt
from ttt import GameStates

"""
give assingment
after 15 minutes pause, discussion time
* how many possible game states (9!)
* what possible states can board be in
* introduce GameStates
* groups must pull and merge
give another 15 minutes
"""

def all_boards():
    for a in itertools.product('xo.', repeat=3):
        for b in itertools.product('xo.', repeat=3):
            for c in itertools.product('xo.', repeat=3):
                yield a + b + c

class UberTestTTT(unittest.TestCase):

    def test_bad_boards(self):

        # or something
        self.assertEqual(GameStates.invalid, ttt.game_state(None))
        self.assertEqual(GameStates.invalid, ttt.game_state("xo.ox.ox."))
        self.assertEqual(GameStates.invalid, ttt.game_state([]))
        self.assertEqual(GameStates.invalid, ttt.game_state(['.']*8))
        self.assertEqual(GameStates.invalid, ttt.game_state(['.']*10))
        self.assertEqual(GameStates.invalid, ttt.game_state(['x']*2 + ['.']*7))
        self.assertEqual(GameStates.invalid, ttt.game_state(['o']*2 + ['.']*7))
        # x goes first
        self.assertEqual(GameStates.invalid, ttt.game_state(['o']*1 + ['.']*8))

    def test_all_perms(self):
        results = {}

        for board in all_boards():
            # if board == list("xoxxoxoxo"):
            #     print board
            e = ttt.game_state(list(board))
            results[e] = results.get(e, 0) + 1

        # {0: 13815, 2: 920, 1: 4520, 3: 412, 4: 16}
        print results
        self.assertEqual(3**9, sum(results.values()))

        self.assertEqual(13815, results[GameStates.invalid])
        self.assertEqual(4520, results[GameStates.unfinished])
        self.assertEqual(920, results[GameStates.x_wins])
        self.assertEqual(412, results[GameStates.o_wins])
        self.assertEqual(16, results[GameStates.draw])
<|endoftext|>