<commit_before>#!/usr/bin/env python

import logging
import pandas as pd
from joblib import Parallel, delayed
import fylearn.fpcga as fpcga
import fylearn.frr as frr
import numpy as np
import fylearn.fuzzylogic as fl
from sklearn.preprocessing import MinMaxScaler
from sklearn.cross_validation import cross_val_score
from sklearn import tree, svm, neighbors
import paper

RUNS = 1

def run_one_classifier(l, X, y):
    test_scores = []
    training_scores = []
    for i in range(RUNS):
        test, training = paper.cross_val_score(l, X, y, cv=10)
        test_scores.extend(test)
        training_scores.extend(training)

    return ( np.mean(test_scores), np.std(test_scores), np.mean(training_scores), np.std(training_scores) )

def run_one_dataset(logger, L, X, y):

    # iterate learners
    output = []
    for l in L:
        # cross validation
        scores = Parallel(n_jobs=-1)(delayed(paper.cross_val_score)(l, X, y, cv=10) for l in L)
        output.extend(scores)

        #print "---"
        #print "dataset", dataset
        logger.info("learner=%s mean=%f std=%f, mean=%f std=%f" % (str(l), scores[0], scores[1], scores[2], scores[3]))

    return output

if __name__ == "__main__":

    logger = logging.getLogger("fpcga")
    logger.setLevel(logging.INFO)
    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    ch = logging.FileHandler('info.txt')
    ch.setLevel(logging.INFO)
    ch.setFormatter(formatter)
    logger.addHandler(ch)

    L = map(lambda x: x[1], paper.learners)

    # iterate over datsets
    import paper
    for dataset in paper.datasets:
        X, y = paper.load(paper.path(dataset))

        output = run_one_dataset(logger, L, X, y)
        print ",".join(dataset) + "," + ",".join(map(str, output))
        #print "%s & %s \\\\" % (dataset[0], " & ".join(output))
<commit_msg>fixup printing<commit_after>#!/usr/bin/env python

import logging
import pandas as pd
from joblib import Parallel, delayed
import fylearn.fpcga as fpcga
import fylearn.frr as frr
import numpy as np
import fylearn.fuzzylogic as fl
from sklearn.preprocessing import MinMaxScaler
from sklearn.cross_validation import cross_val_score
from sklearn import tree, svm, neighbors
import paper

RUNS = 1

def run_one_classifier(l, X, y):
    test_scores = []
    training_scores = []
    for i in range(RUNS):
        test, training = paper.cross_val_score(l, X, y, cv=10)
        test_scores.extend(test)
        training_scores.extend(training)

    return ( np.mean(test_scores), np.std(test_scores), np.mean(training_scores), np.std(training_scores) )

def run_one_dataset(logger, L, X, y):
    # cross validation
    scores = Parallel(n_jobs=-1)(delayed(run_one_classifier)(l, X, y) for l in L)
    return scores

if __name__ == "__main__":

    logger = logging.getLogger("fpcga")
    logger.setLevel(logging.INFO)
    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    ch = logging.FileHandler('info.txt')
    ch.setLevel(logging.INFO)
    ch.setFormatter(formatter)
    logger.addHandler(ch)

    L = map(lambda x: x[1], paper.learners)

    def test_acc_std(x):
        return "$%.2f (%.2f)$" % (x[0] * 100.0, x[1] * 100.0)

    # iterate over datsets
    import paper
    for dataset in paper.datasets:
        X, y = paper.load(paper.path(dataset))

        output = run_one_dataset(logger, L, X, y)
        print ",".join(dataset) + "," + ",".join(map(test_acc_std, output))
        #print "%s & %s \\\\" % (dataset[0], " & ".join(output))
<|endoftext|>


<commit_before>HEADER = """" Name:     hydrangea.vim --- Hydrangea theme for Vim
" Author:   Yuta Taniguchi <yuta.taniguchi.y.t@gmail.com>
" URL:      https://github.com/yuttie/hydrangea-vim
" Version:  3.0.0
" License:  The MIT License (MIT)
"     Copyright (c) 2015-2016 Yuta Taniguchi
"
"     Permission is hereby granted, free of charge, to any person obtaining a copy
"     of this software and associated documentation files (the "Software"), to deal
"     in the Software without restriction, including without limitation the rights
"     to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
"     copies of the Software, and to permit persons to whom the Software is
"     furnished to do so, subject to the following conditions:
"
"     The above copyright notice and this permission notice shall be included in
"     all copies or substantial portions of the Software.
"
"     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
"     IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
"     FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
"     AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
"     LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
"     OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
"     THE SOFTWARE.
"""

try:
    import vim

    def execute(cmd):
        vim.command(cmd)
except ImportError:
    print(HEADER)

    def execute(cmd):
        print(cmd)

# Palette
base03       = ("#1d1f25", 234)
base02       = ("#2c303a", 236)
base01       = ("#343945", 237)
base00       = ("#3e4451", 238)
base0        = ("#555d6f",  59)
base1        = ("#626c82", 242)
base2        = ("#c6d0de", 252)
base3        = ("#e8eff8", 255)
red          = ("#e91e63", 197)
red_dark     = ("#792340",  52)
teal         = ("#36c2c2",  44)
teal_dark    = ("#134242",  23)
blue         = ("#537dd5",  68)
blue_light   = ('#91b5ff', 111)
violet       = ("#996ddb",  98)
violet_light = ("#ceadff", 183)
magenta      = ("#e242ac", 162)
diff_add_fg  = ("#52c4ff",  81)
diff_add_bg  = ("#0d435f",  24)
diff_add_hi  = ("#97dcff", 117)
diff_rm_fg   = ("#e242ac", 162)
diff_rm_bg   = ("#66184c",  53)
diff_rm_hi   = ("#fface3", 218)

# Definitions
color = {}
color['Normal']       = {'fg': base2,            'bg': base02,                                   }
color['Cursor']       = {'fg': 'NONE',           'bg': base3,                 'deco': 'NONE'     }
color['CursorIM']     = {'fg': 'NONE',           'bg': base3,                                    }
color['CursorLine']   = {'fg': 'NONE',           'bg': base01,                'deco': 'bold'     }
color['CursorColumn'] = {'fg': 'NONE',           'bg': base01,                'deco': 'NONE'     }
color['Visual']       = {'fg': 'NONE',           'bg': base00,                'deco': 'NONE'     }
color['VisualNOS']    = {'fg': 'fg',                                          'deco': 'underline'}

color['Folded']       = {'fg': ('#a0a8b0', 248), 'bg': ('#384048', 238),      'deco': 'NONE'     }
color['FoldColumn']   = {'fg': ('#a0a8b0', 248), 'bg': ('#384048', 238),      'deco': 'NONE'     }
color['Folded']       = {'fg': ('#d0e0f0', 253), 'bg': ('#202020', 234),      'deco': 'NONE'     }
color['FoldColumn']   = {'fg': ('#c0c0d0',   7), 'bg': ('#363946', 237),      'deco': 'NONE'     }
color['Title']        = {'fg': ('#f6f3e8', 255), 'bg': 'NONE',                'deco': 'bold'     }
color['StatusLine']   = {'fg': base2,            'bg': base00,                'deco': 'NONE'     }
color['StatusLineNC'] = {'fg': base0,            'bg': base01,                'deco': 'NONE'     }
color['VertSplit']    = {'fg': base01,           'bg': base01,                'deco': 'NONE'     }
color['LineNr']       = {'fg': base0,            'bg': base01,                'deco': 'NONE'     }
color['CursorLineNr'] = {'fg': base3,            'bg': base0,                 'deco': 'bold'     }
color['SpecialKey']   = {'fg': ('#808080',   8), 'bg': ('#343434',   8),      'deco': 'NONE'     }
color['NonText']      = {'fg': base0,            'bg': base02,                'deco': 'NONE'     }
color['MatchParen']   = {'fg': red,              'bg': 'NONE',                'deco': 'bold'     }

color['Comment']      = {'fg': base0,                                         'deco': 'NONE'     }
color['Constant']     = {'fg': teal,             "bg": teal_dark,             'deco': 'NONE'     }
color['String']       = 'Constant'
color['Number']       = 'Constant'
color['Identifier']   = {'fg': base3,                                         'deco': 'bold'     }
color['Function']     = {'fg': base3,                                         'deco': 'bold'     }
color['Statement']    = {'fg': blue,                                          'deco': 'bold'     }
color['Operator']     = {'fg': magenta,                                       'deco': 'NONE'     }
color['Include']      = {'fg': violet,                                        'deco': 'NONE'     }
color['PreProc']      = {'fg': violet_light,                                  'deco': 'NONE'     }
color['Type']         = {'fg': magenta,                                       'deco': 'NONE'     }
color['StorageClass'] = {'fg': blue,                                          'deco': 'bold'     }
color['Structure']    = {'fg': magenta,                                       'deco': 'NONE'     }
color['Typedef']      = {'fg': blue,                                          'deco': 'bold'     }
color['Special']      = {'fg': 'NONE',           'bg': 'NONE',                'deco': 'bold'     }
color['Underlined']   = {'fg': 'fg',                                          'deco': 'underline'}
color['Ignore']       = {'fg': 'bg'                                                              }
color['Error']        = {'fg': red,              'bg': red_dark,              'deco': 'bold'     }
color['Todo']         = {'fg': base2,            'bg': base02,                'deco': 'bold'     }

color['IncSearch']    = {'fg': base3,            'bg': violet_light,          'deco': 'NONE'     }
color['Search']       = {'fg': base3,            'bg': magenta,               'deco': 'NONE'     }
color['Pmenu']        = {'fg': base2,            'bg': base01,                'deco': 'NONE'     }
color['PmenuSel']     = {'fg': base3,            'bg': base00,                'deco': 'bold'     }
color['PmenuSbar']    = {                        'bg': base01,                'deco': 'NONE'     }
color['PmenuThumb']   = {                        'bg': base0,                 'deco': 'NONE'     }
color['TabLine']      = {'fg': base0,            'bg': base01,                'deco': 'NONE'     }
color['TabLineSel']   = {'fg': base02,           'bg': blue,                  'deco': 'bold'     }
color['TabLineFill']  = {'fg': base01,           'bg': base02,                'deco': 'underline'}

color['SpellBad']     = {                                                     'deco': 'undercurl'}
color['SpellCap']     = {                                                     'deco': 'undercurl'}
color['SpellRare']    = {                                                     'deco': 'undercurl'}
color['SpellLocal']   = {                                                     'deco': 'undercurl'}

color['DiffAdd']      = {'fg': diff_add_fg,      'bg': diff_add_bg,           'deco': 'NONE'     }
color['DiffChange']   = {'fg': diff_rm_fg,       'bg': diff_rm_bg,            'deco': 'NONE'     }
color['DiffDelete']   = {'fg': diff_rm_fg,       'bg': diff_rm_bg,            'deco': 'NONE'     }
color['DiffText']     = {'fg': diff_rm_hi,       'bg': diff_rm_bg,            'deco': 'bold'     }

color['diffAdded']    = {'fg': diff_add_fg,      'bg': diff_add_bg,           'deco': 'NONE'     }
color['diffRemoved']  = {'fg': diff_rm_fg,       'bg': diff_rm_bg,            'deco': 'NONE'     }

color['Directory']    = {'fg': ('#c0e0b0', 151),                              'deco': 'NONE'     }
color['ErrorMsg']     = {'fg': red,              'bg': 'NONE',                'deco': 'NONE'     }
color['SignColumn']   = {'fg': ('#9fafaf', 145), 'bg': color['LineNr']['bg'], 'deco': 'NONE'     }
color['MoreMsg']      = {'fg': ('#2e8b57',  29),                              'deco': 'NONE'     }
color['ModeMsg']      = {'fg': ('#76d5f8',  81), 'bg': 'NONE',                'deco': 'NONE'     }
color['Question']     = {'fg': 'fg',                                          'deco': 'NONE'     }
color['WarningMsg']   = {'fg': red,                                           'deco': 'NONE'     }
color['WildMenu']     = {'fg': base3,            'bg': base0,                 'deco': 'bold'     }
color['ColorColumn']  = {'fg': 'NONE',           'bg': ('#403630', 237),      'deco': 'NONE'     }

# GitGutter
color['GitGutterAdd']    = {'fg': diff_add_fg, 'bg': color['SignColumn']['bg'], 'deco': 'bold'}
color['GitGutterChange'] = {'fg': diff_rm_fg,  'bg': color['SignColumn']['bg'], 'deco': 'bold'}
color['GitGutterDelete'] = {'fg': diff_rm_fg,  'bg': color['SignColumn']['bg'], 'deco': 'bold'}

# make
color['makeIdent']      = 'Type'
color['makeSpecTarget'] = 'Special'
color['makeTarget']     = 'Function'
color['makeCommands']   = 'NONE'

# php
color['phpVarSelector'] = 'Identifier'
color['phpIdentifier']  = 'NONE'
color['phpFunctions']   = 'NONE'
color['phpClasses']     = 'NONE'
color['phpFunction']    = 'Function'
color['phpClass']       = 'Type'

# rust
color['rustFuncCall'] = {'fg': blue_light}
color['rustQuestionMark'] = 'Operator'

# vim
color['vimVar'] = 'NONE'

# Apply
execute("""hi clear
if exists('syntax_on')
  syntax reset
endif
let g:colors_name = 'hydrangea'

set background=dark
""")

for name, cdef in color.items():
    if type(cdef) is str:
        # The definition is a string
        if cdef == 'NONE':
            # Disable
            execute('hi ' + name + ' NONE')
            execute('hi link ' + name + ' NONE')
        else:
            # Link
            execute('hi link ' + name + ' ' + cdef)
    elif type(cdef) is dict and len(cdef) > 0:
        # The definition is a dictionary
        def2 = {}
        for key, val in cdef.items():
            if key == 'fg':
                if type(val) is tuple:
                    def2['guifg'] = val[0]
                    def2['ctermfg'] = val[1]
                elif type(val) is str and val == 'NONE':
                    def2['guifg'] = val
                    def2['ctermfg'] = val
                elif type(val) is str:
                    def2['guifg'] = val
                elif type(val) is int:
                    def2['ctermfg'] = val
            elif key == 'bg':
                if type(val) is tuple:
                    def2['guibg'] = val[0]
                    def2['ctermbg'] = val[1]
                elif type(val) is str and val == 'NONE':
                    def2['guibg'] = val
                    def2['ctermbg'] = val
                elif type(val) is str:
                    def2['guibg'] = val
                elif type(val) is int:
                    def2['ctermbg'] = val
            elif key == 'deco':
                if type(val) is tuple:
                    def2['gui'] = val[0]
                    def2['cterm'] = val[1]
                elif type(val) is str:
                    def2['gui'] = val
                    def2['cterm'] = val
        execute('hi ' + name + ' ' + ' '.join(k + '=' + str(v) for k, v in def2.items()))
<commit_msg>Tune colors for folding<commit_after>HEADER = """" Name:     hydrangea.vim --- Hydrangea theme for Vim
" Author:   Yuta Taniguchi <yuta.taniguchi.y.t@gmail.com>
" URL:      https://github.com/yuttie/hydrangea-vim
" Version:  3.0.0
" License:  The MIT License (MIT)
"     Copyright (c) 2015-2016 Yuta Taniguchi
"
"     Permission is hereby granted, free of charge, to any person obtaining a copy
"     of this software and associated documentation files (the "Software"), to deal
"     in the Software without restriction, including without limitation the rights
"     to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
"     copies of the Software, and to permit persons to whom the Software is
"     furnished to do so, subject to the following conditions:
"
"     The above copyright notice and this permission notice shall be included in
"     all copies or substantial portions of the Software.
"
"     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
"     IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
"     FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
"     AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
"     LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
"     OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
"     THE SOFTWARE.
"""

try:
    import vim

    def execute(cmd):
        vim.command(cmd)
except ImportError:
    print(HEADER)

    def execute(cmd):
        print(cmd)

# Palette
base03       = ("#1d1f25", 234)
base02       = ("#2c303a", 236)
base01       = ("#343945", 237)
base00       = ("#3e4451", 238)
base0        = ("#555d6f",  59)
base1        = ("#626c82", 242)
base2        = ("#c6d0de", 252)
base3        = ("#e8eff8", 255)
red          = ("#e91e63", 197)
red_dark     = ("#792340",  52)
teal         = ("#36c2c2",  44)
teal_dark    = ("#134242",  23)
blue         = ("#537dd5",  68)
blue_light   = ('#91b5ff', 111)
violet       = ("#996ddb",  98)
violet_light = ("#ceadff", 183)
magenta      = ("#e242ac", 162)
diff_add_fg  = ("#52c4ff",  81)
diff_add_bg  = ("#0d435f",  24)
diff_add_hi  = ("#97dcff", 117)
diff_rm_fg   = ("#e242ac", 162)
diff_rm_bg   = ("#66184c",  53)
diff_rm_hi   = ("#fface3", 218)

# Definitions
color = {}
color['Normal']       = {'fg': base2,            'bg': base02,                                   }
color['Cursor']       = {'fg': 'NONE',           'bg': base3,                 'deco': 'NONE'     }
color['CursorIM']     = {'fg': 'NONE',           'bg': base3,                                    }
color['CursorLine']   = {'fg': 'NONE',           'bg': base01,                'deco': 'bold'     }
color['CursorColumn'] = {'fg': 'NONE',           'bg': base01,                'deco': 'NONE'     }
color['Visual']       = {'fg': 'NONE',           'bg': base00,                'deco': 'NONE'     }
color['VisualNOS']    = {'fg': 'fg',                                          'deco': 'underline'}

color['Folded']       = {'fg': base2,            'bg': base01,                'deco': 'NONE'     }
color['FoldColumn']   = {'fg': base2,            'bg': base02,                'deco': 'NONE'     }
color['Title']        = {'fg': ('#f6f3e8', 255), 'bg': 'NONE',                'deco': 'bold'     }
color['StatusLine']   = {'fg': base2,            'bg': base00,                'deco': 'NONE'     }
color['StatusLineNC'] = {'fg': base0,            'bg': base01,                'deco': 'NONE'     }
color['VertSplit']    = {'fg': base01,           'bg': base01,                'deco': 'NONE'     }
color['LineNr']       = {'fg': base0,            'bg': base01,                'deco': 'NONE'     }
color['CursorLineNr'] = {'fg': base3,            'bg': base0,                 'deco': 'bold'     }
color['SpecialKey']   = {'fg': ('#808080',   8), 'bg': ('#343434',   8),      'deco': 'NONE'     }
color['NonText']      = {'fg': base0,            'bg': base02,                'deco': 'NONE'     }
color['MatchParen']   = {'fg': red,              'bg': 'NONE',                'deco': 'bold'     }

color['Comment']      = {'fg': base0,                                         'deco': 'NONE'     }
color['Constant']     = {'fg': teal,             "bg": teal_dark,             'deco': 'NONE'     }
color['String']       = 'Constant'
color['Number']       = 'Constant'
color['Identifier']   = {'fg': base3,                                         'deco': 'bold'     }
color['Function']     = {'fg': base3,                                         'deco': 'bold'     }
color['Statement']    = {'fg': blue,                                          'deco': 'bold'     }
color['Operator']     = {'fg': magenta,                                       'deco': 'NONE'     }
color['Include']      = {'fg': violet,                                        'deco': 'NONE'     }
color['PreProc']      = {'fg': violet_light,                                  'deco': 'NONE'     }
color['Type']         = {'fg': magenta,                                       'deco': 'NONE'     }
color['StorageClass'] = {'fg': blue,                                          'deco': 'bold'     }
color['Structure']    = {'fg': magenta,                                       'deco': 'NONE'     }
color['Typedef']      = {'fg': blue,                                          'deco': 'bold'     }
color['Special']      = {'fg': 'NONE',           'bg': 'NONE',                'deco': 'bold'     }
color['Underlined']   = {'fg': 'fg',                                          'deco': 'underline'}
color['Ignore']       = {'fg': 'bg'                                                              }
color['Error']        = {'fg': red,              'bg': red_dark,              'deco': 'bold'     }
color['Todo']         = {'fg': base2,            'bg': base02,                'deco': 'bold'     }

color['IncSearch']    = {'fg': base3,            'bg': violet_light,          'deco': 'NONE'     }
color['Search']       = {'fg': base3,            'bg': magenta,               'deco': 'NONE'     }
color['Pmenu']        = {'fg': base2,            'bg': base01,                'deco': 'NONE'     }
color['PmenuSel']     = {'fg': base3,            'bg': base00,                'deco': 'bold'     }
color['PmenuSbar']    = {                        'bg': base01,                'deco': 'NONE'     }
color['PmenuThumb']   = {                        'bg': base0,                 'deco': 'NONE'     }
color['TabLine']      = {'fg': base0,            'bg': base01,                'deco': 'NONE'     }
color['TabLineSel']   = {'fg': base02,           'bg': blue,                  'deco': 'bold'     }
color['TabLineFill']  = {'fg': base01,           'bg': base02,                'deco': 'underline'}

color['SpellBad']     = {                                                     'deco': 'undercurl'}
color['SpellCap']     = {                                                     'deco': 'undercurl'}
color['SpellRare']    = {                                                     'deco': 'undercurl'}
color['SpellLocal']   = {                                                     'deco': 'undercurl'}

color['DiffAdd']      = {'fg': diff_add_fg,      'bg': diff_add_bg,           'deco': 'NONE'     }
color['DiffChange']   = {'fg': diff_rm_fg,       'bg': diff_rm_bg,            'deco': 'NONE'     }
color['DiffDelete']   = {'fg': diff_rm_fg,       'bg': diff_rm_bg,            'deco': 'NONE'     }
color['DiffText']     = {'fg': diff_rm_hi,       'bg': diff_rm_bg,            'deco': 'bold'     }

color['diffAdded']    = {'fg': diff_add_fg,      'bg': diff_add_bg,           'deco': 'NONE'     }
color['diffRemoved']  = {'fg': diff_rm_fg,       'bg': diff_rm_bg,            'deco': 'NONE'     }

color['Directory']    = {'fg': ('#c0e0b0', 151),                              'deco': 'NONE'     }
color['ErrorMsg']     = {'fg': red,              'bg': 'NONE',                'deco': 'NONE'     }
color['SignColumn']   = {'fg': ('#9fafaf', 145), 'bg': color['LineNr']['bg'], 'deco': 'NONE'     }
color['MoreMsg']      = {'fg': ('#2e8b57',  29),                              'deco': 'NONE'     }
color['ModeMsg']      = {'fg': ('#76d5f8',  81), 'bg': 'NONE',                'deco': 'NONE'     }
color['Question']     = {'fg': 'fg',                                          'deco': 'NONE'     }
color['WarningMsg']   = {'fg': red,                                           'deco': 'NONE'     }
color['WildMenu']     = {'fg': base3,            'bg': base0,                 'deco': 'bold'     }
color['ColorColumn']  = {'fg': 'NONE',           'bg': ('#403630', 237),      'deco': 'NONE'     }

# GitGutter
color['GitGutterAdd']    = {'fg': diff_add_fg, 'bg': color['SignColumn']['bg'], 'deco': 'bold'}
color['GitGutterChange'] = {'fg': diff_rm_fg,  'bg': color['SignColumn']['bg'], 'deco': 'bold'}
color['GitGutterDelete'] = {'fg': diff_rm_fg,  'bg': color['SignColumn']['bg'], 'deco': 'bold'}

# make
color['makeIdent']      = 'Type'
color['makeSpecTarget'] = 'Special'
color['makeTarget']     = 'Function'
color['makeCommands']   = 'NONE'

# php
color['phpVarSelector'] = 'Identifier'
color['phpIdentifier']  = 'NONE'
color['phpFunctions']   = 'NONE'
color['phpClasses']     = 'NONE'
color['phpFunction']    = 'Function'
color['phpClass']       = 'Type'

# rust
color['rustFuncCall'] = {'fg': blue_light}
color['rustQuestionMark'] = 'Operator'

# vim
color['vimVar'] = 'NONE'

# Apply
execute("""hi clear
if exists('syntax_on')
  syntax reset
endif
let g:colors_name = 'hydrangea'

set background=dark
""")

for name, cdef in color.items():
    if type(cdef) is str:
        # The definition is a string
        if cdef == 'NONE':
            # Disable
            execute('hi ' + name + ' NONE')
            execute('hi link ' + name + ' NONE')
        else:
            # Link
            execute('hi link ' + name + ' ' + cdef)
    elif type(cdef) is dict and len(cdef) > 0:
        # The definition is a dictionary
        def2 = {}
        for key, val in cdef.items():
            if key == 'fg':
                if type(val) is tuple:
                    def2['guifg'] = val[0]
                    def2['ctermfg'] = val[1]
                elif type(val) is str and val == 'NONE':
                    def2['guifg'] = val
                    def2['ctermfg'] = val
                elif type(val) is str:
                    def2['guifg'] = val
                elif type(val) is int:
                    def2['ctermfg'] = val
            elif key == 'bg':
                if type(val) is tuple:
                    def2['guibg'] = val[0]
                    def2['ctermbg'] = val[1]
                elif type(val) is str and val == 'NONE':
                    def2['guibg'] = val
                    def2['ctermbg'] = val
                elif type(val) is str:
                    def2['guibg'] = val
                elif type(val) is int:
                    def2['ctermbg'] = val
            elif key == 'deco':
                if type(val) is tuple:
                    def2['gui'] = val[0]
                    def2['cterm'] = val[1]
                elif type(val) is str:
                    def2['gui'] = val
                    def2['cterm'] = val
        execute('hi ' + name + ' ' + ' '.join(k + '=' + str(v) for k, v in def2.items()))
<|endoftext|>


<commit_before>#!/usr/bin/env python3
#
# Copyright (C) 2015 ScyllaDB
#

#
# This file is part of Scylla.
#
# Scylla is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Scylla is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
#
import os
import sys
import argparse
import subprocess
import signal
import shlex
import threading
import concurrent.futures
import io

boost_tests = [
    'bytes_ostream_test',
    'chunked_vector_test',
    'compress_test',
    'continuous_data_consumer_test',
    'types_test',
    'keys_test',
    'mutation_test',
    'mvcc_test',
    'schema_registry_test',
    'range_test',
    'mutation_reader_test',
    'serialized_action_test',
    'cql_query_test',
    'storage_proxy_test',
    'schema_change_test',
    'sstable_mutation_test',
    'sstable_resharding_test',
    'commitlog_test',
    'hash_test',
    'test-serialization',
    'cartesian_product_test',
    'allocation_strategy_test',
    'UUID_test',
    'compound_test',
    'murmur_hash_test',
    'partitioner_test',
    'frozen_mutation_test',
    'canonical_mutation_test',
    'gossiping_property_file_snitch_test',
    'row_cache_test',
    'cache_flat_mutation_reader_test',
    'network_topology_strategy_test',
    'query_processor_test',
    'batchlog_manager_test',
    'logalloc_test',
    'log_heap_test',
    'crc_test',
    'flush_queue_test',
    'config_test',
    'dynamic_bitset_test',
    'gossip_test',
    'managed_vector_test',
    'map_difference_test',
    'memtable_test',
    'mutation_query_test',
    'snitch_reset_test',
    'auth_test',
    'idl_test',
    'range_tombstone_list_test',
    'mutation_fragment_test',
    'flat_mutation_reader_test',
    'anchorless_list_test',
    'database_test',
    'input_stream_test',
    'nonwrapping_range_test',
    'virtual_reader_test',
    'counter_test',
    'cell_locker_test',
    'view_schema_test',
    'view_build_test',
    'view_complex_test',
    'clustering_ranges_walker_test',
    'vint_serialization_test',
    'duration_test',
    'loading_cache_test',
    'castas_fcts_test',
    'big_decimal_test',
    'aggregate_fcts_test',
    'role_manager_test',
    'caching_options_test',
    'auth_resource_test',
    'cql_auth_query_test',
    'enum_set_test',
    'extensions_test',
    'cql_auth_syntax_test',
    'querier_cache',
    'limiting_data_source_test',
]

other_tests = [
    'memory_footprint',
]

def print_progress_succint(test_path, test_args, success, cookie):
    if type(cookie) is int:
        cookie = (0, 1, cookie)

    last_len, n, n_total = cookie
    if success:
        status = "PASSED"
    else:
        status = "FAILED"

    msg = "[{}/{}] {} {} {}".format(n, n_total, status, test_path, ' '.join(test_args))
    if sys.stdout.isatty():
        print('\r' + ' ' * last_len, end='')
        last_len = len(msg)
        print('\r' + msg, end='')
    else:
        print(msg)

    return (last_len, n + 1, n_total)

def print_status_verbose(test_path, test_args, success, cookie):
    if type(cookie) is int:
        cookie = (1, cookie)

    n, n_total = cookie
    if success:
        status = "PASSED"
    else:
        status = "FAILED"

    msg = "[{}/{}] {} {} {}".format(n, n_total, status, test_path, ' '.join(test_args))
    print(msg)

    return (n + 1, n_total)

class Alarm(Exception):
    pass

def alarm_handler(signum, frame):
    raise Alarm

if __name__ == "__main__":
    all_modes = ['debug', 'release']

    parser = argparse.ArgumentParser(description="Scylla test runner")
    parser.add_argument('--fast',  action="store_true",
                        help="Run only fast tests")
    parser.add_argument('--name',  action="store",
                        help="Run only test whose name contains given string")
    parser.add_argument('--mode', choices=all_modes,
                        help="Run only tests for given build mode")
    parser.add_argument('--timeout', action="store", default="300", type=int,
                        help="timeout value for test execution")
    parser.add_argument('--jenkins', action="store",
                        help="jenkins output file prefix")
    parser.add_argument('--verbose', '-v', action='store_true', default=False,
                        help='Verbose reporting')
    args = parser.parse_args()

    print_progress = print_status_verbose if args.verbose else print_progress_succint

    test_to_run = []
    modes_to_run = all_modes if not args.mode else [args.mode]
    for mode in modes_to_run:
        prefix = os.path.join('build', mode, 'tests')
        standard_args =  '--overprovisioned --unsafe-bypass-fsync 1'.split()
        seastar_args = '-c2 -m2G'.split() + standard_args
        for test in other_tests:
            test_to_run.append((os.path.join(prefix, test), 'other', seastar_args))
        for test in boost_tests:
            test_to_run.append((os.path.join(prefix, test), 'boost', seastar_args))

    if 'release' in modes_to_run:
        test_to_run.append(('build/release/tests/lsa_async_eviction_test', 'other',
                            '-c1 -m200M --size 1024 --batch 3000 --count 2000000'.split() + standard_args))
        test_to_run.append(('build/release/tests/lsa_sync_eviction_test', 'other',
                            '-c1 -m100M --count 10 --standard-object-size 3000000'.split() + standard_args))
        test_to_run.append(('build/release/tests/lsa_sync_eviction_test', 'other',
                            '-c1 -m100M --count 24000 --standard-object-size 2048'.split() + standard_args))
        test_to_run.append(('build/release/tests/lsa_sync_eviction_test', 'other',
                            '-c1 -m1G --count 4000000 --standard-object-size 128'.split() + standard_args))
        test_to_run.append(('build/release/tests/row_cache_alloc_stress', 'other',
                            '-c1 -m2G'.split() + standard_args))
        test_to_run.append(('build/release/tests/sstable_test', 'boost', ['-c1'] + standard_args))
        test_to_run.append(('build/release/tests/row_cache_stress_test', 'other', '-c1 -m1G --seconds 10'.split() + standard_args))
    if 'debug' in modes_to_run:
        test_to_run.append(('build/debug/tests/sstable_test', 'boost', ['-c1'] + standard_args))

    if args.name:
        test_to_run = [t for t in test_to_run if args.name in t[0]]

    failed_tests = []

    n_total = len(test_to_run)
    env = os.environ
    # disable false positive due to new (with_alignment(...)) ...
    env['ASAN_OPTIONS'] = 'alloc_dealloc_mismatch=0'
    env['UBSAN_OPTIONS'] = 'print_stacktrace=1'
    env['BOOST_TEST_CATCH_SYSTEM_ERRORS'] = 'no'
    def run_test(path, type, exec_args):
        boost_args = []
        # avoid modifying in-place, it will change test_to_run
        exec_args = exec_args + '--collectd 0'.split()
        file = io.StringIO()
        if args.jenkins and type == 'boost':
            mode = 'release'
            if path.startswith(os.path.join('build', 'debug')):
                mode = 'debug'
            xmlout = (args.jenkins + "." + mode + "." +
                      os.path.basename(path.split()[0]) + ".boost.xml")
            boost_args += ['--report_level=no', '--logger=XML,test_suite,' + xmlout]
        if type == 'boost':
            boost_args += ['--']
        def report_error(out, report_subcause):
            report_subcause()
            if out:
                print('=== stdout START ===', file=file)
                print(out, file=file)
                print('=== stdout END ===', file=file)
        out = None
        success = False
        try:
            out = subprocess.check_output([path] + boost_args + exec_args,
                                stderr=subprocess.STDOUT,
                                timeout=args.timeout,
                                env=env, preexec_fn=os.setsid)
            success = True
        except subprocess.TimeoutExpired as e:
            def report_subcause():
                print('  timed out', file=file)
            report_error(e.output.decode(encoding='UTF-8'), report_subcause=report_subcause)
        except subprocess.CalledProcessError as e:
            def report_subcause():
                print('  with error code {code}\n'.format(code=e.returncode), file=file)
            report_error(e.output.decode(encoding='UTF-8'), report_subcause=report_subcause)
        except Exception as e:
            def report_subcause():
                print('  with error {e}\n'.format(e=e), file=file)
            report_error(e, report_subcause=report_subcause)
        return (path, boost_args + exec_args, success, file.getvalue())
    sysmem = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')
    testmem = 2e9
    executor = concurrent.futures.ThreadPoolExecutor(max_workers=((sysmem - 4e9) // testmem))
    futures = []
    for n, test in enumerate(test_to_run):
        path = test[0]
        test_type = test[1]
        exec_args = test[2] if len(test) >= 3 else []
        futures.append(executor.submit(run_test, path, test_type, exec_args))

    cookie = n_total
    for future in futures:
        test_path, test_args, success, out = future.result()
        cookie = print_progress(test_path, test_args, success, cookie)
        if not success:
            failed_tests.append((test_path, out))

    if not failed_tests:
        print('\nOK.')
    else:
        print('\n\nOutput of the failed tests:')
        for test, out in failed_tests:
            print("Test {} failed:\n{}".format(test, out))
        print('\n\nThe following test(s) have failed:')
        for test, out in failed_tests:
            print('  {}'.format(test))
        print('\nSummary: {} of the total {} tests failed'.format(len(failed_tests), len(test_to_run)))
        sys.exit(1)
<commit_msg>secondary index: update test.py<commit_after>#!/usr/bin/env python3
#
# Copyright (C) 2015 ScyllaDB
#

#
# This file is part of Scylla.
#
# Scylla is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Scylla is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
#
import os
import sys
import argparse
import subprocess
import signal
import shlex
import threading
import concurrent.futures
import io

boost_tests = [
    'bytes_ostream_test',
    'chunked_vector_test',
    'compress_test',
    'continuous_data_consumer_test',
    'types_test',
    'keys_test',
    'mutation_test',
    'mvcc_test',
    'schema_registry_test',
    'range_test',
    'mutation_reader_test',
    'serialized_action_test',
    'cql_query_test',
    'secondary_index_test',
    'storage_proxy_test',
    'schema_change_test',
    'sstable_mutation_test',
    'sstable_resharding_test',
    'commitlog_test',
    'hash_test',
    'test-serialization',
    'cartesian_product_test',
    'allocation_strategy_test',
    'UUID_test',
    'compound_test',
    'murmur_hash_test',
    'partitioner_test',
    'frozen_mutation_test',
    'canonical_mutation_test',
    'gossiping_property_file_snitch_test',
    'row_cache_test',
    'cache_flat_mutation_reader_test',
    'network_topology_strategy_test',
    'query_processor_test',
    'batchlog_manager_test',
    'logalloc_test',
    'log_heap_test',
    'crc_test',
    'flush_queue_test',
    'config_test',
    'dynamic_bitset_test',
    'gossip_test',
    'managed_vector_test',
    'map_difference_test',
    'memtable_test',
    'mutation_query_test',
    'snitch_reset_test',
    'auth_test',
    'idl_test',
    'range_tombstone_list_test',
    'mutation_fragment_test',
    'flat_mutation_reader_test',
    'anchorless_list_test',
    'database_test',
    'input_stream_test',
    'nonwrapping_range_test',
    'virtual_reader_test',
    'counter_test',
    'cell_locker_test',
    'view_schema_test',
    'view_build_test',
    'view_complex_test',
    'clustering_ranges_walker_test',
    'vint_serialization_test',
    'duration_test',
    'loading_cache_test',
    'castas_fcts_test',
    'big_decimal_test',
    'aggregate_fcts_test',
    'role_manager_test',
    'caching_options_test',
    'auth_resource_test',
    'cql_auth_query_test',
    'enum_set_test',
    'extensions_test',
    'cql_auth_syntax_test',
    'querier_cache',
    'limiting_data_source_test',
]

other_tests = [
    'memory_footprint',
]

def print_progress_succint(test_path, test_args, success, cookie):
    if type(cookie) is int:
        cookie = (0, 1, cookie)

    last_len, n, n_total = cookie
    if success:
        status = "PASSED"
    else:
        status = "FAILED"

    msg = "[{}/{}] {} {} {}".format(n, n_total, status, test_path, ' '.join(test_args))
    if sys.stdout.isatty():
        print('\r' + ' ' * last_len, end='')
        last_len = len(msg)
        print('\r' + msg, end='')
    else:
        print(msg)

    return (last_len, n + 1, n_total)

def print_status_verbose(test_path, test_args, success, cookie):
    if type(cookie) is int:
        cookie = (1, cookie)

    n, n_total = cookie
    if success:
        status = "PASSED"
    else:
        status = "FAILED"

    msg = "[{}/{}] {} {} {}".format(n, n_total, status, test_path, ' '.join(test_args))
    print(msg)

    return (n + 1, n_total)

class Alarm(Exception):
    pass

def alarm_handler(signum, frame):
    raise Alarm

if __name__ == "__main__":
    all_modes = ['debug', 'release']

    parser = argparse.ArgumentParser(description="Scylla test runner")
    parser.add_argument('--fast',  action="store_true",
                        help="Run only fast tests")
    parser.add_argument('--name',  action="store",
                        help="Run only test whose name contains given string")
    parser.add_argument('--mode', choices=all_modes,
                        help="Run only tests for given build mode")
    parser.add_argument('--timeout', action="store", default="300", type=int,
                        help="timeout value for test execution")
    parser.add_argument('--jenkins', action="store",
                        help="jenkins output file prefix")
    parser.add_argument('--verbose', '-v', action='store_true', default=False,
                        help='Verbose reporting')
    args = parser.parse_args()

    print_progress = print_status_verbose if args.verbose else print_progress_succint

    test_to_run = []
    modes_to_run = all_modes if not args.mode else [args.mode]
    for mode in modes_to_run:
        prefix = os.path.join('build', mode, 'tests')
        standard_args =  '--overprovisioned --unsafe-bypass-fsync 1'.split()
        seastar_args = '-c2 -m2G'.split() + standard_args
        for test in other_tests:
            test_to_run.append((os.path.join(prefix, test), 'other', seastar_args))
        for test in boost_tests:
            test_to_run.append((os.path.join(prefix, test), 'boost', seastar_args))

    if 'release' in modes_to_run:
        test_to_run.append(('build/release/tests/lsa_async_eviction_test', 'other',
                            '-c1 -m200M --size 1024 --batch 3000 --count 2000000'.split() + standard_args))
        test_to_run.append(('build/release/tests/lsa_sync_eviction_test', 'other',
                            '-c1 -m100M --count 10 --standard-object-size 3000000'.split() + standard_args))
        test_to_run.append(('build/release/tests/lsa_sync_eviction_test', 'other',
                            '-c1 -m100M --count 24000 --standard-object-size 2048'.split() + standard_args))
        test_to_run.append(('build/release/tests/lsa_sync_eviction_test', 'other',
                            '-c1 -m1G --count 4000000 --standard-object-size 128'.split() + standard_args))
        test_to_run.append(('build/release/tests/row_cache_alloc_stress', 'other',
                            '-c1 -m2G'.split() + standard_args))
        test_to_run.append(('build/release/tests/sstable_test', 'boost', ['-c1'] + standard_args))
        test_to_run.append(('build/release/tests/row_cache_stress_test', 'other', '-c1 -m1G --seconds 10'.split() + standard_args))
    if 'debug' in modes_to_run:
        test_to_run.append(('build/debug/tests/sstable_test', 'boost', ['-c1'] + standard_args))

    if args.name:
        test_to_run = [t for t in test_to_run if args.name in t[0]]

    failed_tests = []

    n_total = len(test_to_run)
    env = os.environ
    # disable false positive due to new (with_alignment(...)) ...
    env['ASAN_OPTIONS'] = 'alloc_dealloc_mismatch=0'
    env['UBSAN_OPTIONS'] = 'print_stacktrace=1'
    env['BOOST_TEST_CATCH_SYSTEM_ERRORS'] = 'no'
    def run_test(path, type, exec_args):
        boost_args = []
        # avoid modifying in-place, it will change test_to_run
        exec_args = exec_args + '--collectd 0'.split()
        file = io.StringIO()
        if args.jenkins and type == 'boost':
            mode = 'release'
            if path.startswith(os.path.join('build', 'debug')):
                mode = 'debug'
            xmlout = (args.jenkins + "." + mode + "." +
                      os.path.basename(path.split()[0]) + ".boost.xml")
            boost_args += ['--report_level=no', '--logger=XML,test_suite,' + xmlout]
        if type == 'boost':
            boost_args += ['--']
        def report_error(out, report_subcause):
            report_subcause()
            if out:
                print('=== stdout START ===', file=file)
                print(out, file=file)
                print('=== stdout END ===', file=file)
        out = None
        success = False
        try:
            out = subprocess.check_output([path] + boost_args + exec_args,
                                stderr=subprocess.STDOUT,
                                timeout=args.timeout,
                                env=env, preexec_fn=os.setsid)
            success = True
        except subprocess.TimeoutExpired as e:
            def report_subcause():
                print('  timed out', file=file)
            report_error(e.output.decode(encoding='UTF-8'), report_subcause=report_subcause)
        except subprocess.CalledProcessError as e:
            def report_subcause():
                print('  with error code {code}\n'.format(code=e.returncode), file=file)
            report_error(e.output.decode(encoding='UTF-8'), report_subcause=report_subcause)
        except Exception as e:
            def report_subcause():
                print('  with error {e}\n'.format(e=e), file=file)
            report_error(e, report_subcause=report_subcause)
        return (path, boost_args + exec_args, success, file.getvalue())
    sysmem = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')
    testmem = 2e9
    executor = concurrent.futures.ThreadPoolExecutor(max_workers=((sysmem - 4e9) // testmem))
    futures = []
    for n, test in enumerate(test_to_run):
        path = test[0]
        test_type = test[1]
        exec_args = test[2] if len(test) >= 3 else []
        futures.append(executor.submit(run_test, path, test_type, exec_args))

    cookie = n_total
    for future in futures:
        test_path, test_args, success, out = future.result()
        cookie = print_progress(test_path, test_args, success, cookie)
        if not success:
            failed_tests.append((test_path, out))

    if not failed_tests:
        print('\nOK.')
    else:
        print('\n\nOutput of the failed tests:')
        for test, out in failed_tests:
            print("Test {} failed:\n{}".format(test, out))
        print('\n\nThe following test(s) have failed:')
        for test, out in failed_tests:
            print('  {}'.format(test))
        print('\nSummary: {} of the total {} tests failed'.format(len(failed_tests), len(test_to_run)))
        sys.exit(1)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import neukrill_net.utils as utils
import neukrill_net.image_processing as image_processing
import neukrill_net.augment as augment

import csv
import pickle
from sklearn.externals import joblib
import numpy as np
import glob
import os
import argparse
import six.moves

def main(run_settings_path, verbose=False):
    # this should just run either function depending on the run settings
    settings = utils.Settings('settings.json')
    # test script won't overwrite the pickle, so always force load
    run_settings = utils.load_run_settings(run_settings_path,
            settings,
            settings_path='settings.json',
            force=True)
    # HELLO BOILERPLATE
    if run_settings['model type'] == 'sklearn':
        test_sklearn(run_settings, verbose=verbose)
    elif run_settings['model type'] == 'pylearn2':
        #train_pylearn2(run_settings)
        test_pylearn2(run_settings, verbose=verbose)
    else:
        raise NotImplementedError("Unsupported model type.")

def test_sklearn(run_settings, verbose=False):
    # some more boilerplate here
    # unpack settings
    settings = run_settings['settings']

    # get all training file paths and class names
    image_fname_dict = settings.image_fnames

    # parsed from json, preproc settings are dict
    augment_settings = run_settings["preprocessing"]
    processing = augment.augmentation_wrapper(**augment_settings)

    image_fname_dict = settings.image_fnames

    X, names = utils.load_data(image_fname_dict, processing=processing,
                               verbose=verbose)

    # load the model from where it's _expected_ to be saved
    clf = joblib.load(run_settings['pickle abspath'])
    p = clf.predict_proba(X)

    utils.write_predictions(run_settings['submissions abspath'], p,
            names, settings.classes)

def test_pylearn2(run_settings, batch_size=4075, verbose=False):
    # Based on the script found at:
    #   https://github.com/zygmuntz/pylearn2-practice/blob/master/predict.py

    import pylearn2.utils
    import pylearn2.config
    import neukrill_net.dense_dataset
    import theano

    # unpack settings
    settings = run_settings['settings']

    # first load the model
    if verbose:
        print("Loading model...")
    model = pylearn2.utils.serial.load(run_settings['pickle abspath'])

    # then load the dataset
    if verbose:
        print("Loading data...")
    dataset = neukrill_net.dense_dataset.DensePNGDataset(
            settings_path=run_settings['settings_path'],
            run_settings=run_settings['run_settings_path'],
            train_or_predict='test')

    # then set batches:
    model.set_batch_size(batch_size)

    # then check batch size
    N_images = dataset.X.shape[0]
    # see how much extra we're going to have
    extra = batch_size - N_images%batch_size
    # check that worked
    assert (N_images+extra)%batch_size == 0

    # if we have extra, then we're going to have to pad with zeros
    # this might be a problem, with the massive array we're going
    # to end up with
    if extra > 0:
        dataset.X = np.concatenate((dataset.X,
            np.zeros((extra, dataset.X.shape[1]), dtype=dataset.X.dtype)),
            axis=0)

        #then check that worked:
        assert dataset.X.shape[0]%batch_size == 0

    # make a function to perform the forward propagation in our network
    if verbose:
        print("Compiling Theano function...")
    X = model.get_input_space().make_batch_theano()
    Y = model.fprop(X)
    f = theano.function([X],Y)

    # initialise our results array
    y = np.zeros((N_images, len(settings.classes)))
    # didn't want to use xrange explicitly
    n_batches = int(dataset.X.shape[0]/batch_size)
    for i in range(n_batches):
        if verbose:
            print("Processing batch {0} of {1}".format(i,n_batches))
        # grab a row
        x_arg = dataset.X[i*batch_size:(i+1)*batch_size,:]
        # check if we're dealing with images:
        if X.ndim > 2:
            # if so redefine to topological view
            x_arg = dataset.get_topological_view(x_arg)
        # and append the resulting value to y
        # don't understand why I have to transpose here, but I do
        y[i*batch_size:(i+1)*batch_size,:] = (f(x_arg.astype(X.dtype).T))

    # stupidest solution to augmentation problem
    # just collapse adjacent predictions until we
    # have the right number
    if len(dataset.names) < y.shape[0]:
        y_collapsed = np.zeros((len(dataset.names),y.shape[1]))
        augmentation_factor = int(y.shape[0]/len(dataset.names))
        # collapse every <augmentation_factor> predictions by averaging
        for i,(low,high) in enumerate(zip(range(0,
                        y.shape[0]-augmentation_factor, augmentation_factor),
                        range(augmentation_factor,y.shape[0],
                                                    augmentation_factor))):
            # confused yet?
            pdb.set_trace()
            # slice from low to high and take average down columns
            y_collapsed[i,:] = np.mean(y[low:high,:], axis=0)
        y = y_collapsed

    # then write our results to csv
    if verbose:
        print("Writing csv")
    utils.write_predictions(run_settings['submissions abspath'], y,
            dataset.names, settings.classes)

if __name__ == '__main__':
    # copied code from train.py here instead of making a function
    # because this code may diverge
    # need to argparse for run settings path
    parser = argparse.ArgumentParser(description='Train a model and store a '
                                                 'pickled model file.')
    # nargs='?' will look for a single argument but failover to default
    parser.add_argument('run_settings', metavar='run_settings', type=str,
            nargs='?', default=os.path.join("run_settings","default.json"),
            help="Path to run settings json file.")
    # add verbose option
    parser.add_argument('-v', action="store_true", help="Run verbose.")
    args = parser.parse_args()
    main(args.run_settings, verbose=args.v)
<commit_msg>are we tripping the extra loop?<commit_after>#!/usr/bin/env python

import neukrill_net.utils as utils
import neukrill_net.image_processing as image_processing
import neukrill_net.augment as augment

import csv
import pickle
from sklearn.externals import joblib
import numpy as np
import glob
import os
import argparse
import six.moves

def main(run_settings_path, verbose=False):
    # this should just run either function depending on the run settings
    settings = utils.Settings('settings.json')
    # test script won't overwrite the pickle, so always force load
    run_settings = utils.load_run_settings(run_settings_path,
            settings,
            settings_path='settings.json',
            force=True)
    # HELLO BOILERPLATE
    if run_settings['model type'] == 'sklearn':
        test_sklearn(run_settings, verbose=verbose)
    elif run_settings['model type'] == 'pylearn2':
        #train_pylearn2(run_settings)
        test_pylearn2(run_settings, verbose=verbose)
    else:
        raise NotImplementedError("Unsupported model type.")

def test_sklearn(run_settings, verbose=False):
    # some more boilerplate here
    # unpack settings
    settings = run_settings['settings']

    # get all training file paths and class names
    image_fname_dict = settings.image_fnames

    # parsed from json, preproc settings are dict
    augment_settings = run_settings["preprocessing"]
    processing = augment.augmentation_wrapper(**augment_settings)

    image_fname_dict = settings.image_fnames

    X, names = utils.load_data(image_fname_dict, processing=processing,
                               verbose=verbose)

    # load the model from where it's _expected_ to be saved
    clf = joblib.load(run_settings['pickle abspath'])
    p = clf.predict_proba(X)

    utils.write_predictions(run_settings['submissions abspath'], p,
            names, settings.classes)

def test_pylearn2(run_settings, batch_size=4075, verbose=False):
    # Based on the script found at:
    #   https://github.com/zygmuntz/pylearn2-practice/blob/master/predict.py

    import pylearn2.utils
    import pylearn2.config
    import neukrill_net.dense_dataset
    import theano

    # unpack settings
    settings = run_settings['settings']

    # first load the model
    if verbose:
        print("Loading model...")
    model = pylearn2.utils.serial.load(run_settings['pickle abspath'])

    # then load the dataset
    if verbose:
        print("Loading data...")
    dataset = neukrill_net.dense_dataset.DensePNGDataset(
            settings_path=run_settings['settings_path'],
            run_settings=run_settings['run_settings_path'],
            train_or_predict='test')

    # then set batches:
    model.set_batch_size(batch_size)

    # then check batch size
    N_images = dataset.X.shape[0]
    # see how much extra we're going to have
    extra = batch_size - N_images%batch_size
    # check that worked
    assert (N_images+extra)%batch_size == 0

    # if we have extra, then we're going to have to pad with zeros
    # this might be a problem, with the massive array we're going
    # to end up with
    if extra > 0:
        if verbose:
            print("Extra detected, padding dataset with"
                    " zeros for batch processing")
        dataset.X = np.concatenate((dataset.X,
            np.zeros((extra, dataset.X.shape[1]), dtype=dataset.X.dtype)),
            axis=0)

        #then check that worked:
        assert dataset.X.shape[0]%batch_size == 0

    # make a function to perform the forward propagation in our network
    if verbose:
        print("Compiling Theano function...")
    X = model.get_input_space().make_batch_theano()
    Y = model.fprop(X)
    f = theano.function([X],Y)

    # initialise our results array
    y = np.zeros((N_images, len(settings.classes)))
    # didn't want to use xrange explicitly
    n_batches = int(dataset.X.shape[0]/batch_size)
    for i in range(n_batches):
        if verbose:
            print("Processing batch {0} of {1}".format(i+1,n_batches))
        # grab a row
        x_arg = dataset.X[i*batch_size:(i+1)*batch_size,:]
        # check if we're dealing with images:
        if X.ndim > 2:
            # if so redefine to topological view
            x_arg = dataset.get_topological_view(x_arg)
        # and append the resulting value to y
        # don't understand why I have to transpose here, but I do
        y[i*batch_size:(i+1)*batch_size,:] = (f(x_arg.astype(X.dtype).T))

    # stupidest solution to augmentation problem
    # just collapse adjacent predictions until we
    # have the right number
    if len(dataset.names) < y.shape[0]:
        y_collapsed = np.zeros((len(dataset.names),y.shape[1]))
        augmentation_factor = int(y.shape[0]/len(dataset.names))
        # collapse every <augmentation_factor> predictions by averaging
        for i,(low,high) in enumerate(zip(range(0,
                        y.shape[0]-augmentation_factor, augmentation_factor),
                        range(augmentation_factor,y.shape[0],
                                                    augmentation_factor))):
            # confused yet?
            pdb.set_trace()
            # slice from low to high and take average down columns
            y_collapsed[i,:] = np.mean(y[low:high,:], axis=0)
        y = y_collapsed

    # then write our results to csv
    if verbose:
        print("Writing csv")
    utils.write_predictions(run_settings['submissions abspath'], y,
            dataset.names, settings.classes)

if __name__ == '__main__':
    # copied code from train.py here instead of making a function
    # because this code may diverge
    # need to argparse for run settings path
    parser = argparse.ArgumentParser(description='Train a model and store a '
                                                 'pickled model file.')
    # nargs='?' will look for a single argument but failover to default
    parser.add_argument('run_settings', metavar='run_settings', type=str,
            nargs='?', default=os.path.join("run_settings","default.json"),
            help="Path to run settings json file.")
    # add verbose option
    parser.add_argument('-v', action="store_true", help="Run verbose.")
    args = parser.parse_args()
    main(args.run_settings, verbose=args.v)
<|endoftext|>


<commit_before>__author__ = 'bernardyuan'

from scapy import all
from scapy.layers import  l2
import DLR
# read the pcap file
a = all.rdpcap('DLR/dlr.pcap')
import time
#in order: Beacon, nbr request, link status, locate fault,Announce, Sign on
# fake a packet and send through wireshark

#fabricate packet method 1:
#construct from keywords
# b = l2.Ether(**a[i][l2.Ether].fields)/l2.Dot1Q(**a[i][l2.Dot1Q].fields)/protocol.DLR(**a[i][protocol.DLR].fields)
# remember when constructed from keywords, do change the explicit value of ALL layers into 1
# otherwise use method 2, given below:
# b.explicit = 1
# b.payload.explicit = 1

# method 2:
# construct from raw string
# b0 = l2.Ether(str(a[0]))

# In the following code, we construct a new packet with method2, which could make your machine imitate
# a ring supervisor, and take control of the ring
# newpacket = l2.Ether(**a[30467][l2.Ether].fields)/l2.Dot1Q(**a[30467][l2.Dot1Q].fields)/DLR.DLR(**a[30467][DLR.DLR].fields)
newpacket = l2.Ether(**a[0][l2.Ether].fields)/l2.Dot1Q(**a[0][l2.Dot1Q].fields)/DLR.DLR(**a[0][DLR.DLR].fields)
# newpacket.payload.payload.fields["Node_Num"] = 1
# print newpacket.payload.payload.fields["SignOnInfo"]
# newpacket.payload.payload.fields["SignOnInfo"]["MACAddress"] = ["00:1d:9c:c8:03:e8"]
# newpacket.payload.payload.fields["SignOnInfo"]["IPAddress"] = ["192.168.0.19"]
newpacket.explicit = 1
newpacket.payload.explicit = 1
newpacket.fields["src"] = "00:1d:9c:c8:03:e8"
# newpacket.fields["dst"] = ""
# newpacket.payload.payload.fields["Source_Port"] = 1
newpacket.payload.payload.fields["Supervisor_Precedence"] = 255
newpacket.payload.payload.fields["Reserved"]='\x00\x01\x02\x03\x04\x05\x06\x07\x00\x01\x02\x03\x04\x05\x06\x07\x00\x01\x02\x03'
newpacket.payload.payload.fields["Sequence_ID"] = 0
newpacket.payload.payload.fields["Source_IP"] = '192.168.0.19'

# interval = newpacket.payload.payload.fields["Beacon_Interval"]
start = time.clock()
# print str(newpacket)
# print all.hexdump(newpacket)
all.sendp(str(newpacket),iface='en5')
all.sendp(str(newpacket),iface='en4')

while True:
    if time.clock() - start > 400*1.0/1000000:
        start = time.clock()
    # print str(newpacket)
        newpacket.payload.payload.fields["Sequence_ID"] = newpacket.payload.payload.fields["Sequence_ID"]+1
        newpacket.show()
        all.sendp(str(newpacket),iface='en5')
        all.sendp(str(newpacket),iface='en4')

<commit_msg>updated test.py<commit_after>__author__ = 'bernardyuan'

from scapy import all
from scapy.layers import  l2
import DLR
# read the pcap file
a = all.rdpcap('DLR/dlr.pcap')
import time
#in order: Beacon, nbr request, link status, locate fault,Announce, Sign on
# fake a packet and send through wireshark

#fabricate packet method 1:
#construct from keywords
# b = l2.Ether(**a[i][l2.Ether].fields)/l2.Dot1Q(**a[i][l2.Dot1Q].fields)/protocol.DLR(**a[i][protocol.DLR].fields)
# remember when constructed from keywords, do change the explicit value of ALL layers into 1
# otherwise use method 2, given below:
# b.explicit = 1
# b.payload.explicit = 1

# method 2:
# construct from raw string
# b0 = l2.Ether(str(a[0]))

# In the following code, we construct a new packet with method2, which could make your machine imitate
# a ring supervisor, and take control of the ring
# newpacket = l2.Ether(**a[30467][l2.Ether].fields)/l2.Dot1Q(**a[30467][l2.Dot1Q].fields)/DLR.DLR(**a[30467][DLR.DLR].fields)
beaconpacket = l2.Ether(**a[0][l2.Ether].fields)/l2.Dot1Q(**a[0][l2.Dot1Q].fields)/DLR.DLR(**a[0][DLR.DLR].fields)
announcepacket = l2.Ether(**a[1937][l2.Ether].fields)/l2.Dot1Q(**a[1937][l2.Dot1Q].fields)/DLR.DLR(**a[1937][DLR.DLR].fields)
# newpacket.payload.payload.fields["Node_Num"] = 1
# print newpacket.payload.payload.fields["SignOnInfo"]
# newpacket.payload.payload.fields["SignOnInfo"]["MACAddress"] = ["00:1d:9c:c8:03:e8"]
# newpacket.payload.payload.fields["SignOnInfo"]["IPAddress"] = ["192.168.0.19"]
beaconpacket.explicit = 1
beaconpacket.payload.explicit = 1

announcepacket.explicit=1
announcepacket.payload.explicit=1

beaconpacket.fields["src"] = "00:1d:9c:c8:03:e8"
announcepacket.fields["src"] = "00:1d:9c:c8:03:e8"
# newpacket.payload.payload.fields["Source_Port"] = 1
beaconpacket.payload.payload.fields["Supervisor_Precedence"] = 255
beaconpacket.payload.payload.fields["Reserved"]='\x00\x01\x02\x03\x04\x05\x06\x07\x00\x01\x02\x03\x04\x05\x06\x07\x00\x01\x02\x03'
beaconpacket.payload.payload.fields["Sequence_ID"] = 0
beaconpacket.payload.payload.fields["Source_IP"] = '192.168.0.19'
beaconpacket.payload.payload.fields["Beacon_Timeout"] = 1960
beaconpacket.payload.payload.fields["Beacon_Interval"] = 400

announcepacket.payload.payload.fields["Sequence_ID"] = 0
announcepacket.payload.payload.fields["Source_IP"] = '192.168.0.19'
# interval = newpacket.payload.payload.fields["Beacon_Interval"]
beaconstart = time.clock()
announcestart = time.clock()

announcepacket.payload.payload.fields["Ring_State"] = 2
beaconpacket.payload.payload.fields["Ring_State"] = 2
for i in range(10) :
    all.sendp(str(announcepacket),iface='en5')
    # all.sendp(str(announcepacket),iface='en4')
    all.sendp(str(beaconpacket),iface='en5')
    all.sendp(str(beaconpacket),iface='en4')

announcepacket.payload.payload.fields["Ring_State"] = 1
beaconpacket.payload.payload.fields["Ring_State"] = 1
while True:
    if time.clock() - beaconstart > 400*1.0/1000000:
        beaconstart = time.clock()
    # print str(newpacket)
        beaconpacket.payload.payload.fields["Sequence_ID"] = beaconpacket.payload.payload.fields["Sequence_ID"]+1
        beaconpacket.show()
        beaconpacket.payload.payload.fields["Reserved"]="\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
        all.sendp(str(beaconpacket),iface='en5')
        beaconpacket.payload.payload.fields["Reserved"]="\x01\x02\x03\x04\x05\x01\x02\x03\x04\x05\x01\x02\x03\x04\x05\x01\x02\x03\x04\x05"
        all.sendp(str(beaconpacket),iface='en4')
    if time.clock() - announcestart > 1:
        announcestart = time.clock()
        announcepacket.payload.payload.fields["Sequence_ID"] += 1
        all.sendp(str(announcepacket),iface='en5')
        # all.sendp(str(announcepacket),iface='en4')

<|endoftext|>


<commit_before>Specify your Secret Key, or use this random key [)y%&3rwv8jz1vuqg2a!r4s+7i!4ce8g+9&4rp!188+&ub%3n@6]: <commit_msg>Removing an old file<commit_after><|endoftext|>


<commit_before>import os
import unittest

requires_posix = unittest.skipUnless(os.name == "posix", "Requires POSIX")

@requires_posix
class PbsTestSuite(unittest.TestCase):

    def test_print_command(self):
        from pbs import ls, which
        actual_location = which("ls")
        out = str(ls)
        self.assertEqual(out, actual_location)

    def test_which(self):
        from pbs import which, ls
        self.assertEqual(which("fjoawjefojawe"), None)
        self.assertEqual(which("ls"), str(ls))

    def test_no_arg(self):
        import pwd
        from pbs import whoami
        u1 = unicode(whoami()).strip()
        u2 = pwd.getpwuid(os.geteuid())[0]
        self.assertEqual(u1, u2)

    def test_incompatible_special_args(self):
        from pbs import ls
        with self.assertRaises(TypeError):
            ls(_fg=True, _bg=True)

    def test_exception(self):
        from pbs import ls, ErrorReturnCode_2

        with self.assertRaises(ErrorReturnCode_2):
            ls("/aofwje/garogjao4a/eoan3on")

    def test_command_not_found(self):
        from pbs import CommandNotFound

        with self.assertRaises(CommandNotFound):
            from pbs import aowjgoawjoeijaowjellll

    def test_command_wrapper_equivalence(self):
        from pbs import Command, ls, which

        self.assertEqual(Command(which("ls")), ls)

    def test_short_bool_option(self):
        from pbs import id
        i1 = int(id(u=True))
        i2 = os.geteuid()
        self.assertEqual(i1, i2)

    def test_long_bool_option(self):
        from pbs import id
        i1 = int(id(user=True, real=True))
        i2 = os.getuid()
        self.assertEqual(i1, i2)

    def test_composition(self):
        from pbs import ls, wc
        c1 = int(wc(ls(A=True), l=True))
        c2 = len(os.listdir("."))
        self.assertEqual(c1, c2)

    def test_short_option(self):
        from pbs import sh
        s1 = unicode(sh(c="echo test")).strip()
        s2 = "test"
        self.assertEqual(s1, s2)

    def test_long_option(self):
        from pbs import sed, echo
        out = unicode(sed(echo("test"), expression="s/test/lol/")).strip()
        self.assertEqual(out, "lol")

    def test_command_wrapper(self):
        from pbs import Command, which

        ls = Command(which("ls"))
        wc = Command(which("wc"))

        c1 = int(wc(ls(A=True), l=True))
        c2 = len(os.listdir("."))
        self.assertEqual(c1, c2)

    def test_background(self):
        from pbs import sleep
        import time

        start = time.time()
        sleep_time = .5
        p = sleep(sleep_time, _bg=True)

        now = time.time()
        self.assertTrue(now - start < sleep_time)

        p.wait()
        now = time.time()
        self.assertTrue(now - start > sleep_time)

    def test_with_context(self):
        from pbs import time, ls
        with time:
            out = ls().stderr
        self.assertTrue("pagefaults" in out)

    def test_with_context_args(self):
        from pbs import time, ls
        with time(verbose=True, _with=True):
            out = ls().stderr
        self.assertTrue("Voluntary context switches" in out)

    def test_err_to_out(self):
        from pbs import time, ls
        with time(_with=True):
            out = ls(_err_to_out=True)

        self.assertTrue("pagefaults" in out)

    def test_out_redirection(self):
        import tempfile
        from pbs import ls

        file_obj = tempfile.TemporaryFile()
        out = ls(_out=file_obj)

        self.assertTrue(len(out) == 0)

        file_obj.seek(0)
        actual_out = file_obj.read()

        self.assertTrue(len(actual_out) != 0)

    def test_err_redirection(self):
        import tempfile
        from pbs import time, ls

        file_obj = tempfile.TemporaryFile()

        with time(_with=True):
            out = ls(_err=file_obj)

        file_obj.seek(0)
        actual_out = file_obj.read()

        self.assertTrue(len(actual_out) != 0)

    def test_subcommand(self):
        from pbs import time

        out = time.ls(_err_to_out=True)
        self.assertTrue("pagefaults" in out)

    def test_bake(self):
        from pbs import time, ls
        timed = time.bake("--verbose", _err_to_out=True)
        out = timed.ls()
        self.assertTrue("Voluntary context switches" in out)

    def test_bake_args_come_first(self):
        from pbs import ls
        ls = ls.bake(full_time=True)

        ran = ls("-la").command_ran
        ft = ran.index("full-time")
        self.assertTrue("-la" in ran[ft:])

    def test_output_equivalence(self):
        from pbs import whoami

        iam1 = whoami()
        iam2 = whoami()

        self.assertEqual(iam1, iam2)

    def test_stdout_callback(self):
        import tempfile
        from pbs import python

        py = tempfile.NamedTemporaryFile()

        py.write("""
import sys
import os

# unbuffered stdout
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)

for i in xrange(5): print i
""")
        py.flush()

        stdout = []
        def agg(line): stdout.append(line)

        p = python(py.name, _out=agg)
        p.wait()

        self.assertTrue(len(stdout) == 5)

    def test_stdout_callback_no_wait(self):
        import tempfile
        from pbs import python
        import time

        py = tempfile.NamedTemporaryFile()

        py.write("""
import sys
import os
import time

# unbuffered stdout
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)

for i in xrange(5):
    print i
    time.sleep(.5)
""")
        py.flush()

        stdout = []
        def agg(line): stdout.append(line)

        p = python(py.name, _out=agg)

        # we give a little pause to make sure that the NamedTemporaryFile
        # exists when the python process is actually called
        time.sleep(.5)

        self.assertTrue(len(stdout) != 5)

    def test_stdout_callback_line_buffered(self):
        import tempfile
        from pbs import python

        py = tempfile.NamedTemporaryFile()

        py.write("""
import sys
import os

# unbuffered stdout
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)

for i in xrange(5): print "herpderp"
""")
        py.flush()

        stdout = []
        def agg(line): stdout.append(line)

        p = python(py.name, _out=agg, _bufsize=1)
        p.wait()

        self.assertTrue(len(stdout) == 5)

    def test_stdout_callback_line_unbuffered(self):
        import tempfile
        from pbs import python

        py = tempfile.NamedTemporaryFile()

        py.write("""
import sys
import os

# unbuffered stdout
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)

for i in xrange(5): print "herpderp"
""")
        py.flush()

        stdout = []
        def agg(char): stdout.append(char)

        p = python(py.name, _out=agg, _bufsize=0)
        p.wait()

        # + 5 newlines
        self.assertTrue(len(stdout) == (len("herpderp") * 5 + 5))

    def test_stdout_callback_buffered(self):
        import tempfile
        from pbs import python

        py = tempfile.NamedTemporaryFile()

        py.write("""
import sys
import os

# unbuffered stdout
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)

for i in xrange(5): sys.stdout.write("herpderp")
""")
        py.flush()

        stdout = []
        def agg(chunk): stdout.append(chunk)

        p = python(py.name, _out=agg, _bufsize=4)
        p.wait()

        self.assertTrue(len(stdout) == (len("herp")/2 * 5))

    def test_stdout_callback_with_input(self):
        import tempfile
        from pbs import python

        py = tempfile.NamedTemporaryFile()

        py.write("""
import sys
import os

# unbuffered stdout
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)

for i in xrange(5): print i
derp = raw_input("herp? ")
print derp
""")
        py.flush()

        def agg(line, stdin):
            if line.strip() == "4": stdin.put("derp\n")

        p = python(py.name, _out=agg)
        p.wait()

        self.assertTrue("derp" in p)

    def test_stdout_callback_exit(self):
        import tempfile
        from pbs import python

        py = tempfile.NamedTemporaryFile()

        py.write("""
import sys
import os

# unbuffered stdout
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)

for i in xrange(5): print i
""")
        py.flush()

        stdout = []
        def agg(line):
            line = line.strip()
            stdout.append(line)
            if line == "2": return True

        p = python(py.name, _out=agg)
        p.wait()

        self.assertTrue("4" in p)
        self.assertTrue("4" not in stdout)

    def test_stdout_callback_terminate(self):
        import tempfile
        from pbs import python

        py = tempfile.NamedTemporaryFile()

        py.write("""
import sys
import os
import time

# unbuffered stdout
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)

for i in xrange(5):
    print i
    time.sleep(.5)
""")
        py.flush()

        stdout = []
        def agg(line, stdin, process):
            line = line.strip()
            stdout.append(line)
            if line == "0":
                process.terminate()
                return True

        p = python(py.name, _out=agg)
        p.wait()

        self.assertTrue("4" not in p)
        self.assertTrue("4" not in stdout)

    def test_stdout_callback_kill(self):
        import tempfile
        from pbs import python

        py = tempfile.NamedTemporaryFile()

        py.write("""
import sys
import os
import time

# unbuffered stdout
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)

for i in xrange(5):
    print i
    time.sleep(.5)
""")
        py.flush()

        stdout = []
        def agg(line, stdin, process):
            line = line.strip()
            stdout.append(line)
            if line == "0":
                process.kill()
                return True

        p = python(py.name, _out=agg)
        p.wait()

        self.assertTrue("4" not in p)
        self.assertTrue("4" not in stdout)

if __name__ == "__main__":
    unittest.main()
<commit_msg>refactored out code-creating function<commit_after>import os
import unittest
import tempfile

requires_posix = unittest.skipUnless(os.name == "posix", "Requires POSIX")

def create_tmp_test(code):
    py = tempfile.NamedTemporaryFile()
    py.write(code)
    py.flush()
    return py

@requires_posix
class PbsTestSuite(unittest.TestCase):

    def test_print_command(self):
        from pbs import ls, which
        actual_location = which("ls")
        out = str(ls)
        self.assertEqual(out, actual_location)

    def test_which(self):
        from pbs import which, ls
        self.assertEqual(which("fjoawjefojawe"), None)
        self.assertEqual(which("ls"), str(ls))

    def test_no_arg(self):
        import pwd
        from pbs import whoami
        u1 = unicode(whoami()).strip()
        u2 = pwd.getpwuid(os.geteuid())[0]
        self.assertEqual(u1, u2)

    def test_incompatible_special_args(self):
        from pbs import ls
        with self.assertRaises(TypeError):
            ls(_fg=True, _bg=True)

    def test_exception(self):
        from pbs import ls, ErrorReturnCode_2

        with self.assertRaises(ErrorReturnCode_2):
            ls("/aofwje/garogjao4a/eoan3on")

    def test_command_not_found(self):
        from pbs import CommandNotFound

        with self.assertRaises(CommandNotFound):
            from pbs import aowjgoawjoeijaowjellll

    def test_command_wrapper_equivalence(self):
        from pbs import Command, ls, which

        self.assertEqual(Command(which("ls")), ls)

    def test_short_bool_option(self):
        from pbs import id
        i1 = int(id(u=True))
        i2 = os.geteuid()
        self.assertEqual(i1, i2)

    def test_long_bool_option(self):
        from pbs import id
        i1 = int(id(user=True, real=True))
        i2 = os.getuid()
        self.assertEqual(i1, i2)

    def test_composition(self):
        from pbs import ls, wc
        c1 = int(wc(ls(A=True), l=True))
        c2 = len(os.listdir("."))
        self.assertEqual(c1, c2)

    def test_short_option(self):
        from pbs import sh
        s1 = unicode(sh(c="echo test")).strip()
        s2 = "test"
        self.assertEqual(s1, s2)

    def test_long_option(self):
        from pbs import sed, echo
        out = unicode(sed(echo("test"), expression="s/test/lol/")).strip()
        self.assertEqual(out, "lol")

    def test_command_wrapper(self):
        from pbs import Command, which

        ls = Command(which("ls"))
        wc = Command(which("wc"))

        c1 = int(wc(ls(A=True), l=True))
        c2 = len(os.listdir("."))
        self.assertEqual(c1, c2)

    def test_background(self):
        from pbs import sleep
        import time

        start = time.time()
        sleep_time = .5
        p = sleep(sleep_time, _bg=True)

        now = time.time()
        self.assertTrue(now - start < sleep_time)

        p.wait()
        now = time.time()
        self.assertTrue(now - start > sleep_time)

    def test_with_context(self):
        from pbs import time, ls
        with time:
            out = ls().stderr
        self.assertTrue("pagefaults" in out)

    def test_with_context_args(self):
        from pbs import time, ls
        with time(verbose=True, _with=True):
            out = ls().stderr
        self.assertTrue("Voluntary context switches" in out)

    def test_err_to_out(self):
        from pbs import time, ls
        with time(_with=True):
            out = ls(_err_to_out=True)

        self.assertTrue("pagefaults" in out)

    def test_out_redirection(self):
        import tempfile
        from pbs import ls

        file_obj = tempfile.TemporaryFile()
        out = ls(_out=file_obj)

        self.assertTrue(len(out) == 0)

        file_obj.seek(0)
        actual_out = file_obj.read()

        self.assertTrue(len(actual_out) != 0)

    def test_err_redirection(self):
        import tempfile
        from pbs import time, ls

        file_obj = tempfile.TemporaryFile()

        with time(_with=True):
            out = ls(_err=file_obj)

        file_obj.seek(0)
        actual_out = file_obj.read()

        self.assertTrue(len(actual_out) != 0)

    def test_subcommand(self):
        from pbs import time

        out = time.ls(_err_to_out=True)
        self.assertTrue("pagefaults" in out)

    def test_bake(self):
        from pbs import time, ls
        timed = time.bake("--verbose", _err_to_out=True)
        out = timed.ls()
        self.assertTrue("Voluntary context switches" in out)

    def test_bake_args_come_first(self):
        from pbs import ls
        ls = ls.bake(full_time=True)

        ran = ls("-la").command_ran
        ft = ran.index("full-time")
        self.assertTrue("-la" in ran[ft:])

    def test_output_equivalence(self):
        from pbs import whoami

        iam1 = whoami()
        iam2 = whoami()

        self.assertEqual(iam1, iam2)

    def test_stdout_callback(self):
        from pbs import python

        py = create_tmp_test("""
import sys
import os

# unbuffered stdout
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)

for i in xrange(5): print i
""")

        stdout = []
        def agg(line): stdout.append(line)

        p = python(py.name, _out=agg)
        p.wait()

        self.assertTrue(len(stdout) == 5)

    def test_stdout_callback_no_wait(self):
        from pbs import python
        import time

        py = create_tmp_test("""
import sys
import os
import time

# unbuffered stdout
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)

for i in xrange(5):
    print i
    time.sleep(.5)
""")

        stdout = []
        def agg(line): stdout.append(line)

        p = python(py.name, _out=agg)

        # we give a little pause to make sure that the NamedTemporaryFile
        # exists when the python process is actually called
        time.sleep(.5)

        self.assertTrue(len(stdout) != 5)

    def test_stdout_callback_line_buffered(self):
        from pbs import python

        py = create_tmp_test("""
import sys
import os

# unbuffered stdout
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)

for i in xrange(5): print "herpderp"
""")

        stdout = []
        def agg(line): stdout.append(line)

        p = python(py.name, _out=agg, _bufsize=1)
        p.wait()

        self.assertTrue(len(stdout) == 5)

    def test_stdout_callback_line_unbuffered(self):
        from pbs import python

        py = create_tmp_test("""
import sys
import os

# unbuffered stdout
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)

for i in xrange(5): print "herpderp"
""")

        stdout = []
        def agg(char): stdout.append(char)

        p = python(py.name, _out=agg, _bufsize=0)
        p.wait()

        # + 5 newlines
        self.assertTrue(len(stdout) == (len("herpderp") * 5 + 5))

    def test_stdout_callback_buffered(self):
        from pbs import python

        py = create_tmp_test("""
import sys
import os

# unbuffered stdout
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)

for i in xrange(5): sys.stdout.write("herpderp")
""")

        stdout = []
        def agg(chunk): stdout.append(chunk)

        p = python(py.name, _out=agg, _bufsize=4)
        p.wait()

        self.assertTrue(len(stdout) == (len("herp")/2 * 5))

    def test_stdout_callback_with_input(self):
        from pbs import python

        py = create_tmp_test("""
import sys
import os

# unbuffered stdout
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)

for i in xrange(5): print i
derp = raw_input("herp? ")
print derp
""")

        def agg(line, stdin):
            if line.strip() == "4": stdin.put("derp\n")

        p = python(py.name, _out=agg)
        p.wait()

        self.assertTrue("derp" in p)

    def test_stdout_callback_exit(self):
        from pbs import python

        py = create_tmp_test("""
import sys
import os

# unbuffered stdout
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)

for i in xrange(5): print i
""")

        stdout = []
        def agg(line):
            line = line.strip()
            stdout.append(line)
            if line == "2": return True

        p = python(py.name, _out=agg)
        p.wait()

        self.assertTrue("4" in p)
        self.assertTrue("4" not in stdout)

    def test_stdout_callback_terminate(self):
        from pbs import python

        py = create_tmp_test("""
import sys
import os
import time

# unbuffered stdout
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)

for i in xrange(5):
    print i
    time.sleep(.5)
""")

        stdout = []
        def agg(line, stdin, process):
            line = line.strip()
            stdout.append(line)
            if line == "0":
                process.terminate()
                return True

        p = python(py.name, _out=agg)
        p.wait()

        self.assertTrue("4" not in p)
        self.assertTrue("4" not in stdout)

    def test_stdout_callback_kill(self):
        from pbs import python

        py = create_tmp_test("""
import sys
import os
import time

# unbuffered stdout
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)

for i in xrange(5):
    print i
    time.sleep(.5)
""")

        stdout = []
        def agg(line, stdin, process):
            line = line.strip()
            stdout.append(line)
            if line == "0":
                process.kill()
                return True

        p = python(py.name, _out=agg)
        p.wait()

        self.assertTrue("4" not in p)
        self.assertTrue("4" not in stdout)

if __name__ == "__main__":
    unittest.main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-u

"""
Purpose : Some tests for the HaveiBeenPwnedApi class

To run the tests : $ python -m unittest -v test
"""

import unittest
from pwnedapi import HaveIBeenPwnedApi
from exception import NotValidEmail, BreachNotFound, UnvalidParameters

class TestHaveIBeenPwnedApi(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        """ creating test data set for the test module """
        cls.api = HaveIBeenPwnedApi()

    def test_get_all_dataclasses(self):
        dataclasses = self.api.get_all_dataclasses()
        self.assertIn("Email addresses", dataclasses)
        self.assertIn("Passwords", dataclasses)
        self.assertIn("Usernames", dataclasses)
        self.assertNotIn("Adobe", dataclasses)

    def test_get_all_breaches_name(self):
        breaches = self.api.get_all_breaches_name()
        self.assertIn("AshleyMadison", breaches)
        self.assertIn("Adobe", breaches)
        self.assertNotIn("Google", breaches)

    def test_get_breach(self):
        self.assertEqual("AshleyMadison", self.api.get_breach("AshleyMadison")['Name'])
        self.assertIsInstance(self.api.get_breach("AshleyMadison"), dict)
        self.assertRaises(BreachNotFound, self.api.get_breach, "Google")

    def test_regex_email(self):
        self.assertTrue(self.api.regex_email.match("ericfourrier0@gmail.com"))
        self.assertIsNone(self.api.regex_email.match("tvhmxoleoubgmail.com"))

    def test_check_email(self):
        self.assertEqual(self.api.check_email("ericfourrier0@gmail.com"), [])
        self.assertEqual(self.api.check_email("ericfourrier1@hotmail.com"), [])
        self.assertRaises(NotValidEmail, self.api.check_email, "tvhmxoleoubgmail.com")

    def test_check_pastebin(self):
        self.assertEqual(self.api.check_pastebin("ericfourrier0@gmail.com"), [])
        self.assertEqual(self.api.check_pastebin("ericfourrier1@hotmail.com"), [])
        self.assertNotEqual(self.api.check_pastebin("tvhmxoleoub@gmail.com"), [])
        self.assertRaises(NotValidEmail, self.api.check_pastebin, "tvhmxoleoubgmail.com")
<commit_msg>Correct typo in test.py<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-u

"""
Purpose : Some tests for the HaveiBeenPwnedApi class

To run the tests : $ python -m unittest -v test
"""

import unittest
from pwnedapi.pwnedapi import HaveIBeenPwnedApi
from pwnedapi.exception import NotValidEmail, BreachNotFound, UnvalidParameters

class TestHaveIBeenPwnedApi(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        """ creating test data set for the test module """
        cls.api = HaveIBeenPwnedApi()

    def test_get_all_dataclasses(self):
        dataclasses = self.api.get_all_dataclasses()
        self.assertIn("Email addresses", dataclasses)
        self.assertIn("Passwords", dataclasses)
        self.assertIn("Usernames", dataclasses)
        self.assertNotIn("Adobe", dataclasses)

    def test_get_all_breaches_name(self):
        breaches = self.api.get_all_breaches_name()
        self.assertIn("AshleyMadison", breaches)
        self.assertIn("Adobe", breaches)
        self.assertNotIn("Google", breaches)

    def test_get_breach(self):
        self.assertEqual("AshleyMadison", self.api.get_breach("AshleyMadison")['Name'])
        self.assertIsInstance(self.api.get_breach("AshleyMadison"), dict)
        self.assertRaises(BreachNotFound, self.api.get_breach, "Google")

    def test_regex_email(self):
        self.assertTrue(self.api.regex_email.match("ericfourrier0@gmail.com"))
        self.assertIsNone(self.api.regex_email.match("tvhmxoleoubgmail.com"))

    def test_check_email(self):
        self.assertEqual(self.api.check_email("ericfourrier0@gmail.com"), [])
        self.assertEqual(self.api.check_email("ericfourrier1@hotmail.com"), [])
        self.assertRaises(NotValidEmail, self.api.check_email, "tvhmxoleoubgmail.com")

    def test_check_pastebin(self):
        self.assertEqual(self.api.check_pastebin("ericfourrier0@gmail.com"), [])
        self.assertEqual(self.api.check_pastebin("ericfourrier1@hotmail.com"), [])
        self.assertNotEqual(self.api.check_pastebin("tvhmxoleoub@gmail.com"), [])
        self.assertRaises(NotValidEmail, self.api.check_pastebin, "tvhmxoleoubgmail.com")
<|endoftext|>


<commit_before>
n = int(raw_input("Enter a number divisible by three, and I will give you its cube:  "))

def cube(n):
	return n**3

def by_three(n):
	i = 0
	while (n%3 !=0):
		too_many_tries = (i>2)
		if too_many_tries:
			return "Too Many Tries. You only get three."
		else:
			print "%f is not divisble by three" % n
			n = int(raw_input("Try again:  "))
		i += 1
	return "Cube of %d is %d" % (n,cube(n))

print by_three(n)

<commit_msg>test.py modified on local at deY<commit_after>
n = int(raw_input("Enter a number divisible by three, and I will give you its cube:  "))

def cube(n):
	return n**3

def by_three(n):
	i = 0
	while (n%3 !=0):
		too_many_tries = (i>2)
		if too_many_tries:
			return "Too Many Tries. You only get three."
		else:
			print "%f is not divisble by three" % n
			n = int(raw_input("Try again:  "))
		i += 1
	return "Cube of %d is %d" % (n,cube(n))

print by_three(n)
<|endoftext|>


<commit_before>from robofab.world import RFont

def to_robofab(data):
    """Take .glyphs file data and load it into RFonts.

    Takes in data as a dictionary structured according to
    https://github.com/schriftgestalt/GlyphsSDK/blob/master/GlyphsFileFormat.md
    and returns a list of RFonts, one per master.
    """

    family_name = data['familyName']
    unitsPerEm = data['unitsPerEm']

    #TODO(jamesgk) maybe create one font at a time to reduce memory usage
    rfonts = {}
    for master in data['fontMaster']:
        rfont = RFont()
        rfont.info.familyName = family_name
        rfont.info.unitsPerEm = unitsPerEm
        rfont.info.ascender = master['ascender']
        rfont.info.capHeight = master['capHeight']
        rfont.info.descender = master['descender']
        rfont.info.xHeight = master['xHeight']
        rfonts[master['id']] = rfont

    for glyph in data['glyphs']:
        glyph_name = glyph['glyphname']
        glyph_unicode = glyph.get('unicode')

        for layer in glyph['layers']:
            try:
                rfont = rfonts[layer['layerId']]
            except KeyError:
                print 'Odd data, layer with id', layer['layerId']
                continue

            style_name = layer['name']
            if rfont.info.styleName:
                assert rfont.info.styleName == style_name, ('Inconsistent '
                    'layer id/name pairs between glyphs')
            else:
                rfont.info.styleName = style_name

            glyph = rfont.newGlyph(glyph_name)
            if glyph_unicode:
                glyph.unicode = glyph_unicode
            to_glyph(glyph, layer)

    for master_id, kerning in data['kerning'].items():
        rfont = rfonts[master_id]
        for left, pairs in kerning.items():
            for right, kerning_val in pairs.items():
                rfont.kerning[left, right] = kerning_val

    result = []
    for rfont in rfonts.values():
        rfont.info.postscriptFullName = (
            '%s-%s' % (rfont.info.familyName.replace(' ', ''),
                       rfont.info.styleName.replace(' ', '')))
        result.append(rfont)
    return result

def to_glyph(glyph, layer):
    """Add .glyphs paths, components, and anchors as applicable to an RGlyph."""

    pen = glyph.getPointPen()
    draw_paths(pen, layer.get('paths', []))
    draw_components(pen, layer.get('components', []))
    add_anchors(glyph, layer.get('anchors', []))
    glyph.width = layer['width']

def draw_paths(pen, paths):
    """Draw .glyphs paths onto a pen."""

    for path in paths:
        pen.beginPath()
        '''
        closed = path.get('closed')
        if not closed:
            x, y, node_type, smooth = path['nodes'][0]
            assert node_type == 'LINE', 'Open path starts with off-curve points'
            pen.addPoint((x, y), 'move')
            path['nodes'] = path['nodes'][1:]
        '''
        for x, y, node_type, smooth in path['nodes']:
            node_type = node_type.lower()
            if node_type not in ['line', 'curve']:
                node_type = None
            pen.addPoint((x, y), node_type, smooth)
        pen.endPath()

def draw_components(pen, components):
    """Draw .glyphs components onto a pen, adding them to the parent RGlyph."""

    for component in components:
        pen.addComponent(component['name'],
                         component.get('transform', (1, 0, 0, 1, 0, 0)))

def add_anchors(glyph, anchors):
    """Add .glyphs anchors to an RGlyph."""

    for anchor in anchors:
        glyph.appendAnchor(anchor['name'], anchor['position'])
<commit_msg>Load more info into RFonts, including features.<commit_after>from robofab.world import RFont

def to_robofab(data):
    """Take .glyphs file data and load it into RFonts.

    Takes in data as a dictionary structured according to
    https://github.com/schriftgestalt/GlyphsSDK/blob/master/GlyphsFileFormat.md
    and returns a list of RFonts, one per master.
    """

    copyright = data['copyright']
    date_created = data['date'].strftime('%Y/%m/%d %H:%M:%S')
    designer = data['designer']
    designer_url = data['designerURL']
    family_name = data['familyName']
    manufacturer = data['manufacturer']
    manufacturer_url = data['manufacturerURL']
    units_per_em = data['unitsPerEm']
    version_major = data['versionMajor']
    version_minor = data['versionMinor']

    feature_prefixes = [f['code'] for f in data['featurePrefixes']]
    classes = [(c['name'], c['code']) for c in data['classes']]
    features = [(f['name'], f['code']) for f in data['features']]

    #TODO(jamesgk) maybe create one font at a time to reduce memory usage
    rfonts = {}
    for master in data['fontMaster']:
        rfont = RFont()

        rfont.info.copyright = copyright
        rfont.info.openTypeNameDesigner = designer
        rfont.info.openTypeNameDesignerURL = designer_url
        rfont.info.familyName = family_name
        rfont.info.openTypeNameManufacturer = manufacturer
        rfont.info.openTypeNameManufacturerURL = manufacturer_url
        rfont.info.openTypeHeadCreated = date_created
        rfont.info.unitsPerEm = units_per_em
        rfont.into.versionMajor = version_major
        rfont.info.versionMinor = version_minor

        rfont.info.ascender = master['ascender']
        rfont.info.capHeight = master['capHeight']
        rfont.info.descender = master['descender']
        rfont.info.postscriptStemSnapH = master['horizontalStems']
        rfont.info.postscriptStemSnapV = master['verticalStems']
        rfont.info.xHeight = master['xHeight']

        rfonts[master['id']] = rfont

    for glyph in data['glyphs']:
        glyph_name = glyph['glyphname']
        glyph_unicode = glyph.get('unicode')

        for layer in glyph['layers']:
            try:
                rfont = rfonts[layer['layerId']]
            except KeyError:
                print 'Odd data, layer with id', layer['layerId']
                continue

            style_name = layer['name']
            if rfont.info.styleName:
                assert rfont.info.styleName == style_name, ('Inconsistent '
                    'layer id/name pairs between glyphs')
            else:
                rfont.info.styleName = style_name

            glyph = rfont.newGlyph(glyph_name)
            if glyph_unicode:
                glyph.unicode = glyph_unicode
            to_glyph(glyph, layer)

    for master_id, kerning in data['kerning'].items():
        rfont = rfonts[master_id]
        for left, pairs in kerning.items():
            for right, kerning_val in pairs.items():
                rfont.kerning[left, right] = kerning_val

    result = []
    for rfont in rfonts.values():
        write_features(rfont, feature_prefixes, classes, features)
        rfont.info.postscriptFullName = (
            '%s-%s' % (rfont.info.familyName.replace(' ', ''),
                       rfont.info.styleName.replace(' ', '')))
        result.append(rfont)
    return result

def to_glyph(glyph, layer):
    """Add .glyphs paths, components, and anchors as applicable to an RGlyph."""

    pen = glyph.getPointPen()
    draw_paths(pen, layer.get('paths', []))
    draw_components(pen, layer.get('components', []))
    add_anchors(glyph, layer.get('anchors', []))
    glyph.width = layer['width']

def draw_paths(pen, paths):
    """Draw .glyphs paths onto a pen."""

    for path in paths:
        pen.beginPath()
        '''
        closed = path.get('closed')
        if not closed:
            x, y, node_type, smooth = path['nodes'][0]
            assert node_type == 'LINE', 'Open path starts with off-curve points'
            pen.addPoint((x, y), 'move')
            path['nodes'] = path['nodes'][1:]
        '''
        for x, y, node_type, smooth in path['nodes']:
            node_type = node_type.lower()
            if node_type not in ['line', 'curve']:
                node_type = None
            pen.addPoint((x, y), node_type, smooth)
        pen.endPath()

def draw_components(pen, components):
    """Draw .glyphs components onto a pen, adding them to the parent RGlyph."""

    for component in components:
        pen.addComponent(component['name'],
                         component.get('transform', (1, 0, 0, 1, 0, 0)))

def add_anchors(glyph, anchors):
    """Add .glyphs anchors to an RGlyph."""

    for anchor in anchors:
        glyph.appendAnchor(anchor['name'], anchor['position'])

def write_features(font, feature_prefixes, classes, features):
    """Write an RFont's OpenType feature file."""

    text = ''
    for feature_prefix in feature_prefixes:
        text += feature_prefix + '\n'
    text += '\n'
    for class_info in classes:
        text += '@%s = [%s];\n' % class_info
    text += '\n'
    for name, code in features:
        text += '%s {\n%s\n} %s;\n\n' % (name, code, name)
    font.features.text = text
<|endoftext|>