<commit_before># Copyright (c) 2005 The Regents of The University of Michigan
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Nathan Binkert

import code
import datetime
import os
import socket
import sys

from util import attrdict
import config
from options import OptionParser

__all__ = [ 'options', 'arguments', 'main' ]

def print_list(items, indent=4):
    line = ' ' * indent
    for i,item in enumerate(items):
        if len(line) + len(item) > 76:
            print line
            line = ' ' * indent

        if i < len(items) - 1:
            line += '%s, ' % item
        else:
            line += item
            print line

usage="%prog [m5 options] script.py [script options]"
version="%prog 2.0"
brief_copyright='''
Copyright (c) 2001-2008
The Regents of The University of Michigan
All Rights Reserved
'''

options = OptionParser(usage=usage, version=version,
                       description=brief_copyright)
add_option = options.add_option
set_group = options.set_group
usage = options.usage

# Help options
add_option('-A', "--authors", action="store_true", default=False,
    help="Show author information")
add_option('-B', "--build-info", action="store_true", default=False,
    help="Show build information")
add_option('-C', "--copyright", action="store_true", default=False,
    help="Show full copyright information")
add_option('-R', "--readme", action="store_true", default=False,
    help="Show the readme")
add_option('-N', "--release-notes", action="store_true", default=False,
    help="Show the release notes")

# Options for configuring the base simulator
add_option('-d', "--outdir", metavar="DIR", default=".",
    help="Set the output directory to DIR [Default: %default]")
add_option('-r', "--redirect-stdout", action="store_true", default=False,
           help="Redirect stdout (& stderr, without -e) to file")
add_option('-e', "--redirect-stderr", action="store_true", default=False,
           help="Redirect stderr to file")
add_option("--stdout-file", metavar="FILE", default="simout",
           help="Filename for -r redirection [Default: %default]")
add_option("--stderr-file", metavar="FILE", default="simerr",
           help="Filename for -e redirection [Default: %default]")
add_option('-i', "--interactive", action="store_true", default=False,
    help="Invoke the interactive interpreter after running the script")
add_option("--pdb", action="store_true", default=False,
    help="Invoke the python debugger before running the script")
add_option('-p', "--path", metavar="PATH[:PATH]", action='append', split=':',
    help="Prepend PATH to the system path when invoking the script")
add_option('-q', "--quiet", action="count", default=0,
    help="Reduce verbosity")
add_option('-v', "--verbose", action="count", default=0,
    help="Increase verbosity")

# Statistics options
set_group("Statistics Options")
add_option("--stats-file", metavar="FILE", default="m5stats.txt",
    help="Sets the output file for statistics [Default: %default]")

# Debugging options
set_group("Debugging Options")
add_option("--debug-break", metavar="TIME[,TIME]", action='append', split=',',
    help="Cycle to create a breakpoint")
add_option("--remote-gdb-port", type='int', default=7000,
    help="Remote gdb base port")

# Tracing options
set_group("Trace Options")
add_option("--trace-help", action='store_true',
    help="Print help on trace flags")
add_option("--trace-flags", metavar="FLAG[,FLAG]", action='append', split=',',
    help="Sets the flags for tracing (-FLAG disables a flag)")
add_option("--trace-start", metavar="TIME", type='int',
    help="Start tracing at TIME (must be in ticks)")
add_option("--trace-file", metavar="FILE", default="cout",
    help="Sets the output file for tracing [Default: %default]")
add_option("--trace-ignore", metavar="EXPR", action='append', split=':',
    help="Ignore EXPR sim objects")

# Help options
set_group("Help Options")
add_option("--list-sim-objects", action='store_true', default=False,
    help="List all built-in SimObjects, their parameters and default values")

def main():
    import event
    import info
    import internal

    # load the options.py config file to allow people to set their own
    # default options
    options_file = config.get('options.py')
    if options_file:
        scope = { 'options' : options }
        execfile(options_file, scope)

    arguments = options.parse_args()

    if not os.path.isdir(options.outdir):
        os.makedirs(options.outdir)

    # These filenames are used only if the redirect_std* options are set
    stdout_file = os.path.join(options.outdir, options.stdout_file)
    stderr_file = os.path.join(options.outdir, options.stderr_file)

    # Print redirection notices here before doing any redirection
    if options.redirect_stdout and not options.redirect_stderr:
        print "Redirecting stdout and stderr to", stdout_file
    else:
        if options.redirect_stdout:
            print "Redirecting stdout to", stdout_file
        if options.redirect_stderr:
            print "Redirecting stderr to", stderr_file

    # Now redirect stdout/stderr as desired
    if options.redirect_stdout:
        redir_fd = os.open(stdout_file, os. O_WRONLY | os.O_CREAT | os.O_TRUNC)
        os.dup2(redir_fd, sys.stdout.fileno())
        if not options.redirect_stderr:
            os.dup2(redir_fd, sys.stderr.fileno())

    if options.redirect_stderr:
        redir_fd = os.open(stderr_file, os. O_WRONLY | os.O_CREAT | os.O_TRUNC)
        os.dup2(redir_fd, sys.stderr.fileno())

    done = False

    if options.build_info:
        import defines

        done = True
        print 'Build information:'
        print
        print 'compiled %s' % internal.core.cvar.compileDate;
        print "revision %s" % internal.core.cvar.hgRev
        print "commit date %s" % internal.core.cvar.hgDate
        print 'build options:'
        keys = defines.m5_build_env.keys()
        keys.sort()
        for key in keys:
            val = defines.m5_build_env[key]
            print '    %s = %s' % (key, val)
        print

    if options.copyright:
        done = True
        print info.LICENSE
        print

    if options.authors:
        done = True
        print 'Author information:'
        print
        print info.AUTHORS
        print

    if options.readme:
        done = True
        print 'Readme:'
        print
        print info.README
        print

    if options.release_notes:
        done = True
        print 'Release Notes:'
        print
        print info.RELEASE_NOTES
        print

    if options.trace_help:
        import traceflags

        done = True
        print "Base Flags:"
        traceflags.baseFlags.sort()
        print_list(traceflags.baseFlags, indent=4)
        print
        print "Compound Flags:"
        traceflags.compoundFlags.sort()
        for flag in traceflags.compoundFlags:
            if flag == 'All':
                continue
            print "    %s:" % flag
            print_list(traceflags.compoundFlagMap[flag], indent=8)
            print

    if options.list_sim_objects:
        import SimObject
        done = True
        print "SimObjects:"
        objects = SimObject.allClasses.keys()
        objects.sort()
        for name in objects:
            obj = SimObject.allClasses[name]
            print "    %s" % obj
            params = obj._params.keys()
            params.sort()
            for pname in params:
                param = obj._params[pname]
                default = getattr(param, 'default', '')
                print "        %s" % pname
                if default:
                    print "            default: %s" % default
                print "            desc: %s" % param.desc
                print
            print

    if done:
        sys.exit(0)

    # setting verbose and quiet at the same time doesn't make sense
    if options.verbose > 0 and options.quiet > 0:
        options.usage(2)

    verbose = options.verbose - options.quiet
    if options.verbose >= 0:
        print "M5 Simulator System"
        print brief_copyright
        print
        print "M5 compiled %s" % internal.core.cvar.compileDate;
        print "M5 revision %s" % internal.core.cvar.hgRev
        print "M5 commit date %s" % internal.core.cvar.hgDate

        print "M5 started %s" % datetime.datetime.now().strftime("%b %e %Y %X")
        print "M5 executing on %s" % socket.gethostname()

        print "command line:",
        for argv in sys.argv:
            print argv,
        print

    # check to make sure we can find the listed script
    if not arguments or not os.path.isfile(arguments[0]):
        if arguments and not os.path.isfile(arguments[0]):
            print "Script %s not found" % arguments[0]

        options.usage(2)

    # tell C++ about output directory
    internal.core.setOutputDir(options.outdir)

    # update the system path with elements from the -p option
    sys.path[0:0] = options.path

    import objects

    # set stats options
    internal.stats.initText(options.stats_file)

    # set debugging options
    internal.debug.setRemoteGDBPort(options.remote_gdb_port)
    for when in options.debug_break:
        internal.debug.schedBreakCycle(int(when))

    if options.trace_flags:
        import traceflags

        on_flags = []
        off_flags = []
        for flag in options.trace_flags:
            off = False
            if flag.startswith('-'):
                flag = flag[1:]
                off = True
            if flag not in traceflags.allFlags:
                print >>sys.stderr, "invalid trace flag '%s'" % flag
                sys.exit(1)

            if off:
                off_flags.append(flag)
            else:
                on_flags.append(flag)

        for flag in on_flags:
            internal.trace.set(flag)

        for flag in off_flags:
            internal.trace.clear(flag)

    if options.trace_start:
        def enable_trace():
            internal.trace.cvar.enabled = True

        e = event.create(enable_trace)
        event.mainq.schedule(e, options.trace_start)
    else:
        internal.trace.cvar.enabled = True

    internal.trace.output(options.trace_file)

    for ignore in options.trace_ignore:
        internal.trace.ignore(ignore)

    sys.argv = arguments
    sys.path = [ os.path.dirname(sys.argv[0]) ] + sys.path

    filename = sys.argv[0]
    filedata = file(filename, 'r').read()
    filecode = compile(filedata, filename, 'exec')
    scope = { '__file__' : filename,
              '__name__' : '__m5_main__' }

    # we want readline if we're doing anything interactive
    if options.interactive or options.pdb:
        exec "import readline" in scope

    # if pdb was requested, execfile the thing under pdb, otherwise,
    # just do the execfile normally
    if options.pdb:
        import pdb
        import traceback

        pdb = pdb.Pdb()
        try:
            pdb.run(filecode, scope)
        except SystemExit:
            print "The program exited via sys.exit(). Exit status: ",
            print sys.exc_info()[1]
        except:
            traceback.print_exc()
            print "Uncaught exception. Entering post mortem debugging"
            t = sys.exc_info()[2]
            while t.tb_next is not None:
                t = t.tb_next
                pdb.interaction(t.tb_frame,t)
    else:
        exec filecode in scope

    # once the script is done
    if options.interactive:
        interact = code.InteractiveConsole(scope)
        interact.interact("M5 Interactive Console")

if __name__ == '__main__':
    from pprint import pprint

    # load the options.py config file to allow people to set their own
    # default options
    options_file = config.get('options.py')
    if options_file:
        scope = { 'options' : options }
        execfile(options_file, scope)

    arguments = options.parse_args()

    print 'opts:'
    pprint(options, indent=4)
    print

    print 'args:'
    pprint(arguments, indent=4)
<commit_msg>traceflags: Make "All" a valid trace flag.<commit_after># Copyright (c) 2005 The Regents of The University of Michigan
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Nathan Binkert

import code
import datetime
import os
import socket
import sys

from util import attrdict
import config
from options import OptionParser

__all__ = [ 'options', 'arguments', 'main' ]

def print_list(items, indent=4):
    line = ' ' * indent
    for i,item in enumerate(items):
        if len(line) + len(item) > 76:
            print line
            line = ' ' * indent

        if i < len(items) - 1:
            line += '%s, ' % item
        else:
            line += item
            print line

usage="%prog [m5 options] script.py [script options]"
version="%prog 2.0"
brief_copyright='''
Copyright (c) 2001-2008
The Regents of The University of Michigan
All Rights Reserved
'''

options = OptionParser(usage=usage, version=version,
                       description=brief_copyright)
add_option = options.add_option
set_group = options.set_group
usage = options.usage

# Help options
add_option('-A', "--authors", action="store_true", default=False,
    help="Show author information")
add_option('-B', "--build-info", action="store_true", default=False,
    help="Show build information")
add_option('-C', "--copyright", action="store_true", default=False,
    help="Show full copyright information")
add_option('-R', "--readme", action="store_true", default=False,
    help="Show the readme")
add_option('-N', "--release-notes", action="store_true", default=False,
    help="Show the release notes")

# Options for configuring the base simulator
add_option('-d', "--outdir", metavar="DIR", default=".",
    help="Set the output directory to DIR [Default: %default]")
add_option('-r', "--redirect-stdout", action="store_true", default=False,
           help="Redirect stdout (& stderr, without -e) to file")
add_option('-e', "--redirect-stderr", action="store_true", default=False,
           help="Redirect stderr to file")
add_option("--stdout-file", metavar="FILE", default="simout",
           help="Filename for -r redirection [Default: %default]")
add_option("--stderr-file", metavar="FILE", default="simerr",
           help="Filename for -e redirection [Default: %default]")
add_option('-i', "--interactive", action="store_true", default=False,
    help="Invoke the interactive interpreter after running the script")
add_option("--pdb", action="store_true", default=False,
    help="Invoke the python debugger before running the script")
add_option('-p', "--path", metavar="PATH[:PATH]", action='append', split=':',
    help="Prepend PATH to the system path when invoking the script")
add_option('-q', "--quiet", action="count", default=0,
    help="Reduce verbosity")
add_option('-v', "--verbose", action="count", default=0,
    help="Increase verbosity")

# Statistics options
set_group("Statistics Options")
add_option("--stats-file", metavar="FILE", default="m5stats.txt",
    help="Sets the output file for statistics [Default: %default]")

# Debugging options
set_group("Debugging Options")
add_option("--debug-break", metavar="TIME[,TIME]", action='append', split=',',
    help="Cycle to create a breakpoint")
add_option("--remote-gdb-port", type='int', default=7000,
    help="Remote gdb base port")

# Tracing options
set_group("Trace Options")
add_option("--trace-help", action='store_true',
    help="Print help on trace flags")
add_option("--trace-flags", metavar="FLAG[,FLAG]", action='append', split=',',
    help="Sets the flags for tracing (-FLAG disables a flag)")
add_option("--trace-start", metavar="TIME", type='int',
    help="Start tracing at TIME (must be in ticks)")
add_option("--trace-file", metavar="FILE", default="cout",
    help="Sets the output file for tracing [Default: %default]")
add_option("--trace-ignore", metavar="EXPR", action='append', split=':',
    help="Ignore EXPR sim objects")

# Help options
set_group("Help Options")
add_option("--list-sim-objects", action='store_true', default=False,
    help="List all built-in SimObjects, their parameters and default values")

def main():
    import event
    import info
    import internal

    # load the options.py config file to allow people to set their own
    # default options
    options_file = config.get('options.py')
    if options_file:
        scope = { 'options' : options }
        execfile(options_file, scope)

    arguments = options.parse_args()

    if not os.path.isdir(options.outdir):
        os.makedirs(options.outdir)

    # These filenames are used only if the redirect_std* options are set
    stdout_file = os.path.join(options.outdir, options.stdout_file)
    stderr_file = os.path.join(options.outdir, options.stderr_file)

    # Print redirection notices here before doing any redirection
    if options.redirect_stdout and not options.redirect_stderr:
        print "Redirecting stdout and stderr to", stdout_file
    else:
        if options.redirect_stdout:
            print "Redirecting stdout to", stdout_file
        if options.redirect_stderr:
            print "Redirecting stderr to", stderr_file

    # Now redirect stdout/stderr as desired
    if options.redirect_stdout:
        redir_fd = os.open(stdout_file, os. O_WRONLY | os.O_CREAT | os.O_TRUNC)
        os.dup2(redir_fd, sys.stdout.fileno())
        if not options.redirect_stderr:
            os.dup2(redir_fd, sys.stderr.fileno())

    if options.redirect_stderr:
        redir_fd = os.open(stderr_file, os. O_WRONLY | os.O_CREAT | os.O_TRUNC)
        os.dup2(redir_fd, sys.stderr.fileno())

    done = False

    if options.build_info:
        import defines

        done = True
        print 'Build information:'
        print
        print 'compiled %s' % internal.core.cvar.compileDate;
        print "revision %s" % internal.core.cvar.hgRev
        print "commit date %s" % internal.core.cvar.hgDate
        print 'build options:'
        keys = defines.m5_build_env.keys()
        keys.sort()
        for key in keys:
            val = defines.m5_build_env[key]
            print '    %s = %s' % (key, val)
        print

    if options.copyright:
        done = True
        print info.LICENSE
        print

    if options.authors:
        done = True
        print 'Author information:'
        print
        print info.AUTHORS
        print

    if options.readme:
        done = True
        print 'Readme:'
        print
        print info.README
        print

    if options.release_notes:
        done = True
        print 'Release Notes:'
        print
        print info.RELEASE_NOTES
        print

    if options.trace_help:
        import traceflags

        done = True
        print "Base Flags:"
        traceflags.baseFlags.sort()
        print_list(traceflags.baseFlags, indent=4)
        print
        print "Compound Flags:"
        traceflags.compoundFlags.sort()
        for flag in traceflags.compoundFlags:
            if flag == 'All':
                continue
            print "    %s:" % flag
            print_list(traceflags.compoundFlagMap[flag], indent=8)
            print

    if options.list_sim_objects:
        import SimObject
        done = True
        print "SimObjects:"
        objects = SimObject.allClasses.keys()
        objects.sort()
        for name in objects:
            obj = SimObject.allClasses[name]
            print "    %s" % obj
            params = obj._params.keys()
            params.sort()
            for pname in params:
                param = obj._params[pname]
                default = getattr(param, 'default', '')
                print "        %s" % pname
                if default:
                    print "            default: %s" % default
                print "            desc: %s" % param.desc
                print
            print

    if done:
        sys.exit(0)

    # setting verbose and quiet at the same time doesn't make sense
    if options.verbose > 0 and options.quiet > 0:
        options.usage(2)

    verbose = options.verbose - options.quiet
    if options.verbose >= 0:
        print "M5 Simulator System"
        print brief_copyright
        print
        print "M5 compiled %s" % internal.core.cvar.compileDate;
        print "M5 revision %s" % internal.core.cvar.hgRev
        print "M5 commit date %s" % internal.core.cvar.hgDate

        print "M5 started %s" % datetime.datetime.now().strftime("%b %e %Y %X")
        print "M5 executing on %s" % socket.gethostname()

        print "command line:",
        for argv in sys.argv:
            print argv,
        print

    # check to make sure we can find the listed script
    if not arguments or not os.path.isfile(arguments[0]):
        if arguments and not os.path.isfile(arguments[0]):
            print "Script %s not found" % arguments[0]

        options.usage(2)

    # tell C++ about output directory
    internal.core.setOutputDir(options.outdir)

    # update the system path with elements from the -p option
    sys.path[0:0] = options.path

    import objects

    # set stats options
    internal.stats.initText(options.stats_file)

    # set debugging options
    internal.debug.setRemoteGDBPort(options.remote_gdb_port)
    for when in options.debug_break:
        internal.debug.schedBreakCycle(int(when))

    if options.trace_flags:
        import traceflags

        on_flags = []
        off_flags = []
        for flag in options.trace_flags:
            off = False
            if flag.startswith('-'):
                flag = flag[1:]
                off = True
            if flag not in traceflags.allFlags and flag != "All":
                print >>sys.stderr, "invalid trace flag '%s'" % flag
                sys.exit(1)

            if off:
                off_flags.append(flag)
            else:
                on_flags.append(flag)

        for flag in on_flags:
            internal.trace.set(flag)

        for flag in off_flags:
            internal.trace.clear(flag)

    if options.trace_start:
        def enable_trace():
            internal.trace.cvar.enabled = True

        e = event.create(enable_trace)
        event.mainq.schedule(e, options.trace_start)
    else:
        internal.trace.cvar.enabled = True

    internal.trace.output(options.trace_file)

    for ignore in options.trace_ignore:
        internal.trace.ignore(ignore)

    sys.argv = arguments
    sys.path = [ os.path.dirname(sys.argv[0]) ] + sys.path

    filename = sys.argv[0]
    filedata = file(filename, 'r').read()
    filecode = compile(filedata, filename, 'exec')
    scope = { '__file__' : filename,
              '__name__' : '__m5_main__' }

    # we want readline if we're doing anything interactive
    if options.interactive or options.pdb:
        exec "import readline" in scope

    # if pdb was requested, execfile the thing under pdb, otherwise,
    # just do the execfile normally
    if options.pdb:
        import pdb
        import traceback

        pdb = pdb.Pdb()
        try:
            pdb.run(filecode, scope)
        except SystemExit:
            print "The program exited via sys.exit(). Exit status: ",
            print sys.exc_info()[1]
        except:
            traceback.print_exc()
            print "Uncaught exception. Entering post mortem debugging"
            t = sys.exc_info()[2]
            while t.tb_next is not None:
                t = t.tb_next
                pdb.interaction(t.tb_frame,t)
    else:
        exec filecode in scope

    # once the script is done
    if options.interactive:
        interact = code.InteractiveConsole(scope)
        interact.interact("M5 Interactive Console")

if __name__ == '__main__':
    from pprint import pprint

    # load the options.py config file to allow people to set their own
    # default options
    options_file = config.get('options.py')
    if options_file:
        scope = { 'options' : options }
        execfile(options_file, scope)

    arguments = options.parse_args()

    print 'opts:'
    pprint(options, indent=4)
    print

    print 'args:'
    pprint(arguments, indent=4)
<|endoftext|>


<commit_before>
import unittest

import shorttext.spell.sakaguchi as sk

class TestSCRNN(unittest.TestCase):
    def setUp(self):
        pass

    def tearDown(self):
        pass

    def generalproc(self, operation, typo='langudge', recommendation='language'):
        corrector = sk.SCRNNSpellCorrector(operation)
        corrector.train('I am a nerd . Natural language processing is sosad .')
        self.assertEqual(corrector.correct(typo), recommendation)

    def test_NOISE_INSERT(self):
        self.generalproc('NOISE-INSERT')

    def test_NOISE_DELETE(self):
        self.generalproc('NOISE-DELETE')

    def test_NOISE_REPLACE(self):
        self.generalproc('NOISE-REPLACE', typo='procsesing', recommendation='processing')

    def test_JUMBLE_WHOLE(self):
        self.generalproc('JUMBLE-WHOLE')

    def test_JUMBLE_BEG(self):
        self.generalproc('JUMBLE-BEG')

    def test_JUMBLE_END(self):
        self.generalproc('JUMBLE-END')

    def test_JUMBLE_INT(self):
        self.generalproc('JUMBLE-INT')

if __name__ == '__main__':
    unittest.main()

<commit_msg>spell corrector I/O checked<commit_after>
import unittest

from shorttext.spell.sakaguchi import SCRNNSpellCorrector
from shorttext.smartload import smartload_compact_model

class TestSCRNN(unittest.TestCase):
    def setUp(self):
        pass

    def tearDown(self):
        pass

    def generalproc(self, operation, typo='langudge', recommendation='language'):
        corrector = SCRNNSpellCorrector(operation)
        corrector.train('I am a nerd . Natural language processing is sosad .')
        corrector.save_compact_model('./sosad_'+operation+'_sakuaguchi.bin')

        corrector2 = smartload_compact_model('./sosad_'+operation+'_sakaguchi.bin')
        self.assertEqual(corrector.correct(typo), corrector2.correct(typo))

        print('typo: '+typo+'  recommendation: '+corrector.correct(typo)+' ('+recommendation+')')

    def test_NOISE_INSERT(self):
        self.generalproc('NOISE-INSERT')

    def test_NOISE_DELETE(self):
        self.generalproc('NOISE-DELETE')

    def test_NOISE_REPLACE(self):
        self.generalproc('NOISE-REPLACE', typo='procsesing', recommendation='processing')

    def test_JUMBLE_WHOLE(self):
        self.generalproc('JUMBLE-WHOLE')

    def test_JUMBLE_BEG(self):
        self.generalproc('JUMBLE-BEG')

    def test_JUMBLE_END(self):
        self.generalproc('JUMBLE-END')

    def test_JUMBLE_INT(self):
        self.generalproc('JUMBLE-INT')

if __name__ == '__main__':
    unittest.main()

<|endoftext|>


<commit_before># Copyright (c) 2005 The Regents of The University of Michigan
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Nathan Binkert

import code, optparse, os, socket, sys
from datetime import datetime
from attrdict import attrdict

try:
    import info
except ImportError:
    info = None

__all__ = [ 'options', 'arguments', 'main' ]

usage="%prog [m5 options] script.py [script options]"
version="%prog 2.0"
brief_copyright='''
Copyright (c) 2001-2006
The Regents of The University of Michigan
All Rights Reserved
'''

# there's only one option parsing done, so make it global and add some
# helper functions to make it work well.
parser = optparse.OptionParser(usage=usage, version=version,
                               description=brief_copyright,
                               formatter=optparse.TitledHelpFormatter())
parser.disable_interspersed_args()

# current option group
group = None

def set_group(*args, **kwargs):
    '''set the current option group'''
    global group
    if not args and not kwargs:
        group = None
    else:
        group = parser.add_option_group(*args, **kwargs)

class splitter(object):
    def __init__(self, split):
        self.split = split
    def __call__(self, option, opt_str, value, parser):
        getattr(parser.values, option.dest).extend(value.split(self.split))

def add_option(*args, **kwargs):
    '''add an option to the current option group, or global none set'''

    # if action=split, but allows the option arguments
    # themselves to be lists separated by the split variable'''

    if kwargs.get('action', None) == 'append' and 'split' in kwargs:
        split = kwargs.pop('split')
        kwargs['default'] = []
        kwargs['type'] = 'string'
        kwargs['action'] = 'callback'
        kwargs['callback'] = splitter(split)

    if group:
        return group.add_option(*args, **kwargs)

    return parser.add_option(*args, **kwargs)

def bool_option(name, default, help):
    '''add a boolean option called --name and --no-name.
    Display help depending on which is the default'''

    tname = '--%s' % name
    fname = '--no-%s' % name
    dest = name.replace('-', '_')
    if default:
        thelp = optparse.SUPPRESS_HELP
        fhelp = help
    else:
        thelp = help
        fhelp = optparse.SUPPRESS_HELP

    add_option(tname, action="store_true", default=default, help=thelp)
    add_option(fname, action="store_false", dest=dest, help=fhelp)

# Help options
add_option('-A', "--authors", action="store_true", default=False,
    help="Show author information")
add_option('-C', "--copyright", action="store_true", default=False,
    help="Show full copyright information")
add_option('-R', "--readme", action="store_true", default=False,
    help="Show the readme")
add_option('-N', "--release-notes", action="store_true", default=False,
    help="Show the release notes")

# Options for configuring the base simulator
add_option('-d', "--outdir", metavar="DIR", default=".",
    help="Set the output directory to DIR [Default: %default]")
add_option('-i', "--interactive", action="store_true", default=False,
    help="Invoke the interactive interpreter after running the script")
add_option('-p', "--path", metavar="PATH[:PATH]", action='append', split=':',
    help="Prepend PATH to the system path when invoking the script")
add_option('-q', "--quiet", action="count", default=0,
    help="Reduce verbosity")
add_option('-v', "--verbose", action="count", default=0,
    help="Increase verbosity")

# Statistics options
set_group("Statistics Options")
add_option("--stats-file", metavar="FILE", default="m5stats.txt",
    help="Sets the output file for statistics [Default: %default]")

# Debugging options
set_group("Debugging Options")
add_option("--debug-break", metavar="TIME[,TIME]", action='append', split=',',
    help="Cycle to create a breakpoint")

# Tracing options
set_group("Trace Options")
add_option("--trace-flags", metavar="FLAG[,FLAG]", action='append', split=',',
    help="Sets the flags for tracing")
add_option("--trace-start", metavar="TIME", default='0s',
    help="Start tracing at TIME (must have units)")
add_option("--trace-file", metavar="FILE", default="cout",
    help="Sets the output file for tracing [Default: %default]")
add_option("--trace-circlebuf", metavar="SIZE", type="int", default=0,
    help="If SIZE is non-zero, turn on the circular buffer with SIZE lines")
add_option("--no-trace-circlebuf", action="store_const", const=0,
    dest='trace_circlebuf', help=optparse.SUPPRESS_HELP)
bool_option("trace-dumponexit", default=False,
    help="Dump trace buffer on exit")
add_option("--trace-ignore", metavar="EXPR", action='append', split=':',
    help="Ignore EXPR sim objects")

# Execution Trace options
set_group("Execution Trace Options")
bool_option("speculative", default=True,
    help="Don't capture speculative instructions")
bool_option("print-cycle", default=True,
    help="Don't print cycle numbers in trace output")
bool_option("print-symbol", default=True,
    help="Disable PC symbols in trace output")
bool_option("print-opclass", default=True,
    help="Don't print op class type in trace output")
bool_option("print-thread", default=True,
    help="Don't print thread number in trace output")
bool_option("print-effaddr", default=True,
    help="Don't print effective address in trace output")
bool_option("print-data", default=True,
    help="Don't print result data in trace output")
bool_option("print-iregs", default=False,
    help="Print fetch sequence numbers in trace output")
bool_option("print-fetch-seq", default=False,
    help="Print fetch sequence numbers in trace output")
bool_option("print-cpseq", default=False,
    help="Print correct path sequence numbers in trace output")

options = attrdict()
arguments = []

def usage(exitcode=None):
    print parser.help
    if exitcode is not None:
        sys.exit(exitcode)

def parse_args():
    _opts,args = parser.parse_args()
    opts = attrdict(_opts.__dict__)

    # setting verbose and quiet at the same time doesn't make sense
    if opts.verbose > 0 and opts.quiet > 0:
        usage(2)

    # store the verbosity in a single variable.  0 is default,
    # negative numbers represent quiet and positive values indicate verbose
    opts.verbose -= opts.quiet

    del opts.quiet

    options.update(opts)
    arguments.extend(args)
    return opts,args

def main():
    import cc_main

    parse_args()

    done = False
    if options.copyright:
        done = True
        print info.LICENSE
        print

    if options.authors:
        done = True
        print 'Author information:'
        print
        print info.AUTHORS
        print

    if options.readme:
        done = True
        print 'Readme:'
        print
        print info.README
        print

    if options.release_notes:
        done = True
        print 'Release Notes:'
        print
        print info.RELEASE_NOTES
        print

    if done:
        sys.exit(0)

    if options.verbose >= 0:
        print "M5 Simulator System"
        print brief_copyright
        print
        print "M5 compiled %s" % cc_main.cvar.compileDate;
        print "M5 started %s" % datetime.now().ctime()
        print "M5 executing on %s" % socket.gethostname()

    # check to make sure we can find the listed script
    if not arguments or not os.path.isfile(arguments[0]):
        usage(2)

    # tell C++ about output directory
    cc_main.setOutputDir(options.outdir)

    # update the system path with elements from the -p option
    sys.path[0:0] = options.path

    import objects

    # set stats options
    objects.Statistics.text_file = options.stats_file

    # set debugging options
    objects.Debug.break_cycles = options.debug_break

    # set tracing options
    objects.Trace.flags = options.trace_flags
    objects.Trace.start = options.trace_start
    objects.Trace.file = options.trace_file
    objects.Trace.bufsize = options.trace_circlebuf
    objects.Trace.dump_on_exit = options.trace_dumponexit
    objects.Trace.ignore = options.trace_ignore

    # set execution trace options
    objects.ExecutionTrace.speculative = options.speculative
    objects.ExecutionTrace.print_cycle = options.print_cycle
    objects.ExecutionTrace.pc_symbol = options.print_symbol
    objects.ExecutionTrace.print_opclass = options.print_opclass
    objects.ExecutionTrace.print_thread = options.print_thread
    objects.ExecutionTrace.print_effaddr = options.print_effaddr
    objects.ExecutionTrace.print_data = options.print_data
    objects.ExecutionTrace.print_iregs = options.print_iregs
    objects.ExecutionTrace.print_fetchseq = options.print_fetch_seq
    objects.ExecutionTrace.print_cpseq = options.print_cpseq

    scope = { '__file__' : sys.argv[0] }
    sys.argv = arguments
    sys.path = [ os.path.dirname(sys.argv[0]) ] + sys.path
    exec("import readline", scope)
    execfile(sys.argv[0], scope)

    # once the script is done
    if options.interactive:
        interact = code.InteractiveConsole(scope)
        interact.interact("M5 Interactive Console")

if __name__ == '__main__':
    from pprint import pprint

    parse_args()

    print 'opts:'
    pprint(options, indent=4)
    print

    print 'args:'
    pprint(arguments, indent=4)
<commit_msg>Fix __file__ for scripts<commit_after># Copyright (c) 2005 The Regents of The University of Michigan
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Nathan Binkert

import code, optparse, os, socket, sys
from datetime import datetime
from attrdict import attrdict

try:
    import info
except ImportError:
    info = None

__all__ = [ 'options', 'arguments', 'main' ]

usage="%prog [m5 options] script.py [script options]"
version="%prog 2.0"
brief_copyright='''
Copyright (c) 2001-2006
The Regents of The University of Michigan
All Rights Reserved
'''

# there's only one option parsing done, so make it global and add some
# helper functions to make it work well.
parser = optparse.OptionParser(usage=usage, version=version,
                               description=brief_copyright,
                               formatter=optparse.TitledHelpFormatter())
parser.disable_interspersed_args()

# current option group
group = None

def set_group(*args, **kwargs):
    '''set the current option group'''
    global group
    if not args and not kwargs:
        group = None
    else:
        group = parser.add_option_group(*args, **kwargs)

class splitter(object):
    def __init__(self, split):
        self.split = split
    def __call__(self, option, opt_str, value, parser):
        getattr(parser.values, option.dest).extend(value.split(self.split))

def add_option(*args, **kwargs):
    '''add an option to the current option group, or global none set'''

    # if action=split, but allows the option arguments
    # themselves to be lists separated by the split variable'''

    if kwargs.get('action', None) == 'append' and 'split' in kwargs:
        split = kwargs.pop('split')
        kwargs['default'] = []
        kwargs['type'] = 'string'
        kwargs['action'] = 'callback'
        kwargs['callback'] = splitter(split)

    if group:
        return group.add_option(*args, **kwargs)

    return parser.add_option(*args, **kwargs)

def bool_option(name, default, help):
    '''add a boolean option called --name and --no-name.
    Display help depending on which is the default'''

    tname = '--%s' % name
    fname = '--no-%s' % name
    dest = name.replace('-', '_')
    if default:
        thelp = optparse.SUPPRESS_HELP
        fhelp = help
    else:
        thelp = help
        fhelp = optparse.SUPPRESS_HELP

    add_option(tname, action="store_true", default=default, help=thelp)
    add_option(fname, action="store_false", dest=dest, help=fhelp)

# Help options
add_option('-A', "--authors", action="store_true", default=False,
    help="Show author information")
add_option('-C', "--copyright", action="store_true", default=False,
    help="Show full copyright information")
add_option('-R', "--readme", action="store_true", default=False,
    help="Show the readme")
add_option('-N', "--release-notes", action="store_true", default=False,
    help="Show the release notes")

# Options for configuring the base simulator
add_option('-d', "--outdir", metavar="DIR", default=".",
    help="Set the output directory to DIR [Default: %default]")
add_option('-i', "--interactive", action="store_true", default=False,
    help="Invoke the interactive interpreter after running the script")
add_option('-p', "--path", metavar="PATH[:PATH]", action='append', split=':',
    help="Prepend PATH to the system path when invoking the script")
add_option('-q', "--quiet", action="count", default=0,
    help="Reduce verbosity")
add_option('-v', "--verbose", action="count", default=0,
    help="Increase verbosity")

# Statistics options
set_group("Statistics Options")
add_option("--stats-file", metavar="FILE", default="m5stats.txt",
    help="Sets the output file for statistics [Default: %default]")

# Debugging options
set_group("Debugging Options")
add_option("--debug-break", metavar="TIME[,TIME]", action='append', split=',',
    help="Cycle to create a breakpoint")

# Tracing options
set_group("Trace Options")
add_option("--trace-flags", metavar="FLAG[,FLAG]", action='append', split=',',
    help="Sets the flags for tracing")
add_option("--trace-start", metavar="TIME", default='0s',
    help="Start tracing at TIME (must have units)")
add_option("--trace-file", metavar="FILE", default="cout",
    help="Sets the output file for tracing [Default: %default]")
add_option("--trace-circlebuf", metavar="SIZE", type="int", default=0,
    help="If SIZE is non-zero, turn on the circular buffer with SIZE lines")
add_option("--no-trace-circlebuf", action="store_const", const=0,
    dest='trace_circlebuf', help=optparse.SUPPRESS_HELP)
bool_option("trace-dumponexit", default=False,
    help="Dump trace buffer on exit")
add_option("--trace-ignore", metavar="EXPR", action='append', split=':',
    help="Ignore EXPR sim objects")

# Execution Trace options
set_group("Execution Trace Options")
bool_option("speculative", default=True,
    help="Don't capture speculative instructions")
bool_option("print-cycle", default=True,
    help="Don't print cycle numbers in trace output")
bool_option("print-symbol", default=True,
    help="Disable PC symbols in trace output")
bool_option("print-opclass", default=True,
    help="Don't print op class type in trace output")
bool_option("print-thread", default=True,
    help="Don't print thread number in trace output")
bool_option("print-effaddr", default=True,
    help="Don't print effective address in trace output")
bool_option("print-data", default=True,
    help="Don't print result data in trace output")
bool_option("print-iregs", default=False,
    help="Print fetch sequence numbers in trace output")
bool_option("print-fetch-seq", default=False,
    help="Print fetch sequence numbers in trace output")
bool_option("print-cpseq", default=False,
    help="Print correct path sequence numbers in trace output")

options = attrdict()
arguments = []

def usage(exitcode=None):
    print parser.help
    if exitcode is not None:
        sys.exit(exitcode)

def parse_args():
    _opts,args = parser.parse_args()
    opts = attrdict(_opts.__dict__)

    # setting verbose and quiet at the same time doesn't make sense
    if opts.verbose > 0 and opts.quiet > 0:
        usage(2)

    # store the verbosity in a single variable.  0 is default,
    # negative numbers represent quiet and positive values indicate verbose
    opts.verbose -= opts.quiet

    del opts.quiet

    options.update(opts)
    arguments.extend(args)
    return opts,args

def main():
    import cc_main

    parse_args()

    done = False
    if options.copyright:
        done = True
        print info.LICENSE
        print

    if options.authors:
        done = True
        print 'Author information:'
        print
        print info.AUTHORS
        print

    if options.readme:
        done = True
        print 'Readme:'
        print
        print info.README
        print

    if options.release_notes:
        done = True
        print 'Release Notes:'
        print
        print info.RELEASE_NOTES
        print

    if done:
        sys.exit(0)

    if options.verbose >= 0:
        print "M5 Simulator System"
        print brief_copyright
        print
        print "M5 compiled %s" % cc_main.cvar.compileDate;
        print "M5 started %s" % datetime.now().ctime()
        print "M5 executing on %s" % socket.gethostname()

    # check to make sure we can find the listed script
    if not arguments or not os.path.isfile(arguments[0]):
        usage(2)

    # tell C++ about output directory
    cc_main.setOutputDir(options.outdir)

    # update the system path with elements from the -p option
    sys.path[0:0] = options.path

    import objects

    # set stats options
    objects.Statistics.text_file = options.stats_file

    # set debugging options
    objects.Debug.break_cycles = options.debug_break

    # set tracing options
    objects.Trace.flags = options.trace_flags
    objects.Trace.start = options.trace_start
    objects.Trace.file = options.trace_file
    objects.Trace.bufsize = options.trace_circlebuf
    objects.Trace.dump_on_exit = options.trace_dumponexit
    objects.Trace.ignore = options.trace_ignore

    # set execution trace options
    objects.ExecutionTrace.speculative = options.speculative
    objects.ExecutionTrace.print_cycle = options.print_cycle
    objects.ExecutionTrace.pc_symbol = options.print_symbol
    objects.ExecutionTrace.print_opclass = options.print_opclass
    objects.ExecutionTrace.print_thread = options.print_thread
    objects.ExecutionTrace.print_effaddr = options.print_effaddr
    objects.ExecutionTrace.print_data = options.print_data
    objects.ExecutionTrace.print_iregs = options.print_iregs
    objects.ExecutionTrace.print_fetchseq = options.print_fetch_seq
    objects.ExecutionTrace.print_cpseq = options.print_cpseq

    sys.argv = arguments
    sys.path = [ os.path.dirname(sys.argv[0]) ] + sys.path

    scope = { '__file__' : sys.argv[0] }
    exec("import readline", scope)
    execfile(sys.argv[0], scope)

    # once the script is done
    if options.interactive:
        interact = code.InteractiveConsole(scope)
        interact.interact("M5 Interactive Console")

if __name__ == '__main__':
    from pprint import pprint

    parse_args()

    print 'opts:'
    pprint(options, indent=4)
    print

    print 'args:'
    pprint(arguments, indent=4)
<|endoftext|>


<commit_before># encoding: utf-8

"""
.. codeauthor:: Tsuyoshi Hombashi <tsuyoshi.hombashi@gmail.com>
"""

from __future__ import print_function, unicode_literals

import errno
import os
import platform
import re
import subprocess
from subprocess import PIPE

import pytest
import six
from subprocrunner import SubprocessRunner
from typepy import is_not_null_string, is_null_string

os_type = platform.system()
if os_type in ["Linux", "Darwin"]:
    list_command = "ls"
    list_command_errno = errno.ENOENT
elif os_type == "Windows":
    list_command = "dir"
    list_command_errno = 1
else:
    raise NotImplementedError(os_type)

class Test_SubprocessRunner_run(object):
    @pytest.mark.parametrize(
        ["command", "dry_run", "expected"],
        [
            [list_command, False, 0],
            [list_command, True, 0],
            [list_command + " __not_exist_dir__", False, list_command_errno],
            [list_command + " __not_exist_dir__", True, 0],
        ],
    )
    def test_normal(self, command, dry_run, expected):
        assert SubprocessRunner(command, dry_run=dry_run).run() == expected

    @pytest.mark.skipif("platform.system() == 'Windows'")
    @pytest.mark.parametrize(
        ["command", "expected"], [[list_command + " -l", 0], [[list_command, "-l"], 0]]
    )
    def test_command(self, command, expected):
        assert SubprocessRunner(command).run() == expected

    @pytest.mark.parametrize(
        ["command", "expected"], [["echo test", "test"], [["echo", "test"], "test"]]
    )
    def test_stdout(self, command, expected):
        runner = SubprocessRunner(command)
        runner.run()

        assert runner.command == command
        assert isinstance(runner.command_str, six.string_types)
        assert runner.stdout.strip() == expected
        assert is_null_string(runner.stderr)

    @pytest.mark.skipif("platform.system() == 'Windows'")
    @pytest.mark.parametrize(
        ["command", "ignore_stderr_regexp", "out_regexp", "expected"],
        [
            [list_command + " __not_exist_dir__", None, re.compile("WARNING"), True],
            [
                list_command + " __not_exist_dir__",
                re.compile(re.escape("__not_exist_dir__")),
                re.compile("WARNING"),
                False,
            ],
        ],
    )
    def test_stderr(self, capsys, command, ignore_stderr_regexp, out_regexp, expected):
        import logbook
        import subprocrunner

        logbook.StderrHandler(level=logbook.DEBUG).push_application()
        subprocrunner.set_log_level(logbook.INFO)

        runner = SubprocessRunner(command, ignore_stderr_regexp=ignore_stderr_regexp)
        runner.run()

        assert is_null_string(runner.stdout.strip())
        assert is_not_null_string(runner.stderr.strip())

        out, err = capsys.readouterr()
        print("[sys stdout]\n{}\n".format(out))
        print("[sys stderr]\n{}\n".format(err))
        print("[proc stdout]\n{}\n".format(runner.stdout))
        print("[proc stderr]\n{}\n".format(runner.stderr))

        actual = out_regexp.search(err) is not None
        assert actual == expected

    @pytest.mark.skipif("platform.system() == 'Windows'")
    @pytest.mark.parametrize(
        ["command", "ignore_stderr_regexp", "expected"],
        [
            [[list_command, "__not_exist_dir__"], None, CalledProcessError],
            [[list_command, "__not_exist_dir__"], re.compile(re.escape("__not_exist_dir__")), None],
        ],
    )
    def test_stderr_check(self, command, ignore_stderr_regexp, expected):
        runner = SubprocessRunner(command, ignore_stderr_regexp=ignore_stderr_regexp)

        if ignore_stderr_regexp:
            runner.run(check=True)
        else:
            with pytest.raises(expected):
                runner.run(check=True)

    def test_unicode(self, monkeypatch):
        def monkey_communicate(input=None):
            return ("", "'dummy' は、内部コマンドまたは外部コマンド、" "操作可能なプログラムまたはバッチ ファイルとして認識されていません")

        monkeypatch.setattr(subprocess.Popen, "communicate", monkey_communicate)

        runner = SubprocessRunner(list_command)
        runner.run()

class Test_SubprocessRunner_popen(object):
    @pytest.mark.parametrize(
        ["command", "environ", "expected"],
        [["hostname", None, 0], ["hostname", dict(os.environ), 0]],
    )
    def test_normal(self, command, environ, expected):
        proc = SubprocessRunner(command).popen(env=environ)
        ret_stdout, ret_stderr = proc.communicate()
        assert is_not_null_string(ret_stdout)
        assert is_null_string(ret_stderr)
        assert proc.returncode == expected

    @pytest.mark.skipif("platform.system() == 'Windows'")
    @pytest.mark.parametrize(["command", "pipe_input", "expected"], [["grep a", six.b("aaa"), 0]])
    def test_normal_stdin(self, command, pipe_input, expected):
        proc = SubprocessRunner(command).popen(PIPE)
        ret_stdout, ret_stderr = proc.communicate(input=pipe_input)

        assert is_not_null_string(ret_stdout)
        assert is_null_string(ret_stderr)
        assert proc.returncode == expected

class Test_SubprocessRunner_command_history(object):
    @pytest.mark.parametrize(
        ["command", "dry_run", "expected"], [[list_command, False, 0], [list_command, True, 0]]
    )
    def test_normal(self, command, dry_run, expected):
        SubprocessRunner.is_save_history = False
        SubprocessRunner.clear_history()

        loop_count = 3
        for _i in range(loop_count):
            SubprocessRunner(command, dry_run=dry_run).run()
        assert len(SubprocessRunner.get_history()) == 0

        SubprocessRunner.is_save_history = True
        for _i in range(loop_count):
            SubprocessRunner(command, dry_run=dry_run).run()
        assert len(SubprocessRunner.get_history()) == loop_count
<commit_msg>Add a missing import<commit_after># encoding: utf-8

"""
.. codeauthor:: Tsuyoshi Hombashi <tsuyoshi.hombashi@gmail.com>
"""

from __future__ import print_function, unicode_literals

import errno
import os
import platform
import re
import subprocess
from subprocess import PIPE, CalledProcessError

import pytest
import six
from subprocrunner import SubprocessRunner
from typepy import is_not_null_string, is_null_string

os_type = platform.system()
if os_type in ["Linux", "Darwin"]:
    list_command = "ls"
    list_command_errno = errno.ENOENT
elif os_type == "Windows":
    list_command = "dir"
    list_command_errno = 1
else:
    raise NotImplementedError(os_type)

class Test_SubprocessRunner_run(object):
    @pytest.mark.parametrize(
        ["command", "dry_run", "expected"],
        [
            [list_command, False, 0],
            [list_command, True, 0],
            [list_command + " __not_exist_dir__", False, list_command_errno],
            [list_command + " __not_exist_dir__", True, 0],
        ],
    )
    def test_normal(self, command, dry_run, expected):
        assert SubprocessRunner(command, dry_run=dry_run).run() == expected

    @pytest.mark.skipif("platform.system() == 'Windows'")
    @pytest.mark.parametrize(
        ["command", "expected"], [[list_command + " -l", 0], [[list_command, "-l"], 0]]
    )
    def test_command(self, command, expected):
        assert SubprocessRunner(command).run() == expected

    @pytest.mark.parametrize(
        ["command", "expected"], [["echo test", "test"], [["echo", "test"], "test"]]
    )
    def test_stdout(self, command, expected):
        runner = SubprocessRunner(command)
        runner.run()

        assert runner.command == command
        assert isinstance(runner.command_str, six.string_types)
        assert runner.stdout.strip() == expected
        assert is_null_string(runner.stderr)

    @pytest.mark.skipif("platform.system() == 'Windows'")
    @pytest.mark.parametrize(
        ["command", "ignore_stderr_regexp", "out_regexp", "expected"],
        [
            [list_command + " __not_exist_dir__", None, re.compile("WARNING"), True],
            [
                list_command + " __not_exist_dir__",
                re.compile(re.escape("__not_exist_dir__")),
                re.compile("WARNING"),
                False,
            ],
        ],
    )
    def test_stderr(self, capsys, command, ignore_stderr_regexp, out_regexp, expected):
        import logbook
        import subprocrunner

        logbook.StderrHandler(level=logbook.DEBUG).push_application()
        subprocrunner.set_log_level(logbook.INFO)

        runner = SubprocessRunner(command, ignore_stderr_regexp=ignore_stderr_regexp)
        runner.run()

        assert is_null_string(runner.stdout.strip())
        assert is_not_null_string(runner.stderr.strip())

        out, err = capsys.readouterr()
        print("[sys stdout]\n{}\n".format(out))
        print("[sys stderr]\n{}\n".format(err))
        print("[proc stdout]\n{}\n".format(runner.stdout))
        print("[proc stderr]\n{}\n".format(runner.stderr))

        actual = out_regexp.search(err) is not None
        assert actual == expected

    @pytest.mark.skipif("platform.system() == 'Windows'")
    @pytest.mark.parametrize(
        ["command", "ignore_stderr_regexp", "expected"],
        [
            [[list_command, "__not_exist_dir__"], None, CalledProcessError],
            [[list_command, "__not_exist_dir__"], re.compile(re.escape("__not_exist_dir__")), None],
        ],
    )
    def test_stderr_check(self, command, ignore_stderr_regexp, expected):
        runner = SubprocessRunner(command, ignore_stderr_regexp=ignore_stderr_regexp)

        if ignore_stderr_regexp:
            runner.run(check=True)
        else:
            with pytest.raises(expected):
                runner.run(check=True)

    def test_unicode(self, monkeypatch):
        def monkey_communicate(input=None):
            return ("", "'dummy' は、内部コマンドまたは外部コマンド、" "操作可能なプログラムまたはバッチ ファイルとして認識されていません")

        monkeypatch.setattr(subprocess.Popen, "communicate", monkey_communicate)

        runner = SubprocessRunner(list_command)
        runner.run()

class Test_SubprocessRunner_popen(object):
    @pytest.mark.parametrize(
        ["command", "environ", "expected"],
        [["hostname", None, 0], ["hostname", dict(os.environ), 0]],
    )
    def test_normal(self, command, environ, expected):
        proc = SubprocessRunner(command).popen(env=environ)
        ret_stdout, ret_stderr = proc.communicate()
        assert is_not_null_string(ret_stdout)
        assert is_null_string(ret_stderr)
        assert proc.returncode == expected

    @pytest.mark.skipif("platform.system() == 'Windows'")
    @pytest.mark.parametrize(["command", "pipe_input", "expected"], [["grep a", six.b("aaa"), 0]])
    def test_normal_stdin(self, command, pipe_input, expected):
        proc = SubprocessRunner(command).popen(PIPE)
        ret_stdout, ret_stderr = proc.communicate(input=pipe_input)

        assert is_not_null_string(ret_stdout)
        assert is_null_string(ret_stderr)
        assert proc.returncode == expected

class Test_SubprocessRunner_command_history(object):
    @pytest.mark.parametrize(
        ["command", "dry_run", "expected"], [[list_command, False, 0], [list_command, True, 0]]
    )
    def test_normal(self, command, dry_run, expected):
        SubprocessRunner.is_save_history = False
        SubprocessRunner.clear_history()

        loop_count = 3
        for _i in range(loop_count):
            SubprocessRunner(command, dry_run=dry_run).run()
        assert len(SubprocessRunner.get_history()) == 0

        SubprocessRunner.is_save_history = True
        for _i in range(loop_count):
            SubprocessRunner(command, dry_run=dry_run).run()
        assert len(SubprocessRunner.get_history()) == loop_count
<|endoftext|>


<commit_before>#!/usr/bin/env python3
import os
import errno

from premailer import Premailer
from cssutils import profile
from cssutils.profiles import Profiles, properties, macros

ZULIP_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../')

def configure_cssutils() -> None:
    # These properties are not supported by cssutils by default and will
    # result in warnings when premailer package is run.
    properties[Profiles.CSS_LEVEL_2]['-ms-interpolation-mode'] = r'none|bicubic|nearest-neighbor'
    properties[Profiles.CSS_LEVEL_2]['-ms-text-size-adjust'] = r'none|auto|{percentage}'
    properties[Profiles.CSS_LEVEL_2]['mso-table-lspace'] = r'0|{num}(pt)'
    properties[Profiles.CSS_LEVEL_2]['mso-table-rspace'] = r'0|{num}(pt)'
    properties[Profiles.CSS_LEVEL_2]['-webkit-text-size-adjust'] = r'none|auto|{percentage}'
    properties[Profiles.CSS_LEVEL_2]['mso-hide'] = 'all'
    properties[Profiles.CSS_LEVEL_2]['pointer-events'] = (r'auto|none|visiblePainted|'
                                                          r'visibleFill|visibleStroke|'
                                                          r'visible|painted|fill|stroke|all|inherit')

    profile.addProfiles([(Profiles.CSS_LEVEL_2, properties[Profiles.CSS_LEVEL_2],
                         macros[Profiles.CSS_LEVEL_2])])

def escape_jinja2_characters(text: str) -> str:
    escaped_jinja2_characters = [('%7B%7B%20', '{{ '), ('%20%7D%7D', ' }}'), ('>', '>')]
    for escaped, original in escaped_jinja2_characters:
        text = text.replace(escaped, original)
    return text

if __name__ == "__main__":
    templates_to_inline = set()
    for f in os.listdir(os.path.join(ZULIP_PATH, 'templates', 'zerver', 'emails')):
        if f.endswith('.source.html'):
            templates_to_inline.add(f.split('.source.html')[0])

    configure_cssutils()

    os.chdir(os.path.join(ZULIP_PATH, 'templates', 'zerver', 'emails'))

    try:
        os.makedirs("compiled")
    except OSError as e:
        if e.errno != errno.EEXIST:
            raise

    for template in templates_to_inline:
        template_html_source = template + ".source.html"
        compiled_template_path = os.path.join(os.getcwd(), "compiled", template + ".html")

        with open(template_html_source) as template_source_file:
            template_str = template_source_file.read()
        output = Premailer(template_str,
                           external_styles=["email.css"]).transform()

        output = escape_jinja2_characters(output)

        # Premailer.transform will try to complete the DOM tree,
        # adding html, head, and body tags if they aren't there.
        # While this is correct for the email_base_default template,
        # it is wrong for the other templates that extend this
        # template, since we'll end up with 2 copipes of those tags.
        # Thus, we strip this stuff out if the template extends
        # another template.
        end_block = '\n'
        start = output.find('{% extends')
        end = output.rfind(end_block)
        if start != -1 and end != -1:
            output = output[start:end]
        else:
            # Only the base-default template should not be using this system.
            assert template == 'email_base_default'

        if ('zerver/emails/compiled/email_base_default.html' in output or
                'zerver/emails/email_base_messages.html' in output):
            assert output.count('') == 0
            assert output.count('') == 0
            assert output.count('') == 0
            assert output.count('') == 0

        with open(compiled_template_path, 'w') as final_template_file:
            final_template_file.write(output)
<commit_msg>inline_email_css: Extract strip_unnecesary_tags.<commit_after>#!/usr/bin/env python3
import os
import errno

from premailer import Premailer
from cssutils import profile
from cssutils.profiles import Profiles, properties, macros

ZULIP_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../')

def configure_cssutils() -> None:
    # These properties are not supported by cssutils by default and will
    # result in warnings when premailer package is run.
    properties[Profiles.CSS_LEVEL_2]['-ms-interpolation-mode'] = r'none|bicubic|nearest-neighbor'
    properties[Profiles.CSS_LEVEL_2]['-ms-text-size-adjust'] = r'none|auto|{percentage}'
    properties[Profiles.CSS_LEVEL_2]['mso-table-lspace'] = r'0|{num}(pt)'
    properties[Profiles.CSS_LEVEL_2]['mso-table-rspace'] = r'0|{num}(pt)'
    properties[Profiles.CSS_LEVEL_2]['-webkit-text-size-adjust'] = r'none|auto|{percentage}'
    properties[Profiles.CSS_LEVEL_2]['mso-hide'] = 'all'
    properties[Profiles.CSS_LEVEL_2]['pointer-events'] = (r'auto|none|visiblePainted|'
                                                          r'visibleFill|visibleStroke|'
                                                          r'visible|painted|fill|stroke|all|inherit')

    profile.addProfiles([(Profiles.CSS_LEVEL_2, properties[Profiles.CSS_LEVEL_2],
                         macros[Profiles.CSS_LEVEL_2])])

def escape_jinja2_characters(text: str) -> str:
    escaped_jinja2_characters = [('%7B%7B%20', '{{ '), ('%20%7D%7D', ' }}'), ('>', '>')]
    for escaped, original in escaped_jinja2_characters:
        text = text.replace(escaped, original)
    return text

def strip_unnecesary_tags(text: str) -> str:
    end_block = '\n'
    start_block = '{% extends'
    start = text.find(start_block)
    end = text.rfind(end_block)
    if start != -1 and end != -1:
        text = text[start:end]
        return text
    else:
        raise ValueError("Template does not have %s or %s" % (start_block, end_block))

if __name__ == "__main__":
    templates_to_inline = set()
    for f in os.listdir(os.path.join(ZULIP_PATH, 'templates', 'zerver', 'emails')):
        if f.endswith('.source.html'):
            templates_to_inline.add(f.split('.source.html')[0])

    configure_cssutils()

    os.chdir(os.path.join(ZULIP_PATH, 'templates', 'zerver', 'emails'))

    try:
        os.makedirs("compiled")
    except OSError as e:
        if e.errno != errno.EEXIST:
            raise

    for template in templates_to_inline:
        template_html_source = template + ".source.html"
        compiled_template_path = os.path.join(os.getcwd(), "compiled", template + ".html")

        with open(template_html_source) as template_source_file:
            template_str = template_source_file.read()
        output = Premailer(template_str,
                           external_styles=["email.css"]).transform()

        output = escape_jinja2_characters(output)

        # Premailer.transform will try to complete the DOM tree,
        # adding html, head, and body tags if they aren't there.
        # While this is correct for the email_base_default template,
        # it is wrong for the other templates that extend this
        # template, since we'll end up with 2 copipes of those tags.
        # Thus, we strip this stuff out if the template extends
        # another template.
        if template != 'email_base_default':
            output = strip_unnecesary_tags(output)

        if ('zerver/emails/compiled/email_base_default.html' in output or
                'zerver/emails/email_base_messages.html' in output):
            assert output.count('') == 0
            assert output.count('') == 0
            assert output.count('') == 0
            assert output.count('') == 0

        with open(compiled_template_path, 'w') as final_template_file:
            final_template_file.write(output)
<|endoftext|>


<commit_before># Copyright 2012 OpenStack Foundation
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from tempest.api.compute import base
from tempest import exceptions
from tempest import test

class AbsoluteLimitsNegativeTestJSON(base.BaseV2ComputeTest):

    @classmethod
    def resource_setup(cls):
        super(AbsoluteLimitsNegativeTestJSON, cls).resource_setup()
        cls.client = cls.limits_client
        cls.server_client = cls.servers_client

    @test.attr(type=['negative', 'gate'])
    def test_max_image_meta_exceed_limit(self):
        # We should not create vm with image meta over maxImageMeta limit
        # Get max limit value
        max_meta = self.client.get_specific_absolute_limit('maxImageMeta')

        # Create server should fail, since we are passing > metadata Limit!
        max_meta_data = int(max_meta) + 1

        meta_data = {}
        for xx in range(max_meta_data):
            meta_data[str(xx)] = str(xx)

        # A 403 Forbidden or 413 Overlimit (old behaviour) exception
        # will be raised when out of quota
        self.assertRaises((exceptions.Unauthorized, exceptions.OverLimit),
                          self.server_client.create_server,
                          name='test', meta=meta_data,
                          flavor_ref=self.flavor_ref,
                          image_ref=self.image_ref)
<commit_msg>Do not test the maxImageMeta limit when there is no limit<commit_after># Copyright 2012 OpenStack Foundation
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from tempest.api.compute import base
from tempest import exceptions
from tempest import test

class AbsoluteLimitsNegativeTestJSON(base.BaseV2ComputeTest):

    @classmethod
    def resource_setup(cls):
        super(AbsoluteLimitsNegativeTestJSON, cls).resource_setup()
        cls.client = cls.limits_client
        cls.server_client = cls.servers_client

    @test.attr(type=['negative', 'gate'])
    def test_max_image_meta_exceed_limit(self):
        # We should not create vm with image meta over maxImageMeta limit
        # Get max limit value
        max_meta = self.client.get_specific_absolute_limit('maxImageMeta')

        # No point in running this test if there is no limit.
        if int(max_meta) == -1:
            raise self.skipException('no limit for maxImageMeta')

        # Create server should fail, since we are passing > metadata Limit!
        max_meta_data = int(max_meta) + 1

        meta_data = {}
        for xx in range(max_meta_data):
            meta_data[str(xx)] = str(xx)

        # A 403 Forbidden or 413 Overlimit (old behaviour) exception
        # will be raised when out of quota
        self.assertRaises((exceptions.Unauthorized, exceptions.OverLimit),
                          self.server_client.create_server,
                          name='test', meta=meta_data,
                          flavor_ref=self.flavor_ref,
                          image_ref=self.image_ref)
<|endoftext|>


<commit_before>from working_functions import *

print 'Plotting obs-pred plot for SAD, ...'

datafile = open('./data/shrtnames.txt', 'r')
datareader = csv.reader(datafile)
shrtnames = []
for row in datareader:
    shrtnames.append(row)

shrtnames = shrtnames[0][0].split()

plot_obs_pred_sad(shrtnames, data_dir='./sad/',
                  dest_dir='./figs/sad_figs/')

for shrt_name in shrtnames:
    plot_obs_pred_sad([shrt_name], data_dir='./sad/',
                      dest_dir='./figs/sad_figs/' + shrt_name + '_')

print 'Plotting obs-pred plot for SAD, complete!'
<commit_msg>Add functions from mete-energy/working_functions.py needed to import the sad data and create the observed-predicted plots.<commit_after>from __future__ import division
import numpy as np
import csv
import matplotlib
matplotlib.use('Agg')

import matplotlib.pyplot as plt
from matplotlib.backends.backend_pdf import PdfPages
from mpl_toolkits.axes_grid.inset_locator import inset_axes
from mete import *
import macroecotools

def import_obs_pred_data(input_filename):
    data = np.genfromtxt(input_filename, dtype = "S15,f8,f8",
                                  names = ['site','obs','pred'],
                                  delimiter = ",")
    return data

def get_obs_pred_from_file(datasets, data_dir, filename):
    """Read obs and pred value from a file"""
    sites= []
    obs = []
    pred = []
    for i, dataset in enumerate(datasets):
        obs_pred_data = import_obs_pred_data(data_dir + dataset + filename)
        site_list = [dataset + '_' + x for x in list(obs_pred_data['site'])]
        sites.extend(site_list)
        obs.extend(list(obs_pred_data['obs']))
        pred.extend(list(obs_pred_data['pred']))
    return np.array(sites), np.array(obs), np.array(pred)

def plot_obs_pred(obs, pred, radius, loglog, ax = None, inset = False, sites = None):
    """Generic function to generate an observed vs predicted figure with 1:1 line"""
    if not ax:
        fig = plt.figure(figsize = (3.5, 3.5))
        ax = plt.subplot(111)

    axis_min = 0.9 * min(list(obs[obs > 0]) + list(pred[pred > 0]))
    if loglog:
        axis_max = 3 * max(list(obs)+list(pred))
    else:
        axis_max = 1.1 * max(list(obs)+list(pred))
    macroecotools.plot_color_by_pt_dens(np.array(pred), np.array(obs), radius, loglog=loglog, plot_obj = ax)
    plt.plot([axis_min, axis_max],[axis_min, axis_max], 'k-')
    plt.xlim(axis_min, axis_max)
    plt.ylim(axis_min, axis_max)
    ax.tick_params(axis = 'both', which = 'major', labelsize = 6)
    if loglog:
        plt.annotate(r'$R^2$ = %0.2f' %macroecotools.obs_pred_rsquare(np.log10(obs[(obs != 0) * (pred != 0)]), np.log10(pred[(obs != 0) * (pred != 0)])),
                     xy = (0.05, 0.85), xycoords = 'axes fraction', fontsize = 7)
    else:
        plt.annotate(r'$R^2$ = %0.2f' %macroecotools.obs_pred_rsquare(obs, pred),
                     xy = (0.05, 0.85), xycoords = 'axes fraction', fontsize = 7)
    if inset:
        axins = inset_axes(ax, width="30%", height="30%", loc=4)
        if loglog:
            hist_mete_r2(sites[(obs != 0) * (pred != 0)], np.log10(obs[(obs != 0) * (pred != 0)]),
                         np.log10(pred[(obs != 0) * (pred != 0)]))
        else:
            hist_mete_r2(sites, obs, pred)
        plt.setp(axins, xticks=[], yticks=[])
    return ax

def plot_obs_pred_sad(datasets, data_dir = "./data/", dest_dir = "./", radius = 2, inset = False):
    """Plot the observed vs predicted abundance for each species for multiple datasets."""
    rad_sites, rad_obs, rad_pred = get_obs_pred_from_file(datasets, data_dir, '_obs_pred_rad.csv')
    if inset:
        fig = plot_obs_pred(rad_obs, rad_pred, radius, 1, inset = True, sites = rad_sites)
    else:
        fig = plot_obs_pred(rad_obs, rad_pred, radius, 1)
    fig.set_xlabel('Predicted abundance', labelpad = 4, size = 8)
    fig.set_ylabel('Observed abundance', labelpad = 4, size = 8)
    plt.savefig(dest_dir + 'obs_pred_sad.png', dpi = 400)

print 'Plotting obs-pred plot for SAD, ...'

datafile = open('./data/shrtnames.txt', 'r')
datareader = csv.reader(datafile)
shrtnames = []
for row in datareader:
    shrtnames.append(row)

shrtnames = shrtnames[0][0].split()

plot_obs_pred_sad(shrtnames, data_dir='./sad/',
                  dest_dir='./figs/sad_figs/')

for shrt_name in shrtnames:
    plot_obs_pred_sad([shrt_name], data_dir='./sad/',
                      dest_dir='./figs/sad_figs/' + shrt_name + '_')

print 'Plotting obs-pred plot for SAD, complete!'
<|endoftext|>


<commit_before>import numpy as np

from .geometry import (cross_prod, vect_dot, norm_dir_vect,
                       vect_perpendicular, dist_point2line)

def vortex_position_in_panel(P1, P2, P3, P4):
    """
    For a given panel defined by points P1, P2, P3 and P4
    returns the position of the horseshoe vortex defined
    by points A, B, C and D and its control point P.

            ^
          y |                Points defining the panel are
            |                named clockwise.
     P3--C--|--D--P4         Points defining the horseshoe
      |  |  |  |  |          are named clockwise as well.
      |  |  |  |  |
      |  |  +--P--|---->
      |  |     |  |     x
      |  |     |  |
     P2--B-----A--P1

    Parameters
    ----------
    P1, P2, P3, P4 : array_like
                     Points that define the panel

    Returns
    -------
    results : dict
        P - control point where the boundary condition V*n = 0
            is applied according to the Vortice Lattice Method.
        A, B, C, D - points that define the horseshoe position
    """

    i_inf = np.array([1, 0])  # x_Inf(+) direction vector
    P1P2 = P2 - P1
    P3P4 = P4 - P3
    P1P4 = P4 - P1

    if cross_prod(P1P2, i_inf) != 0:
        msg = 'P1P2 segment not aligned with OX'
        raise ValueError(msg)

    if cross_prod(P3P4, i_inf) != 0:
        msg = 'P3P4 segment not aligned with OX'
        raise ValueError(msg)

    A = P1 + P1P2 / 4
    B = A + P1P2 / 2
    C = P3 + P3P4 / 4
    D = C + P3P4 / 2

    AD = D - A
    P = A + AD / 2

    results = [P, A, B, C, D]

    return results

def v_induced_by_horseshoe_vortex(P, A, B, C, D, gamma=1):
    """
    Induced velocity at point P due to a horseshoe vortex
    spatially positioned by points A, B, C and D, extended
    to x_Inf(+) in a 2D euclidean space. Circulation
    direction: x_Inf(+) -> A -> B -> C -> D -> x_Inf(+)

                ^
              y |                Points defining the horseshoe
    V_inf       | i_3            are named clockwise.
    ->     C----|->--D  ...>...  A direction vector is
    ->     |    |    |           calculated for each vortex.
    -> i_2 ^    +----|------>
    ->	   |         |       x
    ->	   B----<----A  ...<...
                i_1

    Parameters
    ----------
    P, A, B, C, D : array_like
                    P - point of reference
                    A, B, C, D - points of the horseshoe vortex
    gamma : circulation

    Returns
    -------
    v : float
    """

    i_inf = np.array([1, 0])  # x_Inf(+) direction vector
    i_1 = np.array([-1, 0])  # trailing vortex(1) dir. vector
    i_2 = norm_dir_vect(B, C)  # bound vortex(2) dir. vector
    i_3 = i_inf  # trailing vortex(3) dir. vector

    if cross_prod(i_1, i_inf) != 0:
        msg = '1st trailing vortex segment not aligned with OX'
        raise ValueError(msg)

    if cross_prod(i_2, i_inf) == 0:
        msg = 'Bound vortex segment aligned with OX'
        raise ValueError(msg)

    if cross_prod(i_3, i_inf) != 0:
        msg = '2nd trailing vortex segment not aligned with OX'
        raise ValueError(msg)

    # First trailing vortex segment x_Inf(+) -> A - > B
    i_PB = norm_dir_vect(P, B)  # PB direction vector
    h_1 = dist_point2line(P, A, B)  # distance to 1st tr. vortex
    if h_1 == 0:
        v_1 = 0
    else:
        sign_1 = np.sign(cross_prod(i_PB, i_1))
        cos_2 = vect_dot(-i_PB, i_1)
        v_1 = sign_1 * (gamma/(4 * np.pi * h_1)) * (1 - cos_2)

    # Bound vortex segment B -> C
    i_PC = norm_dir_vect(P, C)  # PC direction vector
    h_2 = dist_point2line(P, B, C)  # distance to bound vortex
    if h_2 == 0:
        v_2 = 0
    else:
        sign_2 = np.sign(cross_prod(i_PC, i_2))
        cos_1 = vect_dot(-i_PB, i_2)
        cos_2 = vect_dot(-i_PC, i_2)
        v_2 = sign_2 * (gamma/(4 * np.pi * h_2)) * (cos_1 - cos_2)

    # Second trailing vortex segment C -> D -> x_Inf(+)
    h_3 = dist_point2line(P, C, D)  # distance to 2nd tr. vortex
    if h_3 == 0:
        v_3 = 0
    else:
        sign_3 = np.sign(cross_prod(i_PC, i_3))
        cos_1 = vect_dot(-i_PC, i_3)
        v_3 = sign_3 * (gamma/(4 * np.pi * h_3)) * (cos_1 + 1)

    v1 = v_1 + v_2 + v_3  # Total induced velocity in P
    v2 = v_1 + v_3  # Induced velocity in P due to trailing vortices

    return v1, v2

def v_induced_by_finite_vortex_line(P, A, B, gamma=1):
    """

    Y^						Induced velocity at point P due
     |	  + P(x,y)			to a finite straight line vortex
     |						defined by points A and B.
     +=======+--------> X	Circulation from A --> B.
     A		 B

    Parameters
    ----------
    P, A, B : array_like
              P - point of reference
              A, B - points of the vortex
    gamma : circulation

    Returns
    -------
    v : float
    """

    i = norm_dir_vect(A, B)  # vortex_2 direction vector

    i_PA = norm_dir_vect(P, A)  # PB direction vector
    i_PB = norm_dir_vect(P, B)  # PC direction vector

    h = dist_point2line(P, A, B)  # distance to vortex_2
    sign = np.sign(cross_prod(i_PA, i))

    cos_1 = vect_dot(-i_PA, i)
    cos_2 = vect_dot(-i_PB, i)
    v = sign * (gamma/(4 * np.pi * h)) * (cos_1 - cos_2)

    return v
<commit_msg>replaced np.pi by pi<commit_after>import numpy as np

from .geometry import (cross_prod, vect_dot, norm_dir_vect,
                       vect_perpendicular, dist_point2line)

def vortex_position_in_panel(P1, P2, P3, P4):
    """
    For a given panel defined by points P1, P2, P3 and P4
    returns the position of the horseshoe vortex defined
    by points A, B, C and D and its control point P.

            ^
          y |                Points defining the panel are
            |                named clockwise.
     P3--C--|--D--P4         Points defining the horseshoe
      |  |  |  |  |          are named clockwise as well.
      |  |  |  |  |
      |  |  +--P--|---->
      |  |     |  |     x
      |  |     |  |
     P2--B-----A--P1

    Parameters
    ----------
    P1, P2, P3, P4 : array_like
                     Points that define the panel

    Returns
    -------
    results : dict
        P - control point where the boundary condition V*n = 0
            is applied according to the Vortice Lattice Method.
        A, B, C, D - points that define the horseshoe position
    """

    i_inf = np.array([1, 0])  # x_Inf(+) direction vector
    P1P2 = P2 - P1
    P3P4 = P4 - P3
    P1P4 = P4 - P1

    if cross_prod(P1P2, i_inf) != 0:
        msg = 'P1P2 segment not aligned with OX'
        raise ValueError(msg)

    if cross_prod(P3P4, i_inf) != 0:
        msg = 'P3P4 segment not aligned with OX'
        raise ValueError(msg)

    A = P1 + P1P2 / 4
    B = A + P1P2 / 2
    C = P3 + P3P4 / 4
    D = C + P3P4 / 2

    AD = D - A
    P = A + AD / 2

    results = [P, A, B, C, D]

    return results

def v_induced_by_horseshoe_vortex(P, A, B, C, D, gamma=1):
    """
    Induced velocity at point P due to a horseshoe vortex
    spatially positioned by points A, B, C and D, extended
    to x_Inf(+) in a 2D euclidean space. Circulation
    direction: x_Inf(+) -> A -> B -> C -> D -> x_Inf(+)

                ^
              y |                Points defining the horseshoe
    V_inf       | i_3            are named clockwise.
    ->     C----|->--D  ...>...  A direction vector is
    ->     |    |    |           calculated for each vortex.
    -> i_2 ^    +----|------>
    ->	   |         |       x
    ->	   B----<----A  ...<...
                i_1

    Parameters
    ----------
    P, A, B, C, D : array_like
                    P - point of reference
                    A, B, C, D - points of the horseshoe vortex
    gamma : circulation

    Returns
    -------
    v : float
    """

    pi = np.pi

    i_inf = np.array([1, 0])  # x_Inf(+) direction vector
    i_1 = np.array([-1, 0])  # trailing vortex(1) dir. vector
    i_2 = norm_dir_vect(B, C)  # bound vortex(2) dir. vector
    i_3 = i_inf  # trailing vortex(3) dir. vector

    if cross_prod(i_1, i_inf) != 0:
        msg = '1st trailing vortex segment not aligned with OX'
        raise ValueError(msg)

    if cross_prod(i_2, i_inf) == 0:
        msg = 'Bound vortex segment aligned with OX'
        raise ValueError(msg)

    if cross_prod(i_3, i_inf) != 0:
        msg = '2nd trailing vortex segment not aligned with OX'
        raise ValueError(msg)

    # First trailing vortex segment x_Inf(+) -> A - > B
    i_PB = norm_dir_vect(P, B)  # PB direction vector
    h_1 = dist_point2line(P, A, B)  # distance to 1st tr. vortex
    if h_1 == 0:
        v_1 = 0
    else:
        sign_1 = np.sign(cross_prod(i_PB, i_1))
        cos_2 = vect_dot(-i_PB, i_1)
        v_1 = sign_1 * (gamma/(4 * pi * h_1)) * (1 - cos_2)

    # Bound vortex segment B -> C
    i_PC = norm_dir_vect(P, C)  # PC direction vector
    h_2 = dist_point2line(P, B, C)  # distance to bound vortex
    if h_2 == 0:
        v_2 = 0
    else:
        sign_2 = np.sign(cross_prod(i_PC, i_2))
        cos_1 = vect_dot(-i_PB, i_2)
        cos_2 = vect_dot(-i_PC, i_2)
        v_2 = sign_2 * (gamma/(4 * pi * h_2)) * (cos_1 - cos_2)

    # Second trailing vortex segment C -> D -> x_Inf(+)
    h_3 = dist_point2line(P, C, D)  # distance to 2nd tr. vortex
    if h_3 == 0:
        v_3 = 0
    else:
        sign_3 = np.sign(cross_prod(i_PC, i_3))
        cos_1 = vect_dot(-i_PC, i_3)
        v_3 = sign_3 * (gamma/(4 * pi * h_3)) * (cos_1 + 1)

    v1 = v_1 + v_2 + v_3  # Total induced velocity in P
    v2 = v_1 + v_3  # Induced velocity in P due to trailing vortices

    return v1, v2

def v_induced_by_finite_vortex_line(P, A, B, gamma=1):
    """

    Y^						Induced velocity at point P due
     |	  + P(x,y)			to a finite straight line vortex
     |						defined by points A and B.
     +=======+--------> X	Circulation from A --> B.
     A		 B

    Parameters
    ----------
    P, A, B : array_like
              P - point of reference
              A, B - points of the vortex
    gamma : circulation

    Returns
    -------
    v : float
    """

    i = norm_dir_vect(A, B)  # vortex_2 direction vector

    i_PA = norm_dir_vect(P, A)  # PB direction vector
    i_PB = norm_dir_vect(P, B)  # PC direction vector

    h = dist_point2line(P, A, B)  # distance to vortex_2
    sign = np.sign(cross_prod(i_PA, i))

    cos_1 = vect_dot(-i_PA, i)
    cos_2 = vect_dot(-i_PB, i)
    v = sign * (gamma/(4 * pi * h)) * (cos_1 - cos_2)

    return v
<|endoftext|>


<commit_before>#!/usr/bin/env python
import sys
import os
import re
from collections import defaultdict
from math import floor
from math import isnan

root_dir = sys.argv[1]
prefix = sys.argv[2]
do_iso = sys.argv[3]

method2stats = defaultdict(list)
fileCount = 0;

for f in os.listdir(root_dir):
    fp = root_dir+"/"+f
    if os.path.isfile(fp) and ".out" in f and f.startswith(prefix):
        infile = open(fp)
        fileCount+=1

        for line in infile:
            if do_iso == "F":
                [count, mean, min_v, q1, median, q3, max_v, method, stat] = line.strip().split("\t")
                label = method +"\t"+stat
            else:
                [count, mean, min_v, q1, median, q3, max_v, iso, method, stat] = line.strip().split("\t")
            label = iso + "\t" + method +"\t"+stat

            count = float(count)
            mean = float(mean)
            min_v = float(min_v)
            q1 = float(q1)
            median = float(median)
            q3 = float(q3)
            max_v = float(max_v)

            if not method2stats.has_key(label):
                method2stats[label] = [count, count * mean, min_v, q1, median, q3, max_v]
            else:
                method2stats[label][0] += count
                method2stats[label][1] += count * mean
                method2stats[label][2] = min(min_v,  method2stats[label][2])
                method2stats[label][3] += q1
                method2stats[label][4] += median
                method2stats[label][5] += q3
                method2stats[label][6] = max(max_v,  method2stats[label][6])

        infile.close()

for method in method2stats:
    mean = method2stats[method][1] / method2stats[method][0]
    min_v =  method2stats[method][2]
    q1 =  method2stats[method][3] / fileCount
    median =  method2stats[method][4] / fileCount
    q3 =  method2stats[method][5] / fileCount
    max_v =  method2stats[method][6]

    print "\t".join([str(mean), str(min_v), str(q1), str(median), str(q3), str(max_v), method])
<commit_msg>stats merging<commit_after>#!/usr/bin/env python
import sys
import os
import re
from collections import defaultdict
from math import floor
from math import isnan

root_dir = sys.argv[1]
prefix = sys.argv[2]
do_iso = sys.argv[3]

method2stats = defaultdict(list)
fileCount = 0;

for f in os.listdir(root_dir):
    fp = root_dir+"/"+f
    if os.path.isfile(fp) and ".out" in f and f.startswith(prefix):
        infile = open(fp)
        fileCount+=1

        for line in infile:
            if do_iso == "F":
                [count, mean, min_v, q1, median, q3, max_v, method, stat] = line.strip().split("\t")
                label = method + "\t" + stat
            else:
                [count, mean, min_v, q1, median, q3, max_v, iso, method, stat] = line.strip().split("\t")
                label = iso + "\t" + method + "\t" + stat

            count = float(count)
            mean = float(mean)
            min_v = float(min_v)
            q1 = float(q1)
            median = float(median)
            q3 = float(q3)
            max_v = float(max_v)

            if not method2stats.has_key(label):
                method2stats[label] = [count, count * mean, min_v, q1, median, q3, max_v]
            else:
                method2stats[label][0] += count
                method2stats[label][1] += count * mean
                method2stats[label][2] = min(min_v,  method2stats[label][2])
                method2stats[label][3] += q1
                method2stats[label][4] += median
                method2stats[label][5] += q3
                method2stats[label][6] = max(max_v,  method2stats[label][6])

        infile.close()

for method in method2stats:
    mean = method2stats[method][1] / method2stats[method][0]
    min_v =  method2stats[method][2]
    q1 =  method2stats[method][3] / fileCount
    median =  method2stats[method][4] / fileCount
    q3 =  method2stats[method][5] / fileCount
    max_v =  method2stats[method][6]

    print "\t".join([str(mean), str(min_v), str(q1), str(median), str(q3), str(max_v), method])
<|endoftext|>


<commit_before># Copyright 2021, The TensorFlow Federated Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Libraries providing implementations of federated learning algorithms."""

from tensorflow_federated.python.learning.algorithms.fed_avg import build_unweighted_fed_avg
from tensorflow_federated.python.learning.algorithms.fed_avg import build_weighted_fed_avg
from tensorflow_federated.python.learning.algorithms.fed_avg_with_optimizer_schedule import build_weighted_fed_avg_with_optimizer_schedule
from tensorflow_federated.python.learning.algorithms.fed_prox import build_weighted_fed_prox
from tensorflow_federated.python.learning.algorithms.fed_sgd import build_fed_sgd
<commit_msg>Adds `build_unweighted_fed_prox` to the algorithms API.<commit_after># Copyright 2021, The TensorFlow Federated Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Libraries providing implementations of federated learning algorithms."""

from tensorflow_federated.python.learning.algorithms.fed_avg import build_unweighted_fed_avg
from tensorflow_federated.python.learning.algorithms.fed_avg import build_weighted_fed_avg
from tensorflow_federated.python.learning.algorithms.fed_avg_with_optimizer_schedule import build_weighted_fed_avg_with_optimizer_schedule
from tensorflow_federated.python.learning.algorithms.fed_prox import build_unweighted_fed_prox
from tensorflow_federated.python.learning.algorithms.fed_prox import build_weighted_fed_prox
from tensorflow_federated.python.learning.algorithms.fed_sgd import build_fed_sgd
<|endoftext|>