<commit_before>#!/usr/bin/env python2.7

import re
import os
import glob
import pickle
import platform
import signal
import sys
from StringIO import StringIO
from subprocess import Popen, PIPE
from multiprocessing import Pool, cpu_count
import time

#
# testLogFile
#

testLogFile = open("testlog.txt", "w")

#
# testRoot
#

testBuildFlags = []
testRoot = os.path.dirname(os.path.abspath(__file__))
runTestRoot = testRoot

#
# getClayPlatform, fileForPlatform
#

def getClayPlatform():
    if sys.platform == "win32" or sys.platform == "cygwin":
        return "windows"
    if sys.platform == "darwin":
        return "macosx"
    if sys.platform.startswith("freebsd"):
        return "freebsd"
    if sys.platform.startswith("linux"):
        return "linux"

def getClayPlatformFamily():
    if sys.platform == "win32" or sys.platform == "cygwin":
        return "windows"
    else:
        return "unix"

def getClayBits():
    if platform.architecture()[0] == "64bit":
        return "64"
    if platform.architecture()[0] == "32bit":
        return "32"

clayPlatform = getClayPlatform()
clayPlatformFamily = getClayPlatformFamily()
clayBits = getClayBits()

def fileForPlatform(folder, name, ext):
    platformNames = [
        "%s.%s.%s.%s" % (name, clayPlatform, clayBits, ext),
        "%s.%s.%s.%s" % (name, clayPlatformFamily, clayBits, ext),
        "%s.%s.%s"    % (name, clayPlatform, ext),
        "%s.%s.%s"    % (name, clayPlatformFamily, ext),
        "%s.%s.%s"    % (name, clayBits, ext),
    ]
    for platformName in platformNames:
        fullName = os.path.join(folder, platformName)
        if os.path.isfile(fullName):
            return fullName

    return os.path.join(folder, "%s.%s" % (name, ext))

#
# getCompilerPath
#

def getCompilerPath() :
    buildPath = ["..", "build", "compiler", "src"]
    if sys.platform == "win32" :
        compiler = os.path.join(testRoot, *(buildPath + ["clay.exe"]))
        compiler2 = os.path.join(testRoot, "..", "clay.exe") # for binary distributions
    else :
        compiler = os.path.join(testRoot, *(buildPath + ["clay"]))
        compiler2 = os.path.join(testRoot, "..", "clay") # for binary distributions

    if not os.path.exists(compiler) :
        compiler = compiler2
        if not os.path.exists(compiler) :
            print "could not find the clay compiler"
            sys.exit(-1)
    return compiler

compiler = getCompilerPath()

#
# TestCase
#

class TestCase(object):
    allCases = []
    def __init__(self, folder, testfile, base = None):
        entries = os.listdir(folder)
        self.testfile = os.path.basename(testfile)
        self.path = folder
        if os.path.isfile(testfile):
            self.loadTest(testfile)

        runscript = fileForPlatform(folder, "run", "py")
        if os.path.isfile(runscript):
            self.runscript = runscript
        else:
            self.runscript = None

        buildflags = fileForPlatform(folder, "buildflags", "txt")
        if os.path.isfile(buildflags):
            self.buildflags = open(buildflags).read().split()
        else:
            self.buildflags = []

        for entry in entries:
            fullpath = os.path.join(folder, entry)
            if not os.path.isdir(fullpath):
                continue
            findTestCase(fullpath, self)

    def loadTest(self, testfile):
        TestCase.allCases.append(self)
        self.testfile = testfile

    def cmdline(self, clay) :
        return [clay, "-I" + self.path, "-o", "test.exe"] + testBuildFlags + self.buildflags + [self.testfile]

    def pre_build(self) :
        pass

    def post_build(self) :
        pass

    def post_run(self) :
        [os.unlink(f) for f in glob.glob("temp*")]
        [os.unlink(f) for f in glob.glob("*.data")]

    def match(self, resultout, resulterr, returncode) :
        compilererrfile = fileForPlatform(".", "compilererr", "txt")
        if os.path.isfile(compilererrfile):
            errpattern = open(compilererrfile).read()
            errpattern = errpattern.replace('\r', '').strip()

            if returncode != "compiler error":
                print >>self.testLogBuffer, "compiler did not fail"
                print >>self.testLogBuffer, "expected error pattern"
                print >>self.testLogBuffer, "----------------------"
                print >>self.testLogBuffer, errpattern
                return "compiler did not fail"
            if re.search(errpattern, resulterr):
                return "ok"
            else:
                print >>self.testLogBuffer, "unexpected compiler error"
                print >>self.testLogBuffer, "expected error pattern"
                print >>self.testLogBuffer, "----------------------"
                print >>self.testLogBuffer, errpattern
                print >>self.testLogBuffer, "---------------------"
                print >>self.testLogBuffer, "actual compiler error"
                print >>self.testLogBuffer, "---------------------"
                print >>self.testLogBuffer, resulterr
                return "unexpected compiler error"

        else:
            if returncode == "compiler error":
                print >>self.testLogBuffer, "compiler error"
                print >>self.testLogBuffer, "--------------"
                print >>self.testLogBuffer, resulterr
                return "compiler error"
            outfile = fileForPlatform(".", "out", "txt")
            errfile = fileForPlatform(".", "err", "txt")
            if not os.path.isfile(outfile) :
                print >>self.testLogBuffer, "out.txt missing"
                return "out.txt missing"
            refout = open(outfile).read()
            referr = ""
            if os.path.isfile(errfile):
                referr = open(errfile).read()
            resultout = resultout.replace('\r', '')
            refout    = refout.replace('\r', '')
            resulterr = resulterr.replace('\r', '')
            referr    = referr.replace('\r', '')
            if resultout == refout and resulterr == referr:
                return "ok"
            elif resultout != refout and resulterr != referr:
                print >>self.testLogBuffer, "out.txt and err.txt mismatch"
                print >>self.testLogBuffer, "expected out.txt"
                print >>self.testLogBuffer, "----------------"
                print >>self.testLogBuffer, refout
                print >>self.testLogBuffer, "-------------"
                print >>self.testLogBuffer, "actual output"
                print >>self.testLogBuffer, "-------------"
                print >>self.testLogBuffer, resultout
                print >>self.testLogBuffer, "----------------"
                print >>self.testLogBuffer, "expected err.txt"
                print >>self.testLogBuffer, "----------------"
                print >>self.testLogBuffer, referr
                print >>self.testLogBuffer, "-------------"
                print >>self.testLogBuffer, "actual stderr"
                print >>self.testLogBuffer, "-------------"
                print >>self.testLogBuffer, resulterr
                return "out.txt and err.txt mismatch"
            elif resultout != refout:
                print >>self.testLogBuffer, "out.txt mismatch"
                print >>self.testLogBuffer, "expected out.txt"
                print >>self.testLogBuffer, "----------------"
                print >>self.testLogBuffer, refout
                print >>self.testLogBuffer, "-------------"
                print >>self.testLogBuffer, "actual output"
                print >>self.testLogBuffer, "-------------"
                print >>self.testLogBuffer, resultout
                return "out.txt mismatch"
            elif resulterr != referr:
                print >>self.testLogBuffer, "err.txt mismatch"
                print >>self.testLogBuffer, "expected err.txt"
                print >>self.testLogBuffer, "----------------"
                print >>self.testLogBuffer, referr
                print >>self.testLogBuffer, "-------------"
                print >>self.testLogBuffer, "actual stderr"
                print >>self.testLogBuffer, "-------------"
                print >>self.testLogBuffer, resulterr
                return "err.txt mismatch"

    def run(self):
        self.testLogBuffer = StringIO()
        print >>self.testLogBuffer
        print >>self.testLogBuffer, "====================="
        print >>self.testLogBuffer, self.name()
        print >>self.testLogBuffer, "====================="
        os.chdir(self.path)
        self.pre_build()
        self.post_build()
        resultout, resulterr, returncode = self.runtest()
        self.post_run()
        r = self.match(resultout, resulterr, returncode)
        log = self.testLogBuffer.getvalue()
        self.testLogBuffer.close()

        return (r, log)

    def name(self):
        return os.path.relpath(self.path, testRoot)

    def runtest(self):
        outfilename = "test.exe"
        outfilename = os.path.join(".", outfilename)
        process = Popen(self.cmdline(compiler), stdout=PIPE, stderr=PIPE)
        compilerout, compilererr = process.communicate()
        if process.returncode != 0 :
            return "", "%s\n%s" % (compilerout, compilererr), "compiler error"
        if self.runscript is None:
            commandline = [outfilename]
        else:
            commandline = [sys.executable, self.runscript, outfilename] + testBuildFlags

        process = Popen(commandline, stdout=PIPE, stderr=PIPE)
        resultout, resulterr = process.communicate()
        self.removefile(outfilename)
        return resultout, resulterr, process.returncode

    def removefile(self, filename) :
        # on windows, sometimes, deleting a file
        # immediately after executing it
        # results in a 'access denied' error.
        # so we wait and try again a few times.
        attempts = 1
        while (attempts <= 3) and os.path.exists(filename) :
            try :
                os.unlink(filename)
            except OSError :
                time.sleep(1)
            attempts += 1

class TestModuleCase(TestCase):
    def match(self, resultout, resulterr, returncode) :
        if returncode == 0:
            return "ok"
        elif returncode == "compiler error":
            print >>self.testLogBuffer, "compiler error"
            print >>self.testLogBuffer, "--------------"
            print >>self.testLogBuffer, resulterr
            return "compiler error"
        else:
            print >>self.testLogBuffer, "fail"
            print >>self.testLogBuffer, "----"
            print >>self.testLogBuffer, resultout
            return "fail"

class TestDisabledCase(TestCase):
    def runtest(self):
        return "disabled", "", None
    def match(self, resultout, resulterr, returncode) :
        return "disabled"

#
# runtests
#

def findTestCase(folder, base = None):
    testPath = fileForPlatform(folder, "test", "clay")
    mainPath = fileForPlatform(folder, "main", "clay")
    testDisabledPath = fileForPlatform(folder, "test-disabled", "clay")
    mainDisabledPath = fileForPlatform(folder, "main-disabled", "clay")
    if os.path.isfile(testPath):
        TestModuleCase(folder, testPath, base)
    elif os.path.isfile(testDisabledPath):
        TestDisabledCase(folder, testDisabledPath, base)
    elif os.path.isfile(mainDisabledPath):
        TestDisabledCase(folder, mainDisabledPath, base)
    else:
        TestCase(folder, mainPath, base)

def findTestCases():
    findTestCase(runTestRoot)
    return TestCase.allCases

def initWorker():
    signal.signal(signal.SIGINT, signal.SIG_IGN)

def runTest(t):
    return t.run()

def runTests() :
    testcases = findTestCases()
    pool = Pool(processes = cpu_count(), initializer=initWorker)
    results = pool.imap(runTest, testcases)
    succeeded = []
    failed = []
    disabled = []
    try:
        for test in testcases:
            res, log = results.next()
            if res != "ok" and res != "disabled":
                print >>testLogFile, log,
            testLogFile.flush()
            print "TEST %s: %s" % (test.name(), res)
            if res == "disabled":
                disabled.append(test.name())
            elif res != "ok":
                failed.append(test.name())
            else:
                succeeded.append(test.name())
    except KeyboardInterrupt:
        print "\nInterrupted!"
        pool.terminate()

    print "\nPASSED %d TESTS" % len(succeeded)
    if len(disabled) != 0:
        print "(%d tests disabled)" % len(disabled)
    if len(failed) != 0:
        print "\nFAILED %d TESTS" % len(failed)
        print "Failed tests:\n ",
        print "\n  ".join(failed)
    testLogFile.flush()
    print "Test log written to testlog.txt"

def usage(argv0):
    print "Usage: %s [buildflags... --] [root] [root...]" % argv0
    print "  Runs the Clay test suite. If any root paths are given, only tests"
    print "  in those subdirectories (paths relative to the test/ directory) will"
    print "  be run. Build flags can be passed to the compiler followed by '--'."

def main() :
    global testRoot
    global runTestRoot
    if len(sys.argv) > 1 :
        if sys.argv[1] == "--help" or sys.argv[1] == "/?":
            usage(sys.argv[0])
            return

        try:
            buildFlagSeparator = sys.argv.index("--")
            testBuildFlags = sys.argv[1:buildFlagSeparator]
            runTestRoot = os.path.join(testRoot, *sys.argv[buildFlagSeparator+1:])
        except ValueError:
            testBuildFlags = []
            runTestRoot = os.path.join(testRoot, *sys.argv[1:])
    else:
        testBuildFlags = []
        runTestRoot = testRoot
    startTime = time.time()
    runTests()
    endTime = time.time()
    print ""
    print "time taken = %f seconds" % (endTime - startTime)

if __name__ == "__main__":
    main()
<commit_msg>make runtests script architecture-aware<commit_after>#!/usr/bin/env python2.7

import re
import os
import glob
import pickle
import platform
import signal
import sys
from StringIO import StringIO
from subprocess import Popen, PIPE
from multiprocessing import Pool, cpu_count
import time

#
# testLogFile
#

testLogFile = open("testlog.txt", "w")

#
# testRoot
#

testBuildFlags = []
testRoot = os.path.dirname(os.path.abspath(__file__))
runTestRoot = testRoot

#
# getClayPlatform, fileForPlatform
#

def getClayPlatform():
    if sys.platform in ["win32", "cygwin"]:
        return "windows"
    if sys.platform == "darwin":
        return "macosx"
    if sys.platform.startswith("freebsd"):
        return "freebsd"
    if sys.platform.startswith("linux"):
        return "linux"

def getClayPlatformFamily():
    if sys.platform in ["win32", "cygwin"]:
        return "windows"
    else:
        return "unix"

def getClayArchitecture():
    if platform.machine() in ["i386", "x86_64", "x86-64", "amd64", "AMD64", "x64"]:
        return "x86"
    else:
        return "unknown"

# XXX this needs to determine the bitness of clay executables, not of python
def getClayBits():
    if platform.architecture()[0] == "64bit":
        return "64"
    if platform.architecture()[0] == "32bit":
        return "32"

clayPlatform = getClayPlatform()
clayPlatformFamily = getClayPlatformFamily()
clayArchitecture = getClayArchitecture()
clayBits = getClayBits()

def fileForPlatform(folder, name, ext):
    platformNames = [
        "%s.%s.%s.%s.%s" % (name, clayPlatform, clayArchitecture, clayBits, ext),
        "%s.%s.%s.%s.%s" % (name, clayPlatformFamily, clayArchitecture, clayBits, ext),
        "%s.%s.%s.%s" % (name, clayPlatform, clayArchitecture, ext),
        "%s.%s.%s.%s" % (name, clayPlatform, clayBits, ext),
        "%s.%s.%s.%s" % (name, clayPlatformFamily, clayArchitecture, ext),
        "%s.%s.%s.%s" % (name, clayPlatformFamily, clayBits, ext),
        "%s.%s.%s.%s" % (name, clayArchitecture, clayBits, ext),
        "%s.%s.%s" % (name, clayPlatform, ext),
        "%s.%s.%s" % (name, clayPlatformFamily, ext),
        "%s.%s.%s" % (name, clayArchitecture, ext),
        "%s.%s.%s" % (name, clayBits, ext),
    ]
    for platformName in platformNames:
        fullName = os.path.join(folder, platformName)
        if os.path.isfile(fullName):
            return fullName

    return os.path.join(folder, "%s.%s" % (name, ext))

#
# getCompilerPath
#

def getCompilerPath() :
    buildPath = ["..", "build", "compiler", "src"]
    if sys.platform == "win32" :
        compiler = os.path.join(testRoot, *(buildPath + ["clay.exe"]))
        compiler2 = os.path.join(testRoot, "..", "clay.exe") # for binary distributions
    else :
        compiler = os.path.join(testRoot, *(buildPath + ["clay"]))
        compiler2 = os.path.join(testRoot, "..", "clay") # for binary distributions

    if not os.path.exists(compiler) :
        compiler = compiler2
        if not os.path.exists(compiler) :
            print "could not find the clay compiler"
            sys.exit(-1)
    return compiler

compiler = getCompilerPath()

#
# TestCase
#

class TestCase(object):
    allCases = []
    def __init__(self, folder, testfile, base = None):
        entries = os.listdir(folder)
        self.testfile = os.path.basename(testfile)
        self.path = folder
        if os.path.isfile(testfile):
            self.loadTest(testfile)

        runscript = fileForPlatform(folder, "run", "py")
        if os.path.isfile(runscript):
            self.runscript = runscript
        else:
            self.runscript = None

        buildflags = fileForPlatform(folder, "buildflags", "txt")
        if os.path.isfile(buildflags):
            self.buildflags = open(buildflags).read().split()
        else:
            self.buildflags = []

        for entry in entries:
            fullpath = os.path.join(folder, entry)
            if not os.path.isdir(fullpath):
                continue
            findTestCase(fullpath, self)

    def loadTest(self, testfile):
        TestCase.allCases.append(self)
        self.testfile = testfile

    def cmdline(self, clay) :
        return [clay, "-I" + self.path, "-o", "test.exe"] + testBuildFlags + self.buildflags + [self.testfile]

    def pre_build(self) :
        pass

    def post_build(self) :
        pass

    def post_run(self) :
        [os.unlink(f) for f in glob.glob("temp*")]
        [os.unlink(f) for f in glob.glob("*.data")]

    def match(self, resultout, resulterr, returncode) :
        compilererrfile = fileForPlatform(".", "compilererr", "txt")
        if os.path.isfile(compilererrfile):
            errpattern = open(compilererrfile).read()
            errpattern = errpattern.replace('\r', '').strip()

            if returncode != "compiler error":
                print >>self.testLogBuffer, "compiler did not fail"
                print >>self.testLogBuffer, "expected error pattern"
                print >>self.testLogBuffer, "----------------------"
                print >>self.testLogBuffer, errpattern
                return "compiler did not fail"
            if re.search(errpattern, resulterr):
                return "ok"
            else:
                print >>self.testLogBuffer, "unexpected compiler error"
                print >>self.testLogBuffer, "expected error pattern"
                print >>self.testLogBuffer, "----------------------"
                print >>self.testLogBuffer, errpattern
                print >>self.testLogBuffer, "---------------------"
                print >>self.testLogBuffer, "actual compiler error"
                print >>self.testLogBuffer, "---------------------"
                print >>self.testLogBuffer, resulterr
                return "unexpected compiler error"

        else:
            if returncode == "compiler error":
                print >>self.testLogBuffer, "compiler error"
                print >>self.testLogBuffer, "--------------"
                print >>self.testLogBuffer, resulterr
                return "compiler error"
            outfile = fileForPlatform(".", "out", "txt")
            errfile = fileForPlatform(".", "err", "txt")
            if not os.path.isfile(outfile) :
                print >>self.testLogBuffer, "out.txt missing"
                return "out.txt missing"
            refout = open(outfile).read()
            referr = ""
            if os.path.isfile(errfile):
                referr = open(errfile).read()
            resultout = resultout.replace('\r', '')
            refout    = refout.replace('\r', '')
            resulterr = resulterr.replace('\r', '')
            referr    = referr.replace('\r', '')
            if resultout == refout and resulterr == referr:
                return "ok"
            elif resultout != refout and resulterr != referr:
                print >>self.testLogBuffer, "out.txt and err.txt mismatch"
                print >>self.testLogBuffer, "expected out.txt"
                print >>self.testLogBuffer, "----------------"
                print >>self.testLogBuffer, refout
                print >>self.testLogBuffer, "-------------"
                print >>self.testLogBuffer, "actual output"
                print >>self.testLogBuffer, "-------------"
                print >>self.testLogBuffer, resultout
                print >>self.testLogBuffer, "----------------"
                print >>self.testLogBuffer, "expected err.txt"
                print >>self.testLogBuffer, "----------------"
                print >>self.testLogBuffer, referr
                print >>self.testLogBuffer, "-------------"
                print >>self.testLogBuffer, "actual stderr"
                print >>self.testLogBuffer, "-------------"
                print >>self.testLogBuffer, resulterr
                return "out.txt and err.txt mismatch"
            elif resultout != refout:
                print >>self.testLogBuffer, "out.txt mismatch"
                print >>self.testLogBuffer, "expected out.txt"
                print >>self.testLogBuffer, "----------------"
                print >>self.testLogBuffer, refout
                print >>self.testLogBuffer, "-------------"
                print >>self.testLogBuffer, "actual output"
                print >>self.testLogBuffer, "-------------"
                print >>self.testLogBuffer, resultout
                return "out.txt mismatch"
            elif resulterr != referr:
                print >>self.testLogBuffer, "err.txt mismatch"
                print >>self.testLogBuffer, "expected err.txt"
                print >>self.testLogBuffer, "----------------"
                print >>self.testLogBuffer, referr
                print >>self.testLogBuffer, "-------------"
                print >>self.testLogBuffer, "actual stderr"
                print >>self.testLogBuffer, "-------------"
                print >>self.testLogBuffer, resulterr
                return "err.txt mismatch"

    def run(self):
        self.testLogBuffer = StringIO()
        print >>self.testLogBuffer
        print >>self.testLogBuffer, "====================="
        print >>self.testLogBuffer, self.name()
        print >>self.testLogBuffer, "====================="
        os.chdir(self.path)
        self.pre_build()
        self.post_build()
        resultout, resulterr, returncode = self.runtest()
        self.post_run()
        r = self.match(resultout, resulterr, returncode)
        log = self.testLogBuffer.getvalue()
        self.testLogBuffer.close()

        return (r, log)

    def name(self):
        return os.path.relpath(self.path, testRoot)

    def runtest(self):
        outfilename = "test.exe"
        outfilename = os.path.join(".", outfilename)
        process = Popen(self.cmdline(compiler), stdout=PIPE, stderr=PIPE)
        compilerout, compilererr = process.communicate()
        if process.returncode != 0 :
            return "", "%s\n%s" % (compilerout, compilererr), "compiler error"
        if self.runscript is None:
            commandline = [outfilename]
        else:
            commandline = [sys.executable, self.runscript, outfilename] + testBuildFlags

        process = Popen(commandline, stdout=PIPE, stderr=PIPE)
        resultout, resulterr = process.communicate()
        self.removefile(outfilename)
        return resultout, resulterr, process.returncode

    def removefile(self, filename) :
        # on windows, sometimes, deleting a file
        # immediately after executing it
        # results in a 'access denied' error.
        # so we wait and try again a few times.
        attempts = 1
        while (attempts <= 3) and os.path.exists(filename) :
            try :
                os.unlink(filename)
            except OSError :
                time.sleep(1)
            attempts += 1

class TestModuleCase(TestCase):
    def match(self, resultout, resulterr, returncode) :
        if returncode == 0:
            return "ok"
        elif returncode == "compiler error":
            print >>self.testLogBuffer, "compiler error"
            print >>self.testLogBuffer, "--------------"
            print >>self.testLogBuffer, resulterr
            return "compiler error"
        else:
            print >>self.testLogBuffer, "fail"
            print >>self.testLogBuffer, "----"
            print >>self.testLogBuffer, resultout
            return "fail"

class TestDisabledCase(TestCase):
    def runtest(self):
        return "disabled", "", None
    def match(self, resultout, resulterr, returncode) :
        return "disabled"

#
# runtests
#

def findTestCase(folder, base = None):
    testPath = fileForPlatform(folder, "test", "clay")
    mainPath = fileForPlatform(folder, "main", "clay")
    testDisabledPath = fileForPlatform(folder, "test-disabled", "clay")
    mainDisabledPath = fileForPlatform(folder, "main-disabled", "clay")
    if os.path.isfile(testPath):
        TestModuleCase(folder, testPath, base)
    elif os.path.isfile(testDisabledPath):
        TestDisabledCase(folder, testDisabledPath, base)
    elif os.path.isfile(mainDisabledPath):
        TestDisabledCase(folder, mainDisabledPath, base)
    else:
        TestCase(folder, mainPath, base)

def findTestCases():
    findTestCase(runTestRoot)
    return TestCase.allCases

def initWorker():
    signal.signal(signal.SIGINT, signal.SIG_IGN)

def runTest(t):
    return t.run()

def runTests() :
    testcases = findTestCases()
    pool = Pool(processes = cpu_count(), initializer=initWorker)
    results = pool.imap(runTest, testcases)
    succeeded = []
    failed = []
    disabled = []
    try:
        for test in testcases:
            res, log = results.next()
            if res != "ok" and res != "disabled":
                print >>testLogFile, log,
            testLogFile.flush()
            print "TEST %s: %s" % (test.name(), res)
            if res == "disabled":
                disabled.append(test.name())
            elif res != "ok":
                failed.append(test.name())
            else:
                succeeded.append(test.name())
    except KeyboardInterrupt:
        print "\nInterrupted!"
        pool.terminate()

    print "\nPASSED %d TESTS" % len(succeeded)
    if len(disabled) != 0:
        print "(%d tests disabled)" % len(disabled)
    if len(failed) != 0:
        print "\nFAILED %d TESTS" % len(failed)
        print "Failed tests:\n ",
        print "\n  ".join(failed)
    testLogFile.flush()
    print "Test log written to testlog.txt"

def usage(argv0):
    print "Usage: %s [buildflags... --] [root] [root...]" % argv0
    print "  Runs the Clay test suite. If any root paths are given, only tests"
    print "  in those subdirectories (paths relative to the test/ directory) will"
    print "  be run. Build flags can be passed to the compiler followed by '--'."

def main() :
    global testRoot
    global runTestRoot
    if len(sys.argv) > 1 :
        if sys.argv[1] == "--help" or sys.argv[1] == "/?":
            usage(sys.argv[0])
            return

        try:
            buildFlagSeparator = sys.argv.index("--")
            testBuildFlags = sys.argv[1:buildFlagSeparator]
            runTestRoot = os.path.join(testRoot, *sys.argv[buildFlagSeparator+1:])
        except ValueError:
            testBuildFlags = []
            runTestRoot = os.path.join(testRoot, *sys.argv[1:])
    else:
        testBuildFlags = []
        runTestRoot = testRoot
    startTime = time.time()
    runTests()
    endTime = time.time()
    print ""
    print "time taken = %f seconds" % (endTime - startTime)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>'''
A Python object relational mapper for SQLite.

Author: Fernando Felix do Nascimento Junior
License: MIT License
Homepage: https://github.com/fernandojunior/python-sqlite-orm
'''
import sqlite3

def cut_attrs(obj, keys):
    return dict(i for i in obj.__dict__.items() if i[0] not in keys)

def render_schema(model):
    schema = 'create table {table} (id integer primary key autoincrement, {columns});'  # noqa
    datatypes = {str: 'text', int: 'integer', float: 'real'}
    iscol = lambda key, value: key[0] is not '_' and value in datatypes.keys()
    colrender = lambda key, value: '%s %s' % (key, datatypes[value])
    cols = [colrender(*i) for i in model.__dict__.items() if iscol(*i)]
    values = {'table': model.__name__, 'columns': ', '.join(cols)}
    return schema.format(**values)

class Database(object):

    def __init__(self, *args, **kwargs):
        self.args = args
        self.kwargs = kwargs
        self.connected = False
        self.Model = type('Model%s' % str(self), (Model,), {'db': self})

    @property
    def connection(self):
        if self.connected:
            return self._connection
        self._connection = sqlite3.connect(*self.args, **self.kwargs)
        self._connection.row_factory = sqlite3.Row
        self.connected = True
        return self._connection

    def close(self):
        if self.connected:
            self.connection.close()
        self.connected = False

    def commit(self):
        self.connection.commit()

    def execute(self, sql, *args):
        return self.connection.execute(sql, args)

    def executescript(self, script):
        self.connection.cursor().executescript(script)
        self.commit()

class Manager(object):

    def __init__(self, db, model):
        self.db = db
        self.model = model
        self.tablename = model.__name__
        if not self._hastable():
            self.db.executescript(render_schema(self.model))

    def all(self):
        cursor = self.db.execute('select * from %s' % self.tablename)
        return (self.create(**row) for row in cursor.fetchall())

    def create(self, **kwargs):
        obj = object.__new__(self.model)
        obj.__dict__ = kwargs
        return obj

    def delete(self, obj):
        sql = 'DELETE from %s WHERE id = ?'
        self.db.execute(sql % self.tablename, obj.id)

    def get(self, id):
        sql = 'select * from %s where id = ?' % self.tablename
        cursor = self.db.execute(sql, id)
        row = cursor.fetchone()
        if not row:
            msg = 'Object%s with id does not exist: %s' % (self.model, id)
            raise ValueError(msg)
        return self.create(**row)

    def has(self, id):
        sql = 'select id from %s where id = ?' % self.tablename
        cursor = self.db.execute(sql, id)
        return True if cursor.fetchall() else False

    def save(self, obj):
        if hasattr(obj, 'id') and self.has(obj.id):
            msg = 'Object%s id already registred: %s' % (self.model, obj.id)
            raise ValueError(msg)
        copy_ = cut_attrs(obj, 'id')
        keys = '(%s)' % ', '.join(copy_.keys())  # (key1, ...)
        refs = '(%s)' % ', '.join('?' for i in range(len(copy_)))  # (?, ...)
        sql = 'insert into %s %s values %s' % (self.tablename, keys, refs)
        cursor = self.db.execute(sql, *copy_.values())
        obj.id = cursor.lastrowid
        return obj

    def update(self, obj):
        copy_ = cut_attrs(obj, 'id')
        keys = '= ?, '.join(copy_.keys()) + '= ?'  # key1 = ?, ...
        sql = 'UPDATE %s SET %s WHERE id = ?' % (self.tablename, keys)
        self.db.execute(sql, *(list(copy_.values()) + [obj.id]))

    def _hastable(self):
        sql = 'select name len FROM sqlite_master where type = ? AND name = ?'
        cursor = self.db.execute(sql, 'table', self.tablename)
        return True if cursor.fetchall() else False

class Model(object):

    db = None

    def delete(self):
        return self.__class__.manager().delete(self)

    def save(self):
        return self.__class__.manager().save(self)

    def update(self):
        return self.__class__.manager().update(self)

    @property
    def public(self):
        return dict(i for i in vars(self).items() if i[0][0] is not '_')

    def __repr__(self):
        return str(self.public)

    @classmethod
    def manager(cls, db=None):
        return Manager(db if db else cls.db, cls)
<commit_msg>__dict__ -> vars<commit_after>'''
A Python object relational mapper for SQLite.

Author: Fernando Felix do Nascimento Junior
License: MIT License
Homepage: https://github.com/fernandojunior/python-sqlite-orm
'''
import sqlite3

def cut_attrs(obj, keys):
    return dict(i for i in vars(obj).items() if i[0] not in keys)

def render_schema(model):
    schema = 'create table {table} (id integer primary key autoincrement, {columns});'  # noqa
    datatypes = {str: 'text', int: 'integer', float: 'real'}
    iscol = lambda key, value: key[0] is not '_' and value in datatypes.keys()
    colrender = lambda key, value: '%s %s' % (key, datatypes[value])
    cols = [colrender(*i) for i in vars(model).items() if iscol(*i)]
    values = {'table': model.__name__, 'columns': ', '.join(cols)}
    return schema.format(**values)

class Database(object):

    def __init__(self, *args, **kwargs):
        self.args = args
        self.kwargs = kwargs
        self.connected = False
        self.Model = type('Model%s' % str(self), (Model,), {'db': self})

    @property
    def connection(self):
        if self.connected:
            return self._connection
        self._connection = sqlite3.connect(*self.args, **self.kwargs)
        self._connection.row_factory = sqlite3.Row
        self.connected = True
        return self._connection

    def close(self):
        if self.connected:
            self.connection.close()
        self.connected = False

    def commit(self):
        self.connection.commit()

    def execute(self, sql, *args):
        return self.connection.execute(sql, args)

    def executescript(self, script):
        self.connection.cursor().executescript(script)
        self.commit()

class Manager(object):

    def __init__(self, db, model):
        self.db = db
        self.model = model
        self.tablename = model.__name__
        if not self._hastable():
            self.db.executescript(render_schema(self.model))

    def all(self):
        cursor = self.db.execute('select * from %s' % self.tablename)
        return (self.create(**row) for row in cursor.fetchall())

    def create(self, **kwargs):
        obj = object.__new__(self.model)
        obj.__dict__ = kwargs
        return obj

    def delete(self, obj):
        sql = 'DELETE from %s WHERE id = ?'
        self.db.execute(sql % self.tablename, obj.id)

    def get(self, id):
        sql = 'select * from %s where id = ?' % self.tablename
        cursor = self.db.execute(sql, id)
        row = cursor.fetchone()
        if not row:
            msg = 'Object%s with id does not exist: %s' % (self.model, id)
            raise ValueError(msg)
        return self.create(**row)

    def has(self, id):
        sql = 'select id from %s where id = ?' % self.tablename
        cursor = self.db.execute(sql, id)
        return True if cursor.fetchall() else False

    def save(self, obj):
        if hasattr(obj, 'id') and self.has(obj.id):
            msg = 'Object%s id already registred: %s' % (self.model, obj.id)
            raise ValueError(msg)
        copy_ = cut_attrs(obj, 'id')
        keys = '(%s)' % ', '.join(copy_.keys())  # (key1, ...)
        refs = '(%s)' % ', '.join('?' for i in range(len(copy_)))  # (?, ...)
        sql = 'insert into %s %s values %s' % (self.tablename, keys, refs)
        cursor = self.db.execute(sql, *copy_.values())
        obj.id = cursor.lastrowid
        return obj

    def update(self, obj):
        copy_ = cut_attrs(obj, 'id')
        keys = '= ?, '.join(copy_.keys()) + '= ?'  # key1 = ?, ...
        sql = 'UPDATE %s SET %s WHERE id = ?' % (self.tablename, keys)
        self.db.execute(sql, *(list(copy_.values()) + [obj.id]))

    def _hastable(self):
        sql = 'select name len FROM sqlite_master where type = ? AND name = ?'
        cursor = self.db.execute(sql, 'table', self.tablename)
        return True if cursor.fetchall() else False

class Model(object):

    db = None

    def delete(self):
        return self.__class__.manager().delete(self)

    def save(self):
        return self.__class__.manager().save(self)

    def update(self):
        return self.__class__.manager().update(self)

    @property
    def public(self):
        return dict(i for i in vars(self).items() if i[0][0] is not '_')

    def __repr__(self):
        return str(self.public)

    @classmethod
    def manager(cls, db=None):
        return Manager(db if db else cls.db, cls)
<|endoftext|>


<commit_before>import GeoIP
import urllib2

gi = GeoIP.open("data/GeoLiteCity.dat",GeoIP.GEOIP_STANDARD)

from flask import Flask
app = Flask(__name__)

@app.route("/helloworld")
def hello():
  return "Hello World!"

@app.route("/ip/<ip>/geo/maxmind.json")
def maxmindgeoip(ip):
  gir = gi.record_by_addr(ip)
  return gir

@app.route("/ip/<ip/geo/ipinfo.json")
def geoip_ipinfo(ip):
  # Use http://ipinfo.io/8.8.4.4
  f = urllib2.urlopen('http://www.python.org/')
  return f.read()

if __name__ == "__main__":
    app.run()
<commit_msg>Got Maxmind and IPInfo services working<commit_after>import GeoIP
import urllib2

gi = GeoIP.open("data/GeoLiteCity.dat",GeoIP.GEOIP_STANDARD)

from flask import Flask
app = Flask(__name__)

@app.route("/helloworld")
def hello():
  return "Hello World!"

@app.route("/ip/<ip>/geo/maxmind.json")
def maxmindgeoip(ip):

  gir = gi.record_by_addr(ip)
  print gir
  return '%s' % gir

@app.route("/ip/<ip>/geo/ipinfo.json")
def geoip_ipinfo(ip):
  f = urllib2.urlopen('http://ipinfo.io/%s/json' % (ip))
  return '%s' % f.read()

if __name__ == "__main__":
    app.run()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8
# original version from https://gist.github.com/brantfaircloth/893580
"""
gb_to_bed.py file.gb file.bed genome_name

grab the gene records from a genbank file (edit for other record types).

- requires:  biopython

"""

from Bio import SeqIO
from pprint import pprint

import sys
import pdb

def main():
    outf = open(sys.argv[2], 'w')
    genome = sys.argv[3]
    for record in SeqIO.parse(open(sys.argv[1], "rU"), "genbank") :
        for feature in record.features:
            if feature.type == 'gene':
                counter = 0;
                for part in feature.location.parts:
                    start = part.start.position
                    stop = part.end.position
                    try:
                        name = feature.qualifiers['gene'][0] + "_" + str(counter)
                        counter += 1
                    except:
                        # some features only have a locus tag
                        name = feature.qualifiers['locus_tag'][0] + "_" + str(counter)
                        counter += 1
                        if feature.strand < 0:
                            tmp = start
                            start = stop
                            stop = tmp
                            bed_line = "{0}_gi\t{1}\t{2}\t{0}_{3}\n".format(genome, start, stop, name)
                            sys.stdout.write("\"{0}_{3}\":{{\"karyo\":\"{0}_gi\",\"start\":{1},\"end\":{2},\"group\":\"gen\"}},".format(genome, start, stop, name));
                            outf.write(bed_line)
    outf.close()

if __name__ == '__main__':
    main()
<commit_msg>fixed bug in gb_to_bed_json.py that led to output of locus_tag features only (no genes)<commit_after>#!/usr/bin/env python
# encoding: utf-8
# original version from https://gist.github.com/brantfaircloth/893580
"""
gb_to_bed.py file.gb file.bed genome_name

grab the gene records from a genbank file (edit for other record types).

- requires:  biopython

"""

from Bio import SeqIO
from pprint import pprint

import sys
import pdb

def main():
    outf = open(sys.argv[2], 'w')
    genome = sys.argv[3]
    for record in SeqIO.parse(open(sys.argv[1], "rU"), "genbank") :
        for feature in record.features:
            if feature.type == 'gene':
                counter = 0;
                for part in feature.location.parts:
                    start = part.start.position
                    stop = part.end.position
                    try:
                        name = feature.qualifiers['gene'][0] + "_" + str(counter)
                    except:
                        # some features only have a locus tag
                        name = feature.qualifiers['locus_tag'][0] + "_" + str(counter)
                    counter += 1
                    if feature.strand < 0:
                        tmp = start
                        start = stop
                        stop = tmp
                    bed_line = "{0}_gi\t{1}\t{2}\t{0}_{3}\n".format(genome, start, stop, name)
                    sys.stdout.write("\"{0}_{3}\":{{\"karyo\":\"{0}_gi\",\"start\":{1},\"end\":{2},\"group\":\"gen\"}},".format(genome, start, stop, name));
                    outf.write(bed_line)
    outf.close()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># vim:ts=4:sw=4:noexpandtab

import itertools
import re
import random

from . import storage
from .core import command

class SameName(ValueError):
	pass

class AlreadyLinked(ValueError):
	pass

class Karma(storage.SelectableStorage):
	@classmethod
	def initialize(cls):
		cls.store = cls.from_URI()
		cls._finalizers.append(cls.finalize)

	@classmethod
	def finalize(cls):
		del cls.store

	def link(self, thing1, thing2):
		"""
		Link thing1 and thing2, adding the karma of each into
		a single entry.
		If any thing does not exist, it is created.
		"""
		thing1 = thing1.strip().lower()
		thing2 = thing2.strip().lower()
		if thing1 == thing2:
			raise SameName("Attempted to link two of the same name")
		self.change(thing1, 0)
		self.change(thing2, 0)
		return self._link(thing1, thing2)

class SQLiteKarma(Karma, storage.SQLiteStorage):
	def init_tables(self):
		CREATE_KARMA_VALUES_TABLE = '''
			CREATE TABLE IF NOT EXISTS karma_values (
				karmaid INTEGER NOT NULL,
				karmavalue INTEGER,
				primary key (karmaid)
			)
		'''
		CREATE_KARMA_KEYS_TABLE = '''
			CREATE TABLE IF NOT EXISTS karma_keys (
				karmakey varchar,
				karmaid INTEGER,
				primary key (karmakey)
			)
		'''
		CREATE_KARMA_LOG_TABLE = '''
			CREATE TABLE IF NOT EXISTS karma_log (
				karmakey varchar,
				logid INTEGER,
				change INTEGER
			)
		'''
		self.db.execute(CREATE_KARMA_VALUES_TABLE)
		self.db.execute(CREATE_KARMA_KEYS_TABLE)
		self.db.execute(CREATE_KARMA_LOG_TABLE)
		self.db.commit()

	def lookup(self, thing):
		thing = thing.strip().lower()
		LOOKUP_SQL = 'SELECT karmavalue from karma_keys k join karma_values v on k.karmaid = v.karmaid where k.karmakey = ?'
		try:
			karma = self.db.execute(LOOKUP_SQL, [thing]).fetchone()[0]
		except:
			karma = 0
		if karma is None:
			karma = 0
		return karma

	def set(self, thing, value):
		thing = thing.strip().lower()
		value = int(value)
		UPDATE_SQL = 'UPDATE karma_values SET karmavalue = ? where karmaid = (select karmaid from karma_keys where karmakey = ?)'
		res = self.db.execute(UPDATE_SQL, (value, thing))
		if res.rowcount == 0:
			INSERT_VALUE_SQL = 'INSERT INTO karma_values (karmavalue) VALUES (?)'
			INSERT_KEY_SQL = 'INSERT INTO karma_keys (karmakey, karmaid) VALUES (?, ?)'
			ins = self.db.execute(INSERT_VALUE_SQL, [value])
			self.db.execute(INSERT_KEY_SQL, (thing, ins.lastrowid))
		self.db.commit()

	def change(self, thing, change):
		thing = thing.strip().lower()
		value = int(self.lookup(thing)) + int(change)
		UPDATE_SQL = 'UPDATE karma_values SET karmavalue = ? where karmaid = (select karmaid from karma_keys where karmakey = ?)'
		res = self.db.execute(UPDATE_SQL, (value, thing))
		if res.rowcount == 0:
			INSERT_VALUE_SQL = 'INSERT INTO karma_values (karmavalue) VALUES (?)'
			INSERT_KEY_SQL = 'INSERT INTO karma_keys (karmakey, karmaid) VALUES (?, ?)'
			ins = self.db.execute(INSERT_VALUE_SQL, [value])
			self.db.execute(INSERT_KEY_SQL, (thing, ins.lastrowid))
		self.db.commit()

	def list(self, select=0):
		KARMIC_VALUES_SQL = 'SELECT karmaid, karmavalue from karma_values order by karmavalue desc'
		KARMA_KEYS_SQL = 'SELECT karmakey from karma_keys where karmaid = ?'

		karmalist = self.db.execute(KARMIC_VALUES_SQL).fetchall()
		karmalist.sort(key=lambda x: int(x[1]), reverse=True)
		if select > 0:
			selected = karmalist[:select]
		elif select < 0:
			selected = karmalist[select:]
		else:
			selected = karmalist
		keysandkarma = []
		for karmaid, value in selected:
			keys = [x[0] for x in self.db.execute(KARMA_KEYS_SQL, [karmaid])]
			keysandkarma.append((keys, value))
		return keysandkarma

	def _link(self, thing1, thing2):
		GET_KARMAID_SQL = 'SELECT karmaid FROM karma_keys WHERE karmakey = ?'
		t1id = self.db.execute(GET_KARMAID_SQL, [thing1]).fetchone()[0]
		t1value = self.lookup(thing1)
		t2id = self.db.execute(GET_KARMAID_SQL, [thing2]).fetchone()[0]
		if t1id == t2id:
			raise AlreadyLinked("Those two are already linked")
		t2value = self.lookup(thing2)

		newvalue = t1value + t2value
		# update the keys so t2 points to t1s value
		self.db.execute('UPDATE karma_keys SET karmaid = ? where karmaid = ?', (t1id, t2id))
		# drop the old value row for neatness
		self.db.execute('DELETE FROM karma_values WHERE karmaid = ?', (t2id,))
		# set the new combined value
		self.db.execute('UPDATE karma_values SET karmavalue = ? where karmaid = ?', (newvalue, t1id))
		self.db.commit()

	def _get(self, id):
		"Return keys and value for karma id"
		VALUE_SQL = "SELECT karmavalue from karma_values where karmaid = ?"
		KEYS_SQL = "SELECT karmakey from karma_keys where karmaid = ?"
		value = self.db.execute(VALUE_SQL, [id]).fetchall()[0][0]
		keys_cur = self.db.execute(KEYS_SQL, [id]).fetchall()
		keys = sorted(x[0] for x in keys_cur)
		return keys, value

	def search(self, term):
		query = "SELECT distinct karmaid from karma_keys where karmakey like ?"
		matches = (id for (id,) in self.db.execute(query, '%%' + term + '%%'))
		return (self._lookup(id) for id in matches)

	def export_all(self):
		return self.list()

class MongoDBKarma(Karma, storage.MongoDBStorage):
	collection_name = 'karma'

	def lookup(self, thing):
		thing = thing.strip().lower()
		res = self.db.find_one({'names': thing})
		return res['value'] if res else 0

	def set(self, thing, value):
		thing = thing.strip().lower()
		value = int(value)
		query = {'names': {'$in': [thing]}}
		oper = {'$set': {'value': value}, '$addToSet': {'names': thing}}
		self.db.update(query, oper, upsert=True)

	def change(self, thing, change):
		thing = thing.strip().lower()
		change = int(change)
		query = {'names': {'$in': [thing]}}
		oper = {'$inc': {'value': change}, '$addToSet': {'names': thing}}
		self.db.update(query, oper, upsert=True)

	def list(self, select=0):
		res = list(self.db.find().sort('value', storage.pymongo.DESCENDING))

		if select > 0:
			selected = res[:select]
		elif select < 0:
			selected = res[select:]
		else:
			selected = res
		aslist = lambda val: val if isinstance(val, list) else [val]
		return [
			(aslist(rec['names']), rec['value'])
			for rec in selected
		]

	def _link(self, thing1, thing2):
		rec = self.db.find_one({'names': thing2})
		if thing1 in rec['names']:
			raise AlreadyLinked("Those two are already linked")
		query = {'names': thing1}
		update = {
			'$inc': {'value': rec['value']},
			'$pushAll': {'names': rec['names']},
		}
		self.db.update(query, update)
		self.db.remove(rec)

	def search(self, term):
		pattern = re.compile('.*' + re.escape(term) + '.*')
		return (
			(rec['names'], rec['value'])
			for rec in self.db.find({'names': pattern})
		)

	def import_(self, item):
		names, value = item
		self.db.insert(dict(
			names=names,
			value=value,
		))

	def _all_names(self):
		return set(itertools.chain.from_iterable(
			names
			for names, value in self.search('')
		))

	def repair_duplicate_names(self):
		"""
		Prior to 1101.1.1, pmxbot would incorrectly create new karma records
		for individuals with multiple names.
		This routine corrects those records.
		"""
		for name in self._all_names():
			cur = self.db.find({'names': name})
			main_doc = next(cur)
			for duplicate in cur:
				query = {'_id': main_doc['_id']}
				update = {
					'$inc': {'value': duplicate['value']},
					'$pushAll': {'names': duplicate['names']},
				}
				self.db.update(query, update)
				self.db.remove(duplicate)

@command(aliases="k")
def karma(nick, rest):
	"Return or change the karma value for some(one|thing)"
	karmee = rest.strip('++').strip('--').strip('~~')
	if '++' in rest:
		Karma.store.change(karmee, 1)
	elif '--' in rest:
		Karma.store.change(karmee, -1)
	elif '~~' in rest:
		change = random.choice([-1, 0, 1])
		Karma.store.change(karmee, change)
		if change == 1:
			return "%s karma++" % karmee
		elif change == 0:
			return "%s karma shall remain the same" % karmee
		elif change == -1:
			return "%s karma--" % karmee
	elif '==' in rest:
		t1, t2 = rest.split('==')
		try:
			Karma.store.link(t1, t2)
		except SameName:
			Karma.store.change(nick, -1)
			return "Don't try to link a name to itself!"
		except AlreadyLinked:
			return "Those names were previously linked."
		score = Karma.store.lookup(t1)
		return "%s and %s are now linked and have a score of %s" % (t1, t2, score)
	else:
		karmee = rest or nick
		score = Karma.store.lookup(karmee)
		return "%s has %s karmas" % (karmee, score)

@command("top10", aliases=("top",))
def top10(rest):
	"""
	Return the top n (default 10) highest entities by Karmic value.
	Use negative numbers for the bottom N.
	"""
	if rest:
		topn = int(rest)
	else:
		topn = 10
	selection = Karma.store.list(topn)
	res = ' '.join('(%s: %s)' % (', '.join(n), k) for n, k in selection)
	return res

@command(aliases=("bottom",))
def bottom10(rest):
	"""
	Return the bottom n (default 10) lowest entities by Karmic value.
	Use negative numbers for the bottom N.
	"""
	if rest:
		topn = -int(rest)
	else:
		topn = -10
	selection = Karma.store.list(topn)
	res = ' '.join('(%s: %s)' % (', '.join(n), k) for n, k in selection)
	return res
<commit_msg>Add workaround for #52. I copied the code rather than requiring it to avoid adding a conditional runtime dependency (as pymongo is an implicit dependency, I didn't want to add another implicit dependency coupled with pymongo).<commit_after># vim:ts=4:sw=4:noexpandtab

import itertools
import re
import random

from . import storage
from .core import command

class SameName(ValueError):
	pass

class AlreadyLinked(ValueError):
	pass

class Karma(storage.SelectableStorage):
	@classmethod
	def initialize(cls):
		cls.store = cls.from_URI()
		cls._finalizers.append(cls.finalize)

	@classmethod
	def finalize(cls):
		del cls.store

	def link(self, thing1, thing2):
		"""
		Link thing1 and thing2, adding the karma of each into
		a single entry.
		If any thing does not exist, it is created.
		"""
		thing1 = thing1.strip().lower()
		thing2 = thing2.strip().lower()
		if thing1 == thing2:
			raise SameName("Attempted to link two of the same name")
		self.change(thing1, 0)
		self.change(thing2, 0)
		return self._link(thing1, thing2)

class SQLiteKarma(Karma, storage.SQLiteStorage):
	def init_tables(self):
		CREATE_KARMA_VALUES_TABLE = '''
			CREATE TABLE IF NOT EXISTS karma_values (
				karmaid INTEGER NOT NULL,
				karmavalue INTEGER,
				primary key (karmaid)
			)
		'''
		CREATE_KARMA_KEYS_TABLE = '''
			CREATE TABLE IF NOT EXISTS karma_keys (
				karmakey varchar,
				karmaid INTEGER,
				primary key (karmakey)
			)
		'''
		CREATE_KARMA_LOG_TABLE = '''
			CREATE TABLE IF NOT EXISTS karma_log (
				karmakey varchar,
				logid INTEGER,
				change INTEGER
			)
		'''
		self.db.execute(CREATE_KARMA_VALUES_TABLE)
		self.db.execute(CREATE_KARMA_KEYS_TABLE)
		self.db.execute(CREATE_KARMA_LOG_TABLE)
		self.db.commit()

	def lookup(self, thing):
		thing = thing.strip().lower()
		LOOKUP_SQL = 'SELECT karmavalue from karma_keys k join karma_values v on k.karmaid = v.karmaid where k.karmakey = ?'
		try:
			karma = self.db.execute(LOOKUP_SQL, [thing]).fetchone()[0]
		except:
			karma = 0
		if karma is None:
			karma = 0
		return karma

	def set(self, thing, value):
		thing = thing.strip().lower()
		value = int(value)
		UPDATE_SQL = 'UPDATE karma_values SET karmavalue = ? where karmaid = (select karmaid from karma_keys where karmakey = ?)'
		res = self.db.execute(UPDATE_SQL, (value, thing))
		if res.rowcount == 0:
			INSERT_VALUE_SQL = 'INSERT INTO karma_values (karmavalue) VALUES (?)'
			INSERT_KEY_SQL = 'INSERT INTO karma_keys (karmakey, karmaid) VALUES (?, ?)'
			ins = self.db.execute(INSERT_VALUE_SQL, [value])
			self.db.execute(INSERT_KEY_SQL, (thing, ins.lastrowid))
		self.db.commit()

	def change(self, thing, change):
		thing = thing.strip().lower()
		value = int(self.lookup(thing)) + int(change)
		UPDATE_SQL = 'UPDATE karma_values SET karmavalue = ? where karmaid = (select karmaid from karma_keys where karmakey = ?)'
		res = self.db.execute(UPDATE_SQL, (value, thing))
		if res.rowcount == 0:
			INSERT_VALUE_SQL = 'INSERT INTO karma_values (karmavalue) VALUES (?)'
			INSERT_KEY_SQL = 'INSERT INTO karma_keys (karmakey, karmaid) VALUES (?, ?)'
			ins = self.db.execute(INSERT_VALUE_SQL, [value])
			self.db.execute(INSERT_KEY_SQL, (thing, ins.lastrowid))
		self.db.commit()

	def list(self, select=0):
		KARMIC_VALUES_SQL = 'SELECT karmaid, karmavalue from karma_values order by karmavalue desc'
		KARMA_KEYS_SQL = 'SELECT karmakey from karma_keys where karmaid = ?'

		karmalist = self.db.execute(KARMIC_VALUES_SQL).fetchall()
		karmalist.sort(key=lambda x: int(x[1]), reverse=True)
		if select > 0:
			selected = karmalist[:select]
		elif select < 0:
			selected = karmalist[select:]
		else:
			selected = karmalist
		keysandkarma = []
		for karmaid, value in selected:
			keys = [x[0] for x in self.db.execute(KARMA_KEYS_SQL, [karmaid])]
			keysandkarma.append((keys, value))
		return keysandkarma

	def _link(self, thing1, thing2):
		GET_KARMAID_SQL = 'SELECT karmaid FROM karma_keys WHERE karmakey = ?'
		t1id = self.db.execute(GET_KARMAID_SQL, [thing1]).fetchone()[0]
		t1value = self.lookup(thing1)
		t2id = self.db.execute(GET_KARMAID_SQL, [thing2]).fetchone()[0]
		if t1id == t2id:
			raise AlreadyLinked("Those two are already linked")
		t2value = self.lookup(thing2)

		newvalue = t1value + t2value
		# update the keys so t2 points to t1s value
		self.db.execute('UPDATE karma_keys SET karmaid = ? where karmaid = ?', (t1id, t2id))
		# drop the old value row for neatness
		self.db.execute('DELETE FROM karma_values WHERE karmaid = ?', (t2id,))
		# set the new combined value
		self.db.execute('UPDATE karma_values SET karmavalue = ? where karmaid = ?', (newvalue, t1id))
		self.db.commit()

	def _get(self, id):
		"Return keys and value for karma id"
		VALUE_SQL = "SELECT karmavalue from karma_values where karmaid = ?"
		KEYS_SQL = "SELECT karmakey from karma_keys where karmaid = ?"
		value = self.db.execute(VALUE_SQL, [id]).fetchall()[0][0]
		keys_cur = self.db.execute(KEYS_SQL, [id]).fetchall()
		keys = sorted(x[0] for x in keys_cur)
		return keys, value

	def search(self, term):
		query = "SELECT distinct karmaid from karma_keys where karmakey like ?"
		matches = (id for (id,) in self.db.execute(query, '%%' + term + '%%'))
		return (self._lookup(id) for id in matches)

	def export_all(self):
		return self.list()

# copied from jaraco.mongodb 5.6
def safe_upsert_27707(coll, query, op):
	item = op.pop('$addToSet')
	exists = bool(coll.find_one(query))
	if not exists:
		init = {field: [value] for field, value in item.items()}
		coll.insert(init)
	coll.update(query, op)

class MongoDBKarma(Karma, storage.MongoDBStorage):
	collection_name = 'karma'

	def lookup(self, thing):
		thing = thing.strip().lower()
		res = self.db.find_one({'names': thing})
		return res['value'] if res else 0

	def set(self, thing, value):
		thing = thing.strip().lower()
		value = int(value)
		query = {'names': {'$in': [thing]}}
		oper = {'$set': {'value': value}, '$addToSet': {'names': thing}}
		safe_upsert_27707(self.db, query, oper)

	def change(self, thing, change):
		thing = thing.strip().lower()
		change = int(change)
		query = {'names': {'$in': [thing]}}
		oper = {'$inc': {'value': change}, '$addToSet': {'names': thing}}
		safe_upsert_27707(self.db, query, oper)

	def list(self, select=0):
		res = list(self.db.find().sort('value', storage.pymongo.DESCENDING))

		if select > 0:
			selected = res[:select]
		elif select < 0:
			selected = res[select:]
		else:
			selected = res
		aslist = lambda val: val if isinstance(val, list) else [val]
		return [
			(aslist(rec['names']), rec['value'])
			for rec in selected
		]

	def _link(self, thing1, thing2):
		rec = self.db.find_one({'names': thing2})
		if thing1 in rec['names']:
			raise AlreadyLinked("Those two are already linked")
		query = {'names': thing1}
		update = {
			'$inc': {'value': rec['value']},
			'$pushAll': {'names': rec['names']},
		}
		self.db.update(query, update)
		self.db.remove(rec)

	def search(self, term):
		pattern = re.compile('.*' + re.escape(term) + '.*')
		return (
			(rec['names'], rec['value'])
			for rec in self.db.find({'names': pattern})
		)

	def import_(self, item):
		names, value = item
		self.db.insert(dict(
			names=names,
			value=value,
		))

	def _all_names(self):
		return set(itertools.chain.from_iterable(
			names
			for names, value in self.search('')
		))

	def repair_duplicate_names(self):
		"""
		Prior to 1101.1.1, pmxbot would incorrectly create new karma records
		for individuals with multiple names.
		This routine corrects those records.
		"""
		for name in self._all_names():
			cur = self.db.find({'names': name})
			main_doc = next(cur)
			for duplicate in cur:
				query = {'_id': main_doc['_id']}
				update = {
					'$inc': {'value': duplicate['value']},
					'$pushAll': {'names': duplicate['names']},
				}
				self.db.update(query, update)
				self.db.remove(duplicate)

@command(aliases="k")
def karma(nick, rest):
	"Return or change the karma value for some(one|thing)"
	karmee = rest.strip('++').strip('--').strip('~~')
	if '++' in rest:
		Karma.store.change(karmee, 1)
	elif '--' in rest:
		Karma.store.change(karmee, -1)
	elif '~~' in rest:
		change = random.choice([-1, 0, 1])
		Karma.store.change(karmee, change)
		if change == 1:
			return "%s karma++" % karmee
		elif change == 0:
			return "%s karma shall remain the same" % karmee
		elif change == -1:
			return "%s karma--" % karmee
	elif '==' in rest:
		t1, t2 = rest.split('==')
		try:
			Karma.store.link(t1, t2)
		except SameName:
			Karma.store.change(nick, -1)
			return "Don't try to link a name to itself!"
		except AlreadyLinked:
			return "Those names were previously linked."
		score = Karma.store.lookup(t1)
		return "%s and %s are now linked and have a score of %s" % (t1, t2, score)
	else:
		karmee = rest or nick
		score = Karma.store.lookup(karmee)
		return "%s has %s karmas" % (karmee, score)

@command("top10", aliases=("top",))
def top10(rest):
	"""
	Return the top n (default 10) highest entities by Karmic value.
	Use negative numbers for the bottom N.
	"""
	if rest:
		topn = int(rest)
	else:
		topn = 10
	selection = Karma.store.list(topn)
	res = ' '.join('(%s: %s)' % (', '.join(n), k) for n, k in selection)
	return res

@command(aliases=("bottom",))
def bottom10(rest):
	"""
	Return the bottom n (default 10) lowest entities by Karmic value.
	Use negative numbers for the bottom N.
	"""
	if rest:
		topn = -int(rest)
	else:
		topn = -10
	selection = Karma.store.list(topn)
	res = ' '.join('(%s: %s)' % (', '.join(n), k) for n, k in selection)
	return res
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright 2014,  Digital Reasoning
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# Need this so the stackdio import below doesn't get confused
from __future__ import absolute_import

from django import template
from django.conf import settings

from stackdio.server import __version__

register = template.Library()

@register.simple_tag
def stackdio_version():
    return '{0}'.format(__version__)

@register.simple_tag
def viewmodel(viewmodel):
    require = '{0}stackdio/lib/bower_components/requirejs/require.js'.format(settings.STATIC_URL)
    if settings.DEBUG:
        app = '/js/main/{0}'.format(viewmodel)
        return ''.format(app, require)
    else:
        app = '{0}stackdio/build/{1}.js'.format(settings.STATIC_URL, viewmodel)
        return ''.format(app, require)
<commit_msg>Refactored a stupid mistake<commit_after># -*- coding: utf-8 -*-

# Copyright 2014,  Digital Reasoning
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# Need this so the stackdio import below doesn't get confused
from __future__ import absolute_import

from django import template
from django.conf import settings

from stackdio.server import __version__

register = template.Library()

@register.simple_tag
def stackdio_version():
    return '{0}'.format(__version__)

@register.simple_tag
def viewmodel(viewmodel):
    require = '{0}stackdio/lib/bower_components/requirejs/require.js'.format(settings.STATIC_URL)
    if settings.DEBUG:
        app = '/js/main/{0}'.format(viewmodel)
    else:
        app = '{0}stackdio/build/{1}.js'.format(settings.STATIC_URL, viewmodel)

    return ''.format(app, require)
<|endoftext|>


<commit_before>from __future__ import absolute_import, print_function, division

import urwid

import netlib.http.url
from mitmproxy.console import common
from mitmproxy.console import signals
from mitmproxy.flow import export

def _mkhelp():
    text = []
    keys = [
        ("A", "accept all intercepted flows"),
        ("a", "accept this intercepted flow"),
        ("b", "save request/response body"),
        ("C", "export flow to clipboard"),
        ("d", "delete flow"),
        ("D", "duplicate flow"),
        ("e", "toggle eventlog"),
        ("F", "toggle follow flow list"),
        ("l", "set limit filter pattern"),
        ("L", "load saved flows"),
        ("m", "toggle flow mark"),
        ("M", "toggle marked flow view"),
        ("n", "create a new request"),
        ("E", "export flow to file"),
        ("r", "replay request"),
        ("U", "unmark all marked flows"),
        ("V", "revert changes to request"),
        ("w", "save flows "),
        ("W", "stream flows to file"),
        ("X", "kill and delete flow, even if it's mid-intercept"),
        ("z", "clear flow list or eventlog"),
        ("tab", "tab between eventlog and flow list"),
        ("enter", "view flow"),
        ("|", "run script on this flow"),
    ]
    text.extend(common.format_keyvals(keys, key="key", val="text", indent=4))
    return text
help_context = _mkhelp()

footer = [
    ('heading_key', "?"), ":help ",
]

class LogBufferBox(urwid.ListBox):

    def __init__(self, master):
        self.master = master
        urwid.ListBox.__init__(self, master.logbuffer)

    def keypress(self, size, key):
        key = common.shortcuts(key)
        if key == "z":
            self.master.clear_events()
            key = None
        elif key == "G":
            self.set_focus(len(self.master.logbuffer) - 1)
        elif key == "g":
            self.set_focus(0)
        return urwid.ListBox.keypress(self, size, key)

class BodyPile(urwid.Pile):

    def __init__(self, master):
        h = urwid.Text("Event log")
        h = urwid.Padding(h, align="left", width=("relative", 100))

        self.inactive_header = urwid.AttrWrap(h, "heading_inactive")
        self.active_header = urwid.AttrWrap(h, "heading")

        urwid.Pile.__init__(
            self,
            [
                FlowListBox(master),
                urwid.Frame(
                    LogBufferBox(master),
                    header = self.inactive_header
                )
            ]
        )
        self.master = master

    def keypress(self, size, key):
        if key == "tab":
            self.focus_position = (
                self.focus_position + 1) % len(self.widget_list)
            if self.focus_position == 1:
                self.widget_list[1].header = self.active_header
            else:
                self.widget_list[1].header = self.inactive_header
            key = None
        elif key == "e":
            self.master.toggle_eventlog()
            key = None

        # This is essentially a copypasta from urwid.Pile's keypress handler.
        # So much for "closed for modification, but open for extension".
        item_rows = None
        if len(size) == 2:
            item_rows = self.get_item_rows(size, focus = True)
        i = self.widget_list.index(self.focus_item)
        tsize = self.get_item_size(size, i, True, item_rows)
        return self.focus_item.keypress(tsize, key)

class ConnectionItem(urwid.WidgetWrap):

    def __init__(self, master, state, flow, focus):
        self.master, self.state, self.flow = master, state, flow
        self.f = focus
        w = self.get_text()
        urwid.WidgetWrap.__init__(self, w)

    def get_text(self):
        return common.format_flow(
            self.flow,
            self.f,
            hostheader = self.master.options.showhost,
        )

    def selectable(self):
        return True

    def save_flows_prompt(self, k):
        if k == "a":
            signals.status_prompt_path.send(
                prompt = "Save all flows to",
                callback = self.master.save_flows
            )
        elif k == "m":
            signals.status_prompt_path.send(
                prompt = "Save marked flows to",
                callback = self.master.save_marked_flows
            )
        else:
            signals.status_prompt_path.send(
                prompt = "Save this flow to",
                callback = self.master.save_one_flow,
                args = (self.flow,)
            )

    def stop_server_playback_prompt(self, a):
        if a != "n":
            self.master.stop_server_playback()

    def server_replay_prompt(self, k):
        if k == "a":
            self.master.start_server_playback(
                [i.copy() for i in self.master.state.view],
                self.master.options.kill, self.master.options.rheaders,
                False, self.master.options.nopop,
                self.master.options.replay_ignore_params,
                self.master.options.replay_ignore_content,
                self.master.options.replay_ignore_payload_params,
                self.master.options.replay_ignore_host
            )
        elif k == "t":
            self.master.start_server_playback(
                [self.flow.copy()],
                self.master.options.kill, self.master.options.rheaders,
                False, self.master.options.nopop,
                self.master.options.replay_ignore_params,
                self.master.options.replay_ignore_content,
                self.master.options.replay_ignore_payload_params,
                self.master.options.replay_ignore_host
            )
        else:
            signals.status_prompt_path.send(
                prompt = "Server replay path",
                callback = self.master.server_playback_path
            )

    def mouse_event(self, size, event, button, col, row, focus):
        if event == "mouse press" and button == 1:
            if self.flow.request:
                self.master.view_flow(self.flow)
                return True

    def keypress(self, xxx_todo_changeme, key):
        (maxcol,) = xxx_todo_changeme
        key = common.shortcuts(key)
        if key == "a":
            self.flow.accept_intercept(self.master)
            signals.flowlist_change.send(self)
        elif key == "d":
            self.flow.kill(self.master)
            self.state.delete_flow(self.flow)
            signals.flowlist_change.send(self)
        elif key == "D":
            f = self.master.duplicate_flow(self.flow)
            self.master.state.set_focus_flow(f)
            signals.flowlist_change.send(self)
        elif key == "m":
            self.flow.marked = not self.flow.marked
            signals.flowlist_change.send(self)
        elif key == "M":
            if self.state.mark_filter:
                self.state.disable_marked_filter()
            else:
                self.state.enable_marked_filter()
            signals.flowlist_change.send(self)
        elif key == "r":
            r = self.master.replay_request(self.flow)
            if r:
                signals.status_message.send(message=r)
            signals.flowlist_change.send(self)
        elif key == "S":
            if not self.master.server_playback:
                signals.status_prompt_onekey.send(
                    prompt = "Server Replay",
                    keys = (
                        ("all flows", "a"),
                        ("this flow", "t"),
                        ("file", "f"),
                    ),
                    callback = self.server_replay_prompt,
                )
            else:
                signals.status_prompt_onekey.send(
                    prompt = "Stop current server replay?",
                    keys = (
                        ("yes", "y"),
                        ("no", "n"),
                    ),
                    callback = self.stop_server_playback_prompt,
                )
        elif key == "U":
            for f in self.state.flows:
                self.state.set_flow_marked(f, False)
            signals.flowlist_change.send(self)
        elif key == "V":
            if not self.flow.modified():
                signals.status_message.send(message="Flow not modified.")
                return
            self.state.revert(self.flow)
            signals.flowlist_change.send(self)
            signals.status_message.send(message="Reverted.")
        elif key == "w":
            signals.status_prompt_onekey.send(
                self,
                prompt = "Save",
                keys = (
                    ("all flows", "a"),
                    ("this flow", "t"),
                    ("marked flows", "m"),
                ),
                callback = self.save_flows_prompt,
            )
        elif key == "X":
            self.flow.kill(self.master)
        elif key == "enter":
            if self.flow.request:
                self.master.view_flow(self.flow)
        elif key == "|":
            signals.status_prompt_path.send(
                prompt = "Send flow to script",
                callback = self.master.run_script_once,
                args = (self.flow,)
            )
        elif key == "E":
            signals.status_prompt_onekey.send(
                self,
                prompt = "Export to file",
                keys = [(e[0], e[1]) for e in export.EXPORTERS],
                callback = common.export_to_clip_or_file,
                args = (None, self.flow, common.ask_save_path)
            )
        elif key == "C":
            signals.status_prompt_onekey.send(
                self,
                prompt = "Export to clipboard",
                keys = [(e[0], e[1]) for e in export.EXPORTERS],
                callback = common.export_to_clip_or_file,
                args = (None, self.flow, common.copy_to_clipboard_or_prompt)
            )
        elif key == "b":
            common.ask_save_body(None, self.flow)
        else:
            return key

class FlowListWalker(urwid.ListWalker):

    def __init__(self, master, state):
        self.master, self.state = master, state
        signals.flowlist_change.connect(self.sig_flowlist_change)

    def sig_flowlist_change(self, sender):
        self._modified()

    def get_focus(self):
        f, i = self.state.get_focus()
        f = ConnectionItem(self.master, self.state, f, True) if f else None
        return f, i

    def set_focus(self, focus):
        ret = self.state.set_focus(focus)
        return ret

    def get_next(self, pos):
        f, i = self.state.get_next(pos)
        f = ConnectionItem(self.master, self.state, f, False) if f else None
        return f, i

    def get_prev(self, pos):
        f, i = self.state.get_prev(pos)
        f = ConnectionItem(self.master, self.state, f, False) if f else None
        return f, i

class FlowListBox(urwid.ListBox):

    def __init__(self, master):
        # type: (mitmproxy.console.master.ConsoleMaster) -> None
        self.master = master
        super(FlowListBox, self).__init__(FlowListWalker(master, master.state))

    def get_method_raw(self, k):
        if k:
            self.get_url(k)

    def get_method(self, k):
        if k == "e":
            signals.status_prompt.send(
                self,
                prompt = "Method",
                text = "",
                callback = self.get_method_raw
            )
        else:
            method = ""
            for i in common.METHOD_OPTIONS:
                if i[1] == k:
                    method = i[0].upper()
            self.get_url(method)

    def get_url(self, method):
        signals.status_prompt.send(
            prompt = "URL",
            text = "http://www.example.com/",
            callback = self.new_request,
            args = (method,)
        )

    def new_request(self, url, method):
        parts = netlib.http.url.parse(str(url))
        if not parts:
            signals.status_message.send(message="Invalid Url")
            return
        scheme, host, port, path = parts
        f = self.master.create_request(method, scheme, host, port, path)
        self.master.state.set_focus_flow(f)
        signals.flowlist_change.send(self)

    def keypress(self, size, key):
        key = common.shortcuts(key)
        if key == "A":
            self.master.accept_all()
            signals.flowlist_change.send(self)
        elif key == "z":
            self.master.clear_flows()
        elif key == "e":
            self.master.toggle_eventlog()
        elif key == "g":
            self.master.state.set_focus(0)
            signals.flowlist_change.send(self)
        elif key == "G":
            self.master.state.set_focus(self.master.state.flow_count())
            signals.flowlist_change.send(self)
        elif key == "l":
            signals.status_prompt.send(
                prompt = "Limit",
                text = self.master.state.limit_txt,
                callback = self.master.set_limit
            )
        elif key == "L":
            signals.status_prompt_path.send(
                self,
                prompt = "Load flows",
                callback = self.master.load_flows_callback
            )
        elif key == "n":
            signals.status_prompt_onekey.send(
                prompt = "Method",
                keys = common.METHOD_OPTIONS,
                callback = self.get_method
            )
        elif key == "F":
            self.master.toggle_follow_flows()
        elif key == "W":
            if self.master.options.outfile:
                self.master.options.outfile = None
            else:
                signals.status_prompt_path.send(
                    self,
                    prompt="Stream flows to",
                    callback= lambda path: self.master.options.update(outfile=(path, "ab"))
                )
        else:
            return urwid.ListBox.keypress(self, size, key)
<commit_msg>Update unmark all flows "U"<commit_after>from __future__ import absolute_import, print_function, division

import urwid

import netlib.http.url
from mitmproxy.console import common
from mitmproxy.console import signals
from mitmproxy.flow import export

def _mkhelp():
    text = []
    keys = [
        ("A", "accept all intercepted flows"),
        ("a", "accept this intercepted flow"),
        ("b", "save request/response body"),
        ("C", "export flow to clipboard"),
        ("d", "delete flow"),
        ("D", "duplicate flow"),
        ("e", "toggle eventlog"),
        ("F", "toggle follow flow list"),
        ("l", "set limit filter pattern"),
        ("L", "load saved flows"),
        ("m", "toggle flow mark"),
        ("M", "toggle marked flow view"),
        ("n", "create a new request"),
        ("E", "export flow to file"),
        ("r", "replay request"),
        ("U", "unmark all marked flows"),
        ("V", "revert changes to request"),
        ("w", "save flows "),
        ("W", "stream flows to file"),
        ("X", "kill and delete flow, even if it's mid-intercept"),
        ("z", "clear flow list or eventlog"),
        ("tab", "tab between eventlog and flow list"),
        ("enter", "view flow"),
        ("|", "run script on this flow"),
    ]
    text.extend(common.format_keyvals(keys, key="key", val="text", indent=4))
    return text
help_context = _mkhelp()

footer = [
    ('heading_key', "?"), ":help ",
]

class LogBufferBox(urwid.ListBox):

    def __init__(self, master):
        self.master = master
        urwid.ListBox.__init__(self, master.logbuffer)

    def keypress(self, size, key):
        key = common.shortcuts(key)
        if key == "z":
            self.master.clear_events()
            key = None
        elif key == "G":
            self.set_focus(len(self.master.logbuffer) - 1)
        elif key == "g":
            self.set_focus(0)
        return urwid.ListBox.keypress(self, size, key)

class BodyPile(urwid.Pile):

    def __init__(self, master):
        h = urwid.Text("Event log")
        h = urwid.Padding(h, align="left", width=("relative", 100))

        self.inactive_header = urwid.AttrWrap(h, "heading_inactive")
        self.active_header = urwid.AttrWrap(h, "heading")

        urwid.Pile.__init__(
            self,
            [
                FlowListBox(master),
                urwid.Frame(
                    LogBufferBox(master),
                    header = self.inactive_header
                )
            ]
        )
        self.master = master

    def keypress(self, size, key):
        if key == "tab":
            self.focus_position = (
                self.focus_position + 1) % len(self.widget_list)
            if self.focus_position == 1:
                self.widget_list[1].header = self.active_header
            else:
                self.widget_list[1].header = self.inactive_header
            key = None
        elif key == "e":
            self.master.toggle_eventlog()
            key = None

        # This is essentially a copypasta from urwid.Pile's keypress handler.
        # So much for "closed for modification, but open for extension".
        item_rows = None
        if len(size) == 2:
            item_rows = self.get_item_rows(size, focus = True)
        i = self.widget_list.index(self.focus_item)
        tsize = self.get_item_size(size, i, True, item_rows)
        return self.focus_item.keypress(tsize, key)

class ConnectionItem(urwid.WidgetWrap):

    def __init__(self, master, state, flow, focus):
        self.master, self.state, self.flow = master, state, flow
        self.f = focus
        w = self.get_text()
        urwid.WidgetWrap.__init__(self, w)

    def get_text(self):
        return common.format_flow(
            self.flow,
            self.f,
            hostheader = self.master.options.showhost,
        )

    def selectable(self):
        return True

    def save_flows_prompt(self, k):
        if k == "a":
            signals.status_prompt_path.send(
                prompt = "Save all flows to",
                callback = self.master.save_flows
            )
        elif k == "m":
            signals.status_prompt_path.send(
                prompt = "Save marked flows to",
                callback = self.master.save_marked_flows
            )
        else:
            signals.status_prompt_path.send(
                prompt = "Save this flow to",
                callback = self.master.save_one_flow,
                args = (self.flow,)
            )

    def stop_server_playback_prompt(self, a):
        if a != "n":
            self.master.stop_server_playback()

    def server_replay_prompt(self, k):
        if k == "a":
            self.master.start_server_playback(
                [i.copy() for i in self.master.state.view],
                self.master.options.kill, self.master.options.rheaders,
                False, self.master.options.nopop,
                self.master.options.replay_ignore_params,
                self.master.options.replay_ignore_content,
                self.master.options.replay_ignore_payload_params,
                self.master.options.replay_ignore_host
            )
        elif k == "t":
            self.master.start_server_playback(
                [self.flow.copy()],
                self.master.options.kill, self.master.options.rheaders,
                False, self.master.options.nopop,
                self.master.options.replay_ignore_params,
                self.master.options.replay_ignore_content,
                self.master.options.replay_ignore_payload_params,
                self.master.options.replay_ignore_host
            )
        else:
            signals.status_prompt_path.send(
                prompt = "Server replay path",
                callback = self.master.server_playback_path
            )

    def mouse_event(self, size, event, button, col, row, focus):
        if event == "mouse press" and button == 1:
            if self.flow.request:
                self.master.view_flow(self.flow)
                return True

    def keypress(self, xxx_todo_changeme, key):
        (maxcol,) = xxx_todo_changeme
        key = common.shortcuts(key)
        if key == "a":
            self.flow.accept_intercept(self.master)
            signals.flowlist_change.send(self)
        elif key == "d":
            self.flow.kill(self.master)
            self.state.delete_flow(self.flow)
            signals.flowlist_change.send(self)
        elif key == "D":
            f = self.master.duplicate_flow(self.flow)
            self.master.state.set_focus_flow(f)
            signals.flowlist_change.send(self)
        elif key == "m":
            self.flow.marked = not self.flow.marked
            signals.flowlist_change.send(self)
        elif key == "M":
            if self.state.mark_filter:
                self.state.disable_marked_filter()
            else:
                self.state.enable_marked_filter()
            signals.flowlist_change.send(self)
        elif key == "r":
            r = self.master.replay_request(self.flow)
            if r:
                signals.status_message.send(message=r)
            signals.flowlist_change.send(self)
        elif key == "S":
            if not self.master.server_playback:
                signals.status_prompt_onekey.send(
                    prompt = "Server Replay",
                    keys = (
                        ("all flows", "a"),
                        ("this flow", "t"),
                        ("file", "f"),
                    ),
                    callback = self.server_replay_prompt,
                )
            else:
                signals.status_prompt_onekey.send(
                    prompt = "Stop current server replay?",
                    keys = (
                        ("yes", "y"),
                        ("no", "n"),
                    ),
                    callback = self.stop_server_playback_prompt,
                )
        elif key == "U":
            for f in self.state.flows:
                f.marked = False
            signals.flowlist_change.send(self)
        elif key == "V":
            if not self.flow.modified():
                signals.status_message.send(message="Flow not modified.")
                return
            self.state.revert(self.flow)
            signals.flowlist_change.send(self)
            signals.status_message.send(message="Reverted.")
        elif key == "w":
            signals.status_prompt_onekey.send(
                self,
                prompt = "Save",
                keys = (
                    ("all flows", "a"),
                    ("this flow", "t"),
                    ("marked flows", "m"),
                ),
                callback = self.save_flows_prompt,
            )
        elif key == "X":
            self.flow.kill(self.master)
        elif key == "enter":
            if self.flow.request:
                self.master.view_flow(self.flow)
        elif key == "|":
            signals.status_prompt_path.send(
                prompt = "Send flow to script",
                callback = self.master.run_script_once,
                args = (self.flow,)
            )
        elif key == "E":
            signals.status_prompt_onekey.send(
                self,
                prompt = "Export to file",
                keys = [(e[0], e[1]) for e in export.EXPORTERS],
                callback = common.export_to_clip_or_file,
                args = (None, self.flow, common.ask_save_path)
            )
        elif key == "C":
            signals.status_prompt_onekey.send(
                self,
                prompt = "Export to clipboard",
                keys = [(e[0], e[1]) for e in export.EXPORTERS],
                callback = common.export_to_clip_or_file,
                args = (None, self.flow, common.copy_to_clipboard_or_prompt)
            )
        elif key == "b":
            common.ask_save_body(None, self.flow)
        else:
            return key

class FlowListWalker(urwid.ListWalker):

    def __init__(self, master, state):
        self.master, self.state = master, state
        signals.flowlist_change.connect(self.sig_flowlist_change)

    def sig_flowlist_change(self, sender):
        self._modified()

    def get_focus(self):
        f, i = self.state.get_focus()
        f = ConnectionItem(self.master, self.state, f, True) if f else None
        return f, i

    def set_focus(self, focus):
        ret = self.state.set_focus(focus)
        return ret

    def get_next(self, pos):
        f, i = self.state.get_next(pos)
        f = ConnectionItem(self.master, self.state, f, False) if f else None
        return f, i

    def get_prev(self, pos):
        f, i = self.state.get_prev(pos)
        f = ConnectionItem(self.master, self.state, f, False) if f else None
        return f, i

class FlowListBox(urwid.ListBox):

    def __init__(self, master):
        # type: (mitmproxy.console.master.ConsoleMaster) -> None
        self.master = master
        super(FlowListBox, self).__init__(FlowListWalker(master, master.state))

    def get_method_raw(self, k):
        if k:
            self.get_url(k)

    def get_method(self, k):
        if k == "e":
            signals.status_prompt.send(
                self,
                prompt = "Method",
                text = "",
                callback = self.get_method_raw
            )
        else:
            method = ""
            for i in common.METHOD_OPTIONS:
                if i[1] == k:
                    method = i[0].upper()
            self.get_url(method)

    def get_url(self, method):
        signals.status_prompt.send(
            prompt = "URL",
            text = "http://www.example.com/",
            callback = self.new_request,
            args = (method,)
        )

    def new_request(self, url, method):
        parts = netlib.http.url.parse(str(url))
        if not parts:
            signals.status_message.send(message="Invalid Url")
            return
        scheme, host, port, path = parts
        f = self.master.create_request(method, scheme, host, port, path)
        self.master.state.set_focus_flow(f)
        signals.flowlist_change.send(self)

    def keypress(self, size, key):
        key = common.shortcuts(key)
        if key == "A":
            self.master.accept_all()
            signals.flowlist_change.send(self)
        elif key == "z":
            self.master.clear_flows()
        elif key == "e":
            self.master.toggle_eventlog()
        elif key == "g":
            self.master.state.set_focus(0)
            signals.flowlist_change.send(self)
        elif key == "G":
            self.master.state.set_focus(self.master.state.flow_count())
            signals.flowlist_change.send(self)
        elif key == "l":
            signals.status_prompt.send(
                prompt = "Limit",
                text = self.master.state.limit_txt,
                callback = self.master.set_limit
            )
        elif key == "L":
            signals.status_prompt_path.send(
                self,
                prompt = "Load flows",
                callback = self.master.load_flows_callback
            )
        elif key == "n":
            signals.status_prompt_onekey.send(
                prompt = "Method",
                keys = common.METHOD_OPTIONS,
                callback = self.get_method
            )
        elif key == "F":
            self.master.toggle_follow_flows()
        elif key == "W":
            if self.master.options.outfile:
                self.master.options.outfile = None
            else:
                signals.status_prompt_path.send(
                    self,
                    prompt="Stream flows to",
                    callback= lambda path: self.master.options.update(outfile=(path, "ab"))
                )
        else:
            return urwid.ListBox.keypress(self, size, key)
<|endoftext|>


<commit_before>'Lance solr dans un sous-processus.'
import os
import subprocess

def start():
    solr_args = ['cd apache-solr-3.6.1/example\njava -jar start.jar']
    devnull = open(os.devnull, 'wb')
    print 'Lancement de Solr...'
    return subprocess.Popen(args=solr_args,
                            stdout=devnull, stderr=devnull, shell=True)<commit_msg>Autorise l'exécution directe de Solr par mon script.<commit_after>'Lance solr dans un sous-processus.'
import os
import subprocess

def start():
    solr_args = ['cd apache-solr-3.6.1/example\njava -jar start.jar']
    devnull = open(os.devnull, 'wb')
    print 'Lancement de Solr...'
    return subprocess.Popen(args=solr_args,
                            stdout=devnull, stderr=devnull, shell=True)

if __name__ == '__main__':
    start()
<|endoftext|>