<commit_before>import sys
import os
import re

res_dir = "../src/main/resources/resource"

keyFiles = {}

# Builds a list of all translation keys in the list of lines.
def indexFile(lines):
    allKeys = []
    for line in lines:
        comment = line.find("#")
        if (comment != 0):
            index = line.find("=")
            while ((index > 0) and (line[index-1]=="\\")):
                index = line.find("=", index+1)
            if (index > 0):
                allKeys.append(line[0:index])
    allKeys.sort()
    return allKeys

# Finds all keys in the first list that are not present in the second list:
def findMissingKeys(first, second):
    missing = []
    for key in first:
        if not key in second:
            missing.append(key)
    return missing

# Appends all the given keys to the file:
def appendMissingKeys(filename, keys):
    file = open(filename, "a")
    file.write("\n")
    for key in keys:
        file.write(key+"=\n")

def handleFileSet(mainFile, files, changeFiles):
    f1 = open(mainFile)
    lines = f1.readlines()
    f1.close()
    keys = indexFile(lines)
    keysPresent = []
    for i in range(0, len(files)):
        f2 = open(files[i])
        lines = f2.readlines()
        f2.close()
        keysPresent.append(indexFile(lines))
        missing = findMissingKeys(keys, keysPresent[i])

        print "\n\nFile '"+files[i]+"'\n"
        if len(missing) == 0:
            print "----> No missing keys."
        else:
            print "----> Missing keys:"
            for key in missing:
                print key

            if changeFiles == 1:
                print "Update file?",
                if raw_input() in ['y', 'Y']:
                    appendMissingKeys(files[i], missing)
            print ""

        # See if this file has keys that are not in the main file:
        redundant = findMissingKeys(keysPresent[i], keys)
        if len(redundant) > 0:
            print "----> Possible obsolete keys (not in English language file):"
            for key in redundant:
                print key

            #if changeFiles == 1:
            #   print "Update file?",
            #   if raw_input() in ['y', 'Y']:
            #       removeRedundantKeys(files[i], redundant)
            print ""

def handleJavaCode(filename, lines, keyList, notTermList):
    #Extract first string parameter from Globals.lang call. E.g., Globals.lang("Default")
    reOnlyString = r'"((\\"|[^"])*)"[^"]*'
    patt = re.compile(r'Globals\s*\.\s*lang\s*\(\s*' + reOnlyString)
    #second pattern as Mr Dlib contribution indirectly uses Global.lang
    patta = re.compile(r'LocalizationSupport.message\(' + reOnlyString)
    pattOnlyString = re.compile(reOnlyString)

    #Find multiline Globals lang statements. E.g.:
    #Globals.lang("This is my string" +
    # "with a long text")
    patt2 = re.compile(r'Globals\s*\.\s*lang\s*\(([^)])*$')

    pattPlus = re.compile(r'^\s*\+')

    eList = list(enumerate(lines.split("\n")))
    i = 0
    while i < len(eList):
        linenum, curline = eList[i]

        #Remove Java single line comments
        if curline.find("http://") < 0:
            curline = re.sub("//.*", "", curline)

        while (curline != ""):
            result = patt.search(curline)
            if (not result):
                result = patta.search(curline)
            result2 = patt2.search(curline)

            found = ""

            if result2 and curline.find('",') < 0:
                # not terminated
                # but it could be a multiline string
                if result:
                    curText = result.group(1)
                    searchForPlus = True
                else:
                    curText = ""
                    searchForPlus = False
                origI = i
                #inspect next line
                while i+1 < len(eList):
                    linenum2, curline2 = eList[i+1]
                    if (not searchForPlus) or pattPlus.search(curline2):
                        #from now on, we always have to search for a plus
                        searchForPlus = True

                        #The current line has been handled here, therefore indicate to handle the next line
                        i = i+1
                        linenum = linenum2
                        curline = curline2

                        #Search for the occurence of a string
                        result = pattOnlyString.search(curline2)
                        if result:
                            curText = curText + result.group(1)
                            #check for several strings in this line
                            if curline2.count('\"') > 2:
                                break
                            #check for several arguments in the line
                            if curline2.find('",') > 0:
                                break
                            if curline2.endswith(")"):
                                break
                        else:
                            #plus sign at the beginning found, but no string
                            break
                    else:
                        #no continuation found
                        break

                if origI == i:
                    print "%s:%d: Not terminated: %s"%(filename, linenum+1, curline)
                else:
                    found = curText

            if result or (found != ""):
                if (found == ""):
                    #not a multiline string, found via the single line matching
                    #full string in one line
                    found = result.group(1)

                found = found.replace(" ", "_")
                #replace characters that need to be escaped in the language file
                found = found.replace("=", r"\=").replace(":",r"\:")
                #replace Java-escaped " to plain "
                found = found.replace(r'\"','"')
                #Java-escaped \ to plain \ need not to be converted - they have to be kept
                #e.g., "\\#" has to be contained as "\\#" in the key
                #found = found.replace('\\\\','\\')
                if (found != "") and (found not in keyList):
                    keyList.append(found)
                    keyFiles[found] = (filename, linenum)
                    #print "Adding ", found
                #else:
                #   print "Not adding: "+found

            #Prepare a possible second run (multiple Globals.lang on this line)
            if result:
                lastPos = result.span()[1]
                #regular expression is greedy. It will match until Globals.lang("
                #therefore, we have to adjust lastPos
                lastPos = lastPos - 14
                if len(curline) <= lastPos:
                    curline = ""
                else:
                    curline = curline[lastPos:]
            else:
                #terminate processing of this line, continue to next line
                curline = ""

        i = i+1

# Find all Java source files in the given directory, and read the lines of each,
# calling handleJavaCode on the contents:
def handleDir(lists, dirname, fnames):
    keyList, notTermList = lists
    for file in fnames:
        if len(file) > 6 and file[(len(file)-5):len(file)] == ".java":
            fl = open(dirname+os.sep+file)
            lines = fl.read()
            fl.close()
            handleJavaCode(dirname + os.sep + file, lines, keyList, notTermList)

# Go through subdirectories and call handleDir on all diroctories:
def traverseFileTree(dir):
    keyList = []
    notTermList = []
    os.path.walk(dir, handleDir, (keyList, notTermList))
    print "Keys found: "+str(len(keyList))
    return keyList

# Searches out all translation calls in the Java source files, and reports which
# are not present in the given resource file.
#
# arg: mainFile: a .properties file with the keys to sync with
def findNewKeysInJavaCode(mainFile, dir, update):
    keystempo = []
    keyListtempo = []
    f1 = open(mainFile)
    lines = f1.readlines()
    f1.close()
    keys = indexFile(lines)
    keyList = traverseFileTree(dir)

    # Open the file again, for appending:
    if update:
        f1 = open(mainFile, "a")
        f1.write("\n")

    # Look for keys that are used in the code, but not present in the language file:
    for key in keyList:
        value = key.replace("\\:",":").replace("\\=", "=")
        if key not in keys:
            fileName, lineNum = keyFiles[key]
            print "%s:%i:Missing key: %s"%(fileName, lineNum + 1, value)
            if update:
                f1.write(key+"="+value+"\n")

    # Look for keys in the language file that are not used in the code:
    for key in keys:
        if key not in keyList:
            print "Possible obsolete key: "+key

    if update:
        f1.close()

def lookForDuplicates(file, displayKeys):
    duplicount = 0
    f1 = open(file)
    lines = f1.readlines()
    f1.close()
    mappings = {}
    emptyVals = 0
    for line in lines:
        comment = line.find("#")
        index = line.find("=")
        if (comment != 0) and (index > 0):
            key = line[0:index]
            value = line[index+1:].strip()
            if key in mappings:
                mappings[key].append(value)
                duplicount += 1
                if displayKeys:
		  print "Duplicate: "+file+": "+key+" =",
		  print mappings[key]
            else:
                mappings[key] = [value]
                if value == "":
		    emptyVals = emptyVals + 1
                    if displayKeys:
		      print "Empty value: "+file+": "+key
                #print "New: "+value
    if duplicount > 0:
	dupstring = str(duplicount)+" duplicates. "
    else:
	dupstring = ""
    if emptyVals > 0:
        emptStr = str(emptyVals)+" empty values. "
    else:
	emptStr = ""
    if duplicount+emptyVals > 0:
	okString = ""
    else:
	okString = "ok"
    print file+": "+dupstring+emptStr+okString
    #print file+": "+str(emptyVals)+" empty values."

############# Main part ###################

if len(sys.argv) == 1:
    print """This program must be run from the jabref base directory.

Usage: syncLang.py option
Option can be one of the following:

    -c: Search the language files for empty and duplicate translations. Display only
        counts for duplicated and empty values in each language file.

    -d: Search the language files for empty and duplicate translations.
        For each duplicate set found, a list will be printed showing the various
        translations for the same key. There is currently to option to remove duplicates
        automatically.

    -s [-u]: Search the Java source files for language keys. All keys that are found in the source files
        but not in "JabRef_en.properties" are listed. If the -u option is specified, these keys will
        automatically be added to "JabRef_en.properties".

        The program will also list "Not terminated" keys. These are keys that are concatenated over
        more than one line, that the program is not (currently) able to resolve.

        Finally, the program will list "Possible obsolete keys". These are keys that are present in
        "JabRef_en.properties", but could not be found in the Java source code. Note that the
        "Not terminated" keys will be likely to appear here, since they were not resolved.

    -t [-u]: Compare the contents of "JabRef_en.properties" and "Menu_en.properties" against the other
        language files. The program will list for all the other files which keys from the English
        file are missing. Additionally, the program will list keys in the other files which are
        not present in the English file - possible obsolete keys.

        If the -u option is specified, all missing keys will automatically be added to the files.
        There is currently no option to remove obsolete keys automatically.
"""

elif (len(sys.argv) >= 2) and (sys.argv[1] == "-s"):
    if (len(sys.argv) >= 3) and (sys.argv[2] == "-u"):
        update = 1
    else:
        update = 0
    findNewKeysInJavaCode(os.path.join(res_dir, "JabRef_en.properties"), ".", update)

elif (len(sys.argv) >= 2) and (sys.argv[1] == "-t"):
    if (len(sys.argv) >= 3) and (sys.argv[2] == "-u"):
        changeFiles = 1
    else:
        changeFiles = 0

    filesJabRef = filter(lambda s: (s.startswith('JabRef_') and not (s.startswith('JabRef_en'))), os.listdir(res_dir));
    filesJabRef = [os.path.join(res_dir, i) for i in filesJabRef];
    filesMenu = filter(lambda s: (s.startswith('Menu_') and not (s.startswith('Menu_en'))), os.listdir(res_dir));
    filesMenu = [os.path.join(res_dir, i) for i in filesMenu];

    handleFileSet(os.path.join(res_dir, "JabRef_en.properties"), filesJabRef, changeFiles)
    handleFileSet(os.path.join(res_dir, "Menu_en.properties"), filesMenu, changeFiles)

elif (len(sys.argv) >= 2) and ((sys.argv[1] == "-d") or (sys.argv[1] == "-c")):
    files = filter(lambda s: (s.startswith('JabRef_') and not (s.startswith('JabRef_en'))), os.listdir(res_dir));
    files.extend(filter(lambda s: (s.startswith('Menu_') and not (s.startswith('Menu_en'))), os.listdir(res_dir)));
    files = [os.path.join(res_dir, i) for i in files];
    for file in files:
        lookForDuplicates(file, sys.argv[1] == "-d")
<commit_msg>Update syncLang.py script for new location of localization files.<commit_after>import sys
import os
import re

res_dir = "../src/main/resources/l10n"

keyFiles = {}

# Builds a list of all translation keys in the list of lines.
def indexFile(lines):
    allKeys = []
    for line in lines:
        comment = line.find("#")
        if (comment != 0):
            index = line.find("=")
            while ((index > 0) and (line[index-1]=="\\")):
                index = line.find("=", index+1)
            if (index > 0):
                allKeys.append(line[0:index])
    allKeys.sort()
    return allKeys

# Finds all keys in the first list that are not present in the second list:
def findMissingKeys(first, second):
    missing = []
    for key in first:
        if not key in second:
            missing.append(key)
    return missing

# Appends all the given keys to the file:
def appendMissingKeys(filename, keys):
    file = open(filename, "a")
    file.write("\n")
    for key in keys:
        file.write(key+"=\n")

def handleFileSet(mainFile, files, changeFiles):
    f1 = open(mainFile)
    lines = f1.readlines()
    f1.close()
    keys = indexFile(lines)
    keysPresent = []
    for i in range(0, len(files)):
        f2 = open(files[i])
        lines = f2.readlines()
        f2.close()
        keysPresent.append(indexFile(lines))
        missing = findMissingKeys(keys, keysPresent[i])

        print "\n\nFile '"+files[i]+"'\n"
        if len(missing) == 0:
            print "----> No missing keys."
        else:
            print "----> Missing keys:"
            for key in missing:
                print key

            if changeFiles == 1:
                print "Update file?",
                if raw_input() in ['y', 'Y']:
                    appendMissingKeys(files[i], missing)
            print ""

        # See if this file has keys that are not in the main file:
        redundant = findMissingKeys(keysPresent[i], keys)
        if len(redundant) > 0:
            print "----> Possible obsolete keys (not in English language file):"
            for key in redundant:
                print key

            #if changeFiles == 1:
            #   print "Update file?",
            #   if raw_input() in ['y', 'Y']:
            #       removeRedundantKeys(files[i], redundant)
            print ""

def handleJavaCode(filename, lines, keyList, notTermList):
    #Extract first string parameter from Globals.lang call. E.g., Globals.lang("Default")
    reOnlyString = r'"((\\"|[^"])*)"[^"]*'
    patt = re.compile(r'Globals\s*\.\s*lang\s*\(\s*' + reOnlyString)
    #second pattern as Mr Dlib contribution indirectly uses Global.lang
    patta = re.compile(r'LocalizationSupport.message\(' + reOnlyString)
    pattOnlyString = re.compile(reOnlyString)

    #Find multiline Globals lang statements. E.g.:
    #Globals.lang("This is my string" +
    # "with a long text")
    patt2 = re.compile(r'Globals\s*\.\s*lang\s*\(([^)])*$')

    pattPlus = re.compile(r'^\s*\+')

    eList = list(enumerate(lines.split("\n")))
    i = 0
    while i < len(eList):
        linenum, curline = eList[i]

        #Remove Java single line comments
        if curline.find("http://") < 0:
            curline = re.sub("//.*", "", curline)

        while (curline != ""):
            result = patt.search(curline)
            if (not result):
                result = patta.search(curline)
            result2 = patt2.search(curline)

            found = ""

            if result2 and curline.find('",') < 0:
                # not terminated
                # but it could be a multiline string
                if result:
                    curText = result.group(1)
                    searchForPlus = True
                else:
                    curText = ""
                    searchForPlus = False
                origI = i
                #inspect next line
                while i+1 < len(eList):
                    linenum2, curline2 = eList[i+1]
                    if (not searchForPlus) or pattPlus.search(curline2):
                        #from now on, we always have to search for a plus
                        searchForPlus = True

                        #The current line has been handled here, therefore indicate to handle the next line
                        i = i+1
                        linenum = linenum2
                        curline = curline2

                        #Search for the occurence of a string
                        result = pattOnlyString.search(curline2)
                        if result:
                            curText = curText + result.group(1)
                            #check for several strings in this line
                            if curline2.count('\"') > 2:
                                break
                            #check for several arguments in the line
                            if curline2.find('",') > 0:
                                break
                            if curline2.endswith(")"):
                                break
                        else:
                            #plus sign at the beginning found, but no string
                            break
                    else:
                        #no continuation found
                        break

                if origI == i:
                    print "%s:%d: Not terminated: %s"%(filename, linenum+1, curline)
                else:
                    found = curText

            if result or (found != ""):
                if (found == ""):
                    #not a multiline string, found via the single line matching
                    #full string in one line
                    found = result.group(1)

                found = found.replace(" ", "_")
                #replace characters that need to be escaped in the language file
                found = found.replace("=", r"\=").replace(":",r"\:")
                #replace Java-escaped " to plain "
                found = found.replace(r'\"','"')
                #Java-escaped \ to plain \ need not to be converted - they have to be kept
                #e.g., "\\#" has to be contained as "\\#" in the key
                #found = found.replace('\\\\','\\')
                if (found != "") and (found not in keyList):
                    keyList.append(found)
                    keyFiles[found] = (filename, linenum)
                    #print "Adding ", found
                #else:
                #   print "Not adding: "+found

            #Prepare a possible second run (multiple Globals.lang on this line)
            if result:
                lastPos = result.span()[1]
                #regular expression is greedy. It will match until Globals.lang("
                #therefore, we have to adjust lastPos
                lastPos = lastPos - 14
                if len(curline) <= lastPos:
                    curline = ""
                else:
                    curline = curline[lastPos:]
            else:
                #terminate processing of this line, continue to next line
                curline = ""

        i = i+1

# Find all Java source files in the given directory, and read the lines of each,
# calling handleJavaCode on the contents:
def handleDir(lists, dirname, fnames):
    keyList, notTermList = lists
    for file in fnames:
        if len(file) > 6 and file[(len(file)-5):len(file)] == ".java":
            fl = open(dirname+os.sep+file)
            lines = fl.read()
            fl.close()
            handleJavaCode(dirname + os.sep + file, lines, keyList, notTermList)

# Go through subdirectories and call handleDir on all diroctories:
def traverseFileTree(dir):
    keyList = []
    notTermList = []
    os.path.walk(dir, handleDir, (keyList, notTermList))
    print "Keys found: "+str(len(keyList))
    return keyList

# Searches out all translation calls in the Java source files, and reports which
# are not present in the given resource file.
#
# arg: mainFile: a .properties file with the keys to sync with
def findNewKeysInJavaCode(mainFile, dir, update):
    keystempo = []
    keyListtempo = []
    f1 = open(mainFile)
    lines = f1.readlines()
    f1.close()
    keys = indexFile(lines)
    keyList = traverseFileTree(dir)

    # Open the file again, for appending:
    if update:
        f1 = open(mainFile, "a")
        f1.write("\n")

    # Look for keys that are used in the code, but not present in the language file:
    for key in keyList:
        value = key.replace("\\:",":").replace("\\=", "=")
        if key not in keys:
            fileName, lineNum = keyFiles[key]
            print "%s:%i:Missing key: %s"%(fileName, lineNum + 1, value)
            if update:
                f1.write(key+"="+value+"\n")

    # Look for keys in the language file that are not used in the code:
    for key in keys:
        if key not in keyList:
            print "Possible obsolete key: "+key

    if update:
        f1.close()

def lookForDuplicates(file, displayKeys):
    duplicount = 0
    f1 = open(file)
    lines = f1.readlines()
    f1.close()
    mappings = {}
    emptyVals = 0
    for line in lines:
        comment = line.find("#")
        index = line.find("=")
        if (comment != 0) and (index > 0):
            key = line[0:index]
            value = line[index+1:].strip()
            if key in mappings:
                mappings[key].append(value)
                duplicount += 1
                if displayKeys:
		  print "Duplicate: "+file+": "+key+" =",
		  print mappings[key]
            else:
                mappings[key] = [value]
                if value == "":
		    emptyVals = emptyVals + 1
                    if displayKeys:
		      print "Empty value: "+file+": "+key
                #print "New: "+value
    if duplicount > 0:
	dupstring = str(duplicount)+" duplicates. "
    else:
	dupstring = ""
    if emptyVals > 0:
        emptStr = str(emptyVals)+" empty values. "
    else:
	emptStr = ""
    if duplicount+emptyVals > 0:
	okString = ""
    else:
	okString = "ok"
    print file+": "+dupstring+emptStr+okString
    #print file+": "+str(emptyVals)+" empty values."

############# Main part ###################

if len(sys.argv) == 1:
    print """This program must be run from the jabref base directory.

Usage: syncLang.py option
Option can be one of the following:

    -c: Search the language files for empty and duplicate translations. Display only
        counts for duplicated and empty values in each language file.

    -d: Search the language files for empty and duplicate translations.
        For each duplicate set found, a list will be printed showing the various
        translations for the same key. There is currently to option to remove duplicates
        automatically.

    -s [-u]: Search the Java source files for language keys. All keys that are found in the source files
        but not in "JabRef_en.properties" are listed. If the -u option is specified, these keys will
        automatically be added to "JabRef_en.properties".

        The program will also list "Not terminated" keys. These are keys that are concatenated over
        more than one line, that the program is not (currently) able to resolve.

        Finally, the program will list "Possible obsolete keys". These are keys that are present in
        "JabRef_en.properties", but could not be found in the Java source code. Note that the
        "Not terminated" keys will be likely to appear here, since they were not resolved.

    -t [-u]: Compare the contents of "JabRef_en.properties" and "Menu_en.properties" against the other
        language files. The program will list for all the other files which keys from the English
        file are missing. Additionally, the program will list keys in the other files which are
        not present in the English file - possible obsolete keys.

        If the -u option is specified, all missing keys will automatically be added to the files.
        There is currently no option to remove obsolete keys automatically.
"""

elif (len(sys.argv) >= 2) and (sys.argv[1] == "-s"):
    if (len(sys.argv) >= 3) and (sys.argv[2] == "-u"):
        update = 1
    else:
        update = 0
    findNewKeysInJavaCode(os.path.join(res_dir, "JabRef_en.properties"), ".", update)

elif (len(sys.argv) >= 2) and (sys.argv[1] == "-t"):
    if (len(sys.argv) >= 3) and (sys.argv[2] == "-u"):
        changeFiles = 1
    else:
        changeFiles = 0

    filesJabRef = filter(lambda s: (s.startswith('JabRef_') and not (s.startswith('JabRef_en'))), os.listdir(res_dir));
    filesJabRef = [os.path.join(res_dir, i) for i in filesJabRef];
    filesMenu = filter(lambda s: (s.startswith('Menu_') and not (s.startswith('Menu_en'))), os.listdir(res_dir));
    filesMenu = [os.path.join(res_dir, i) for i in filesMenu];

    handleFileSet(os.path.join(res_dir, "JabRef_en.properties"), filesJabRef, changeFiles)
    handleFileSet(os.path.join(res_dir, "Menu_en.properties"), filesMenu, changeFiles)

elif (len(sys.argv) >= 2) and ((sys.argv[1] == "-d") or (sys.argv[1] == "-c")):
    files = filter(lambda s: (s.startswith('JabRef_') and not (s.startswith('JabRef_en'))), os.listdir(res_dir));
    files.extend(filter(lambda s: (s.startswith('Menu_') and not (s.startswith('Menu_en'))), os.listdir(res_dir)));
    files = [os.path.join(res_dir, i) for i in files];
    for file in files:
        lookForDuplicates(file, sys.argv[1] == "-d")
<|endoftext|>


<commit_before>"""
spaceshooter.py
Author: Marcus Helble
Credit: None

Assignment:
Write and submit a program that implements the spacewar game:
https://github.com/HHS-IntroProgramming/Spacewar
"""
import ggame
from ggame import App, RectangleAsset, ImageAsset, Sprite, LineStyle, Color, Frame
from random import random
SCREEN_WIDTH = 1530
SCREEN_HEIGHT = 930
pi = 3.1415926535897932384626433832795028841971693993751058209749445923078164062862089986280348253421170679
class SpaceShip(Sprite):
    asset = ImageAsset("images/four_spaceship_by_albertov_with_thrust.png",
        Frame(227,0,292-227,125), 4, 'vertical')
    def __init__(self, position):
        super().__init__(SpaceShip.asset, position)
        self.vr = 0.01
        self.thrust = 0
        self.thrustframe = 1
        self.rx = 1
        self.ry = -1
        self.rxa = 0
        self.rxb = 0
        self.rya = 0
        self.ryb = 0
        self.c = 0
        self.d = 0
        self.visible = True

class SpaceGame(App):

    def __init__(self, width, height):
        super().__init__(width, height)
        black = Color(0, 1)
        noline = LineStyle(0, black)
        bg_asset = RectangleAsset(width, height, noline, black)
        bg = Sprite(bg_asset, (0,0))
        SpaceShip((100,100))
        SpaceShip((150,150))
        SpaceShip((200,50))

    def step(self):
        for ship in self.getSpritesbyClass(SpaceShip):
            ship.step()

myapp = SpaceGame(SCREEN_WIDTH, SCREEN_HEIGHT)
myapp.run()<commit_msg>Updated from Brython Server: November 14, 2016 1:40:16 PM EST<commit_after>"""
spaceshooter.py
Author: Marcus Helble
Credit: None

Assignment:
Write and submit a program that implements the spacewar game:
https://github.com/HHS-IntroProgramming/Spacewar
"""
import ggame
from ggame import App, RectangleAsset, ImageAsset, Sprite, LineStyle, Color, Frame
from random import random
SCREEN_WIDTH = 1530
SCREEN_HEIGHT = 930
pi = 3.1415926535897932384626433832795028841971693993751058209749445923078164062862089986280348253421170679
class SpaceShip(Sprite):
    asset = ImageAsset("images/four_spaceship_by_albertov_with_thrust.png",
        Frame(227,0,292-227,125), 4, 'vertical')
    def __init__(self, position):
        super().__init__(SpaceShip.asset, position)
        self.vr = 0.01
        self.thrust = 0
        self.thrustframe = 1
        self.rx = 1
        self.ry = -1
        self.rxa = 0
        self.rxb = 0
        self.rya = 0
        self.ryb = 0
        self.c = 0
        self.d = 0
        self.visible = True
    def __init__(self, position):
        super().__init__(SpaceShip.asset, position)
        self.vx = 1
        self.vy = 1
        self.vr = 0.01
        self.thrust = 0
        self.thrustframe = 1
        SpaceGame.listenKeyEvent("keydown", "space", self.thrustOn)
        SpaceGame.listenKeyEvent("keyup", "space", self.thrustOff)
        self.fxcenter = self.fycenter = 0.5

    def step(self):
        self.x += self.vx
        self.y += self.vy
        self.rotation += self.vr
        if self.thrust == 1:
            self.setImage(self.thrustframe)
            self.thrustframe += 1
            if self.thrustframe == 4:
                self.thrustframe = 1
        else:
            self.setImage(0)

    def thrustOn(self, event):
        self.thrust = 1

    def thrustOff(self, event):
        self.thrust = 0
class SpaceGame(App):

    def __init__(self, width, height):
        super().__init__(width, height)
        black = Color(0, 1)
        noline = LineStyle(0, black)
        bg_asset = RectangleAsset(width, height, noline, black)
        bg = Sprite(bg_asset, (0,0))
        SpaceShip((100,100))
        SpaceShip((150,150))
        SpaceShip((200,50))

    def step(self):
        for ship in self.getSpritesbyClass(SpaceShip):
            ship.step()

myapp = SpaceGame(SCREEN_WIDTH, SCREEN_HEIGHT)
myapp.run()<|endoftext|>


<commit_before>__author__ = "Leif Azzopardi"

import os
import sys

from measures.tar_rulers import TarRuler, TarAggRuler
from seeker.trec_qrel_handler import TrecQrelHandler

def main(results_file, qrel_file):

    qrh = TrecQrelHandler(qrel_file)
    #print(qrh.get_topic_list()) # show what qrel topics have been read in
    #print( len(qrh.get_topic_list())) # show how many

    def get_value_and_check(qrh,seen_dict, topic_id, doc_id):
        # checks to make sure the document is in the qrels and was retrieved by the pubmed query
        v = 0
        if doc_id in seen_dict:
            print("{} Duplicate {}".format(topic_id, doc_id))
            v = None
        else:
            seen_dict[d] = 1
            v = qrh.get_value(topic_id, doc_id)
            if v >= 3:
                v = None
        return v

    curr_topic_id = ""
    seen_dict = {}

    tml = []
    tar_ruler = None
    with open(results_file,"r") as rf:
        while rf:
            line = rf.readline()
            if not line:
                break
            (topic_id,action,doc_id, rank, score, team) = line.split()

            if topic_id == curr_topic_id:
                # accumulate
                #v = qrh.get_value(curr_topic_id, doc_id.strip())
                d = doc_id.strip()
                v = get_value_and_check(qrh, seen_dict, curr_topic_id, d)

                if v is not None:
                    tar_ruler.update(v,v,action)
            else:
                if curr_topic_id is not "":
                    tar_ruler.finalize()
                    tml.append(tar_ruler)
                    tar_ruler.print_scores()
                # new topic
                dl = qrh.get_doc_list(topic_id)
                num_docs = len(dl)
                num_rels = 0
                num_rels_in_set = 0
                num_docs_in_set = num_docs
                for d in dl:
                    val = qrh.get_value(topic_id, d)
                    if val > 0:
                        num_rels = num_rels + 1
                    if val == 1 or val == 2:
                        num_rels_in_set = num_rels_in_set + 1

                    if val == -1 or val > 2:
                        num_docs_in_set = num_docs_in_set - 1

                #print("D: {0} DS: {1} R: {2} RS: {3} ".format(num_docs,num_docs_in_set,num_rels, num_rels_in_set))
                tar_ruler = TarRuler(topic_id,num_docs_in_set, num_rels_in_set)

                # reset seen list
                seen_dict = {}
                d = doc_id.strip()

                v = get_value_and_check(qrh, seen_dict, curr_topic_id, d)
                if v is not None:
                    tar_ruler.update(v,v,action)

                curr_topic_id = topic_id

        tar_ruler.finalize()
        tml.append(tar_ruler)
        tar_ruler.print_scores()

        agg_tar = TarAggRuler()
        for tar in tml:
            agg_tar.update(tar)
        agg_tar.finalize()
        agg_tar.print_scores()

def usage(args):
    print("Usage: {0} <qrel_file> <results_file>".format(args[0]))

if __name__ == "__main__":
    filename = None
    format = "TOP"
    if len(sys.argv) >= 2:
        qrels = sys.argv[1]

    if len(sys.argv)==3:
        results = sys.argv[2]
    else:
        usage(sys.argv)
        exit(1)

    if os.path.exists( results ) and os.path.exists(qrels):
        main(results,qrels)
    else:
        usage(sys.argv)
<commit_msg>resolved the bug of the missing rel doc and updated the script for the case of zero rel docs in a topic<commit_after>__author__ = "Leif Azzopardi"

import os
import sys

from measures.tar_rulers import TarRuler, TarAggRuler
from seeker.trec_qrel_handler import TrecQrelHandler

def main(results_file, qrel_file):

    qrh = TrecQrelHandler(qrel_file)
    #print(qrh.get_topic_list()) # show what qrel topics have been read in
    #print( len(qrh.get_topic_list())) # show how many

    def get_value_and_check(qrh,seen_dict, topic_id, doc_id):
        # checks to make sure the document is in the qrels and was retrieved by the pubmed query
        v = 0
        if doc_id in seen_dict:
            print("{} Duplicate {}".format(topic_id, doc_id))
            v = None
        else:
            seen_dict[d] = 1
            v = qrh.get_value(topic_id, doc_id)
            if v >= 3:
                v = None
        return v

    skip_topic = False
    curr_topic_id = ""
    seen_dict = {}

    tml = []
    tar_ruler = None
    with open(results_file,"r") as rf:
        while rf:
            line = rf.readline()
            if not line:
                break
            (topic_id,action,doc_id, rank, score, team) = line.split()

            if (topic_id == curr_topic_id):
                if (skip_topic is False):
                    # accumulate
                    #v = qrh.get_value(curr_topic_id, doc_id.strip())
                    d = doc_id.strip()
                    v = get_value_and_check(qrh, seen_dict, curr_topic_id, d)

                    if v is not None:
                        tar_ruler.update(v,v,action)
                else:
                    continue
            else:
                print(topic_id)
                if curr_topic_id is not "":
                    if skip_topic is False:
                        tar_ruler.finalize()
                        tml.append(tar_ruler)
                        tar_ruler.print_scores()
                    else:
                        skip_topic = False

                # new topic
                curr_topic_id = topic_id
                dl = qrh.get_doc_list(topic_id)
                num_docs = len(dl)
                num_rels = 0
                num_rels_in_set = 0
                num_docs_in_set = num_docs
                for d in dl:
                    val = qrh.get_value(topic_id, d)
                    if val > 0:
                        num_rels = num_rels + 1
                    if val == 1 or val == 2:
                        num_rels_in_set = num_rels_in_set + 1

                    if val == -1 or val > 2:
                        num_docs_in_set = num_docs_in_set - 1

                if (num_rels_in_set == 0):
                    print(curr_topic_id)
                    skip_topic = True
                    continue

                #print("D: {0} DS: {1} R: {2} RS: {3} ".format(num_docs,num_docs_in_set,num_rels, num_rels_in_set))
                tar_ruler = TarRuler(topic_id,num_docs_in_set, num_rels_in_set)

                # reset seen list
                seen_dict = {}
                d = doc_id.strip()

                v = get_value_and_check(qrh, seen_dict, curr_topic_id, d)
                if v is not None:
                    tar_ruler.update(v,v,action)

        if skip_topic is False:
            tar_ruler.finalize()
            tml.append(tar_ruler)
            tar_ruler.print_scores()

        agg_tar = TarAggRuler()
        for tar in tml:
            agg_tar.update(tar)
        agg_tar.finalize()
        agg_tar.print_scores()

def usage(args):
    print("Usage: {0} <qrel_file> <results_file>".format(args[0]))

if __name__ == "__main__":
    filename = None
    format = "TOP"
    if len(sys.argv) >= 2:
        qrels = sys.argv[1]

    if len(sys.argv)==3:
        results = sys.argv[2]
    else:
        usage(sys.argv)
        exit(1)

    if os.path.exists( results ) and os.path.exists(qrels):
        main(results,qrels)
    else:
        usage(sys.argv)
<|endoftext|>


<commit_before>"""
spaceshooter.py
Author: vinzentmoesch
Credit:
http://freegameassets.blogspot.com/2013/09/asteroids-and-planets-if-you-needed-to.html

Assignment:
Write and submit a program that implements the spacewar game:
https://github.com/HHS-IntroProgramming/Spacewar
"""
"""
tutorial4.py
by E. Dennison
"""
from ggame import App, RectangleAsset, ImageAsset, Sprite, LineStyle, Color, Frame
from ggame import App, Sprite, ImageAsset, Frame
from ggame import SoundAsset, Sound, TextAsset, Color
import math
from time import time
import random

# zufallszahl
def zufaellig(stellen, komma):
    zufaelligout = round((random.random())*(10**stellen), komma)
    return zufaelligout

#Hintergrund
class Stars(Sprite):

    asset = ImageAsset("images/starswithoutspacesmall.jpg")
    width = 7485
    height = 4930

    def __init__(self, position):
        super().__init__(Stars.asset, position)
        self.scale = 0.23

class astroid(Sprite):
    asset = ImageAsset("images/asteroid1.png",
    Frame(0,0,72,72), 4, 'vertical')

    def __init__(self, position):
        super().__init__(astroid.asset, position)
        self.avx = 0
        self.avy = 0
        self.avr = 0.05
        self.number = 0
        self.randomx = 0
        self.randomy = 0
        self.fxcenter = self.fycenter = 0.5
        self.randomxn = 0
        self.randomyn = 0

    def step(self):
        self.randomx = zufaellig(0, 3)
        self.randomy = zufaellig(0, 3)
        self.randomxn = zufaellig(0,1)
        self.randomyn = zufaellig(0, 1)

        #completely random movement
        if self.number >= 100:
            self.number = 0
            if self.randomxn >= 0.6:
                self.avx = (self.randomx*-1)*3
            else:
                self.avx = self.randomx*3
            if self.randomyn >= 0.6:
                self.avy = (self.randomy*-1)*3
            else:
                self.avy = self.randomy*3
        self.number += 1

        self.rotation += self.avr
        self.x += self.avx
        self.y += self.avy

class SpaceShip(Sprite):
    """
    Animated space ship
    """
    asset = ImageAsset("images/UFO2.png",
        Frame(0,0,485,490), 6, 'vertical')

    def __init__(self, position):
        super().__init__(SpaceShip.asset, position)
        self.scale = 0.15
        self.vx = 0
        self.vy = 0
        self.vr = 0
        self.thrustL = 0
        self.thrustR = 0
        self.thrustU = 0
        self.thrustD = 0
        self.panic = 0
        self.thrustframe = 1
        self.imagenumber = 0
        SpaceGame.listenKeyEvent("keydown", "left arrow", self.thrustLOn)
        SpaceGame.listenKeyEvent("keyup", "left arrow", self.thrustLOff)
        SpaceGame.listenKeyEvent("keydown", "right arrow", self.thrustROn)
        SpaceGame.listenKeyEvent("keyup", "right arrow", self.thrustROff)
        SpaceGame.listenKeyEvent("keydown", "p", self.panicOn)
        SpaceGame.listenKeyEvent("keyup", "p", self.panicOff)
        SpaceGame.listenKeyEvent("keydown", "up arrow", self.thrustUOn)
        SpaceGame.listenKeyEvent("keyup", "up arrow", self.thrustUOff)
        SpaceGame.listenKeyEvent("keydown", "down arrow", self.thrustDOn)
        SpaceGame.listenKeyEvent("keyup", "down arrow", self.thrustDOff)

        self.fxcenter = self.fycenter = 0.5

    def step(self):
        if self.thrustL == 1:
            self.vx -= 0.06
        if self.thrustR == 1:
            self.vx += 0.06
        if self.thrustU == 1:
            self.vy -= 0.06
        if self.thrustD == 1:
            self.vy += 0.06
        if self.panic == 1:
            self.vx = 0
            self.vy = 0
            self.panic = 0
        if self.panic == -1:
            self.vx = 0
            self.vy = 0
            self.panic = 0
        self.x += self.vx
        self.y += self.vy
        self.rotation += self.vr
        if self.thrustL == 1 or self.thrustR == 1 or self.thrustU == 1 or self.thrustD == 1 or self.panic == 1:
            self.setImage(self.thrustframe)
            self.imagenumber += 1
            if self.imagenumber == 9:
                self.thrustframe += 1
                if self.thrustframe >= 8:
                    self.thrustframe = 1
                print(self.thrustframe)
                self.imagenumber = 0

        else:
            self.setImage(0)
            print(self.thrustframe)

    def thrustLOn(self, event):
        self.thrustL = 1

    def thrustLOff(self, event):
        self.thrustL = -1

    def thrustROn(self, event):
        self.thrustR = 1

    def thrustROff(self, event):
        self.thrustR = -1

    def thrustUOn(self, event):
        self.thrustU = 1

    def thrustUOff(self, event):
        self.thrustU = -1

    def thrustDOn(self, event):
        self.thrustD = 1

    def thrustDOff(self, event):
        self.thrustD = -1

    def panicOn(self, event):
        self.panic = 1

    def panicOff(self, event):
        self.panic = -1

class SpaceGame(App):
    """
    Tutorial4 space game example.
    """
    def __init__(self, width, height):
        super().__init__(width, height)
        Stars((0,0))
        SpaceShip((700,500))
        astroid((500,500))
        astroid((500,500))
        astroid((500,500))
        astroid((500,500))
        astroid((500,500))

    def step(self):
        for ship in self.getSpritesbyClass(SpaceShip):
            ship.step()
        for Bstroid in self.getSpritesbyClass(astroid):
            Bstroid.step()

myapp = SpaceGame(1900, 950)
myapp.run() <commit_msg>Updated from Brython Server: 11/18/2016 1:58:23 PM<commit_after>"""
spaceshooter.py
Author: vinzentmoesch
Credit:
http://freegameassets.blogspot.com/2013/09/asteroids-and-planets-if-you-needed-to.html

Assignment:
Write and submit a program that implements the spacewar game:
https://github.com/HHS-IntroProgramming/Spacewar
"""
"""
tutorial4.py
by E. Dennison
"""
from ggame import App, RectangleAsset, ImageAsset, Sprite, LineStyle, Color, Frame
from ggame import App, Sprite, ImageAsset, Frame
from ggame import SoundAsset, Sound, TextAsset, Color
import math
from time import time
import random

# zufallszahl
def zufaellig(stellen, komma):
    zufaelligout = round((random.random())*(10**stellen), komma)
    return zufaelligout

#Hintergrund
class Stars(Sprite):

    asset = ImageAsset("images/starswithoutspacesmall.jpg")
    width = 7485
    height = 4930

    def __init__(self, position):
        super().__init__(Stars.asset, position)
        self.scale = 0.23

class astroid(Sprite):
    asset = ImageAsset("images/asteroid1.png",
    Frame(0,0,72,72), 4, 'vertical')

    def __init__(self, position):
        super().__init__(astroid.asset, position)
        self.avx = 0
        self.avy = 0
        self.avr = 0.05
        self.number = 0
        self.randomx = 0
        self.randomy = 0
        self.fxcenter = self.fycenter = 0.5
        self.randomxn = 0
        self.randomyn = 0

    def step(self):
        self.randomx = zufaellig(0, 3)
        self.randomy = zufaellig(0, 3)
        self.randomxn = zufaellig(0,1)
        self.randomyn = zufaellig(0, 1)

        #completely random movement
        if self.number >= 100:
            self.number = 0
            if self.randomxn >= 0.6:
                self.avx = (self.randomx*-1)*3
            else:
                self.avx = self.randomx*3
            if self.randomyn >= 0.6:
                self.avy = (self.randomy*-1)*3
            else:
                self.avy = self.randomy*3
        self.number += 1

        self.rotation += self.avr
        self.x += self.avx
        self.y += self.avy

class SpaceShip(Sprite):
    """
    Animated space ship
    """
    asset = ImageAsset("images/UFO2.png",
        Frame(0,0,485,490), 6, 'vertical')

    def __init__(self, position):
        super().__init__(SpaceShip.asset, position)
        self.scale = 0.15
        self.vx = 0
        self.vy = 0
        self.vr = 0
        self.thrustL = 0
        self.thrustR = 0
        self.thrustU = 0
        self.thrustD = 0
        self.panic = 0
        self.thrustframe = 1
        self.imagenumber = 0
        SpaceGame.listenKeyEvent("keydown", "left arrow", self.thrustLOn)
        SpaceGame.listenKeyEvent("keyup", "left arrow", self.thrustLOff)
        SpaceGame.listenKeyEvent("keydown", "right arrow", self.thrustROn)
        SpaceGame.listenKeyEvent("keyup", "right arrow", self.thrustROff)
        SpaceGame.listenKeyEvent("keydown", "p", self.panicOn)
        SpaceGame.listenKeyEvent("keyup", "p", self.panicOff)
        SpaceGame.listenKeyEvent("keydown", "up arrow", self.thrustUOn)
        SpaceGame.listenKeyEvent("keyup", "up arrow", self.thrustUOff)
        SpaceGame.listenKeyEvent("keydown", "down arrow", self.thrustDOn)
        SpaceGame.listenKeyEvent("keyup", "down arrow", self.thrustDOff)

        self.fxcenter = self.fycenter = 0.5

    def step(self):
        if self.thrustL == 1:
            self.vx -= 0.06
        if self.thrustR == 1:
            self.vx += 0.06
        if self.thrustU == 1:
            self.vy -= 0.06
        if self.thrustD == 1:
            self.vy += 0.06
        if self.panic == 1:
            self.vx = 0
            self.vy = 0
            self.panic = 0
        if self.panic == -1:
            self.vx = 0
            self.vy = 0
            self.panic = 0
        self.x += self.vx
        self.y += self.vy
        self.rotation += self.vr
        if self.thrustL == 1 or self.thrustR == 1 or self.thrustU == 1 or self.thrustD == 1 or self.panic == 1:
            self.setImage(self.thrustframe)
            self.imagenumber += 1
            if self.imagenumber == 9:
                self.thrustframe += 1
                if self.thrustframe >= 6:
                    self.thrustframe = 1
                print(self.thrustframe)
                self.imagenumber = 0

        else:
            self.setImage(0)
            print(self.thrustframe)

    def thrustLOn(self, event):
        self.thrustL = 1

    def thrustLOff(self, event):
        self.thrustL = -1

    def thrustROn(self, event):
        self.thrustR = 1

    def thrustROff(self, event):
        self.thrustR = -1

    def thrustUOn(self, event):
        self.thrustU = 1

    def thrustUOff(self, event):
        self.thrustU = -1

    def thrustDOn(self, event):
        self.thrustD = 1

    def thrustDOff(self, event):
        self.thrustD = -1

    def panicOn(self, event):
        self.panic = 1

    def panicOff(self, event):
        self.panic = -1

class SpaceGame(App):
    """
    Tutorial4 space game example.
    """
    def __init__(self, width, height):
        super().__init__(width, height)
        Stars((0,0))
        SpaceShip((700,500))
        astroid((500,500))
        astroid((500,500))
        astroid((500,500))
        astroid((500,500))
        astroid((500,500))

    def step(self):
        for ship in self.getSpritesbyClass(SpaceShip):
            ship.step()
        for Bstroid in self.getSpritesbyClass(astroid):
            Bstroid.step()

myapp = SpaceGame(1900, 950)
myapp.run() <|endoftext|>


<commit_before>from flask import Flask, render_template, url_for
from bs4 import BeautifulSoup
import requests
import arrow
import os
import threading
import time

app = Flask(__name__)

wiki_page = "https://en.wikipedia.org/wiki/List_of_Falcon_9_and_Falcon_Heavy_launches"
next_date = ""
update_thread = threading.Thread()

def pr(s):
    print(repr(s))

def refresh_next_date():
    global next_date
    html = requests.get(wiki_page).content
    soup = BeautifulSoup(html, 'html.parser')
    main_content = soup.find(id='mw-content-text')
    found_future_launches = False

    for c in main_content.find_all(['h2', 'table']):
        if c is None: continue
        #print(repr(c))
        if "Future launches" in c.get_text() and c.name == "h2":
            found_future_launches = True
            continue
        if found_future_launches and c.name == 'table':
            e1 = c
            e2 = e1.find_all('tr')[2]
            date_elem = e2.find_all('td')[0]
            pr(date_elem)

            while True:
                try:
                    date_elem.sup.extract()
                except:
                    break

            date_text = date_elem.get_text()
            next_date = arrow.get(date_text, "MMMM D, YYYY")
            try:
                next_date = arrow.get(date_text, "MMMM D, YYYY, HH:mm")
            except:
                pass
            print(next_date)
            break

def update_forever():
    while True:
        refresh_next_date()
        time.sleep(60)

update_thread = threading.Thread(target=update_forever)
#update_thread.start()

@app.route("/")
def spacex_template():
    return render_template('index.html', next_date=next_date)

if __name__ == "__main__":
    app.run(host='0.0.0.0', port=int(os.environ.get('PORT', 5000)))

    url_for('static', filename='countdown.min.js')
    url_for('static', filename='moment-countdown.min.js')<commit_msg>Start update thread<commit_after>from flask import Flask, render_template, url_for
from bs4 import BeautifulSoup
import requests
import arrow
import os
import threading
import time

app = Flask(__name__)

wiki_page = "https://en.wikipedia.org/wiki/List_of_Falcon_9_and_Falcon_Heavy_launches"
next_date = ""
update_thread = threading.Thread()

def pr(s):
    print(repr(s))

def refresh_next_date():
    global next_date
    html = requests.get(wiki_page).content
    soup = BeautifulSoup(html, 'html.parser')
    main_content = soup.find(id='mw-content-text')
    found_future_launches = False

    for c in main_content.find_all(['h2', 'table']):
        if c is None: continue
        #print(repr(c))
        if "Future launches" in c.get_text() and c.name == "h2":
            found_future_launches = True
            continue
        if found_future_launches and c.name == 'table':
            e1 = c
            e2 = e1.find_all('tr')[2]
            date_elem = e2.find_all('td')[0]
            pr(date_elem)

            while True:
                try:
                    date_elem.sup.extract()
                except:
                    break

            date_text = date_elem.get_text()
            next_date = arrow.get(date_text, "MMMM D, YYYY")
            try:
                next_date = arrow.get(date_text, "MMMM D, YYYY, HH:mm")
            except:
                pass
            print(next_date)
            break

def update_forever():
    while True:
        refresh_next_date()
        time.sleep(60)

update_thread = threading.Thread(target=update_forever)
update_thread.start()

@app.route("/")
def spacex_template():
    return render_template('index.html', next_date=next_date)

if __name__ == "__main__":
    app.run(host='0.0.0.0', port=int(os.environ.get('PORT', 5000)))

    url_for('static', filename='countdown.min.js')
    url_for('static', filename='moment-countdown.min.js')<|endoftext|>


<commit_before># -*- coding: iso-8859-1 -*-

"""
wsgidav_app
===========

:Author: Martin Wendt, moogle(at)wwwendt.de
:Author: Ho Chun Wei, fuzzybr80(at)gmail.com (author of original PyFileServer)
:Copyright: Lesser GNU Public License, see LICENSE file attached with package

WSGI container, that handles the HTTP requests. This object is passed to the
WSGI server and represents our WsgiDAV application to the outside.

Configuration
-------------

provider_mapping
    Type: dictionary, default: {}
    {shareName: DAVProvider,
    }
user_mapping
    Type: dictionary, default: {}
host
    Type: str, default: 'localhost'
port
    Type: int, default: 8080
ext_servers
    Type: string list
enable_loggers
    List
propsmanager
    Default: None (use property_manager.PropertyManager)
locksmanager
    Default: None (use lock_manager.LockManager)
domaincontroller
    Default: None (use domain_controller.WsgiDAVDomainController(user_mapping))
verbose
    Type: int, default: 2
    0 no output (excepting application exceptions)
    1 - show single line request summaries (for HTTP logging)
    2 - show additional events
    3 - show full request/response header info (HTTP Logging) request body and GET response bodies not shown

# HTTP Authentication Options
"acceptbasic": True,    # Allow basic authentication, True or False
"acceptdigest": True,   # Allow digest authentication, True or False
"defaultdigest": True,  # True (default digest) or False (default basic)

# Organizational Information - printed as a footer on html output
"response_trailer": None,

See DEVELOPERS.txt_ for more information about the WsgiDAV architecture.

.. _DEVELOPERS.txt: http://wiki.wsgidav-dev.googlecode.com/hg/DEVELOPERS.html
"""
from fs_dav_provider import FilesystemProvider
from wsgidav.dir_browser import WsgiDavDirBrowser
from wsgidav.dav_provider import DAVProvider
import time
import sys
import threading
import urllib
import util
from error_printer import ErrorPrinter
from debug_filter import WsgiDavDebugFilter
from http_authenticator import HTTPAuthenticator
from request_resolver import RequestResolver
from domain_controller import WsgiDAVDomainController
from property_manager import PropertyManager
from lock_manager import LockManager

__docformat__ = "reStructuredText"

# Use these settings, if config file does not define them (or is totally missing)
DEFAULT_CONFIG = {
    "mount_path": None,  # Application root, e.g. <mount_path>/<share_name>/<res_path>
    "provider_mapping": {},
    "host": "localhost",
    "port": 8080,
    "ext_servers": [
#                   "paste",
#                   "cherrypy",
#                   "wsgiref",
                   "wsgidav",
                   ],

    "enable_loggers": [
                      ],

    "propsmanager": None,  # None: use property_manager.PropertyManager
    "locksmanager": None,  # None: use lock_manager.LockManager

    # HTTP Authentication Options
    "user_mapping": {},       # dictionary of dictionaries
    "domaincontroller": None, # None: domain_controller.WsgiDAVDomainController(user_mapping)
    "acceptbasic": True,      # Allow basic authentication, True or False
    "acceptdigest": True,     # Allow digest authentication, True or False
    "defaultdigest": True,    # True (default digest) or False (default basic)

    # Verbose Output
    "verbose": 2,        # 0 - no output (excepting application exceptions)
                         # 1 - show single line request summaries (for HTTP logging)
                         # 2 - show additional events
                         # 3 - show full request/response header info (HTTP Logging)
                         #     request body and GET response bodies not shown

    # Organizational Information - printed as a footer on html output
    "response_trailer": None,
}

def _checkConfig(config):
    mandatoryFields = ["provider_mapping",
                       ]
    for field in mandatoryFields:
        if not field in config:
            raise ValueError("Invalid configuration: missing required field '%s'" % field)

class WsgiDAVApp(object):

    def __init__(self, config):
        self.config = config

        util.initLogging(config["verbose"], config.get("enable_loggers", []))

        util.log("Default encoding: %s (file system: %s)" % (sys.getdefaultencoding(), sys.getfilesystemencoding()))

        # Evaluate configuration and set defaults
        _checkConfig(config)
        provider_mapping = self.config["provider_mapping"]
        response_trailer = config.get("response_trailer", "")
        self._verbose = config.get("verbose", 2)

        locksManager = config.get("locksmanager")
        if not locksManager:
            locksManager = LockManager()

        propsManager = config.get("propsmanager")
        if not propsManager:
            propsManager = PropertyManager()

        mount_path = config.get("mount_path")

        user_mapping = self.config.get("user_mapping", {})
        domainController = config.get("domaincontroller") or WsgiDAVDomainController(user_mapping)
        isDefaultDC = isinstance(domainController, WsgiDAVDomainController)

        # authentication fields
        authacceptbasic = config.get("acceptbasic", True)
        authacceptdigest = config.get("acceptdigest", True)
        authdefaultdigest = config.get("defaultdigest", True)

        # Check configuration for NTDomainController
        # We don't use 'isinstance', because include would fail on non-windows boxes.
        wdcName = "NTDomainController"
        if domainController.__class__.__name__ == wdcName:
            if authacceptdigest or authdefaultdigest or not authacceptbasic:
                print >>sys.stderr, "WARNING: %s requires basic authentication.\n\tSet acceptbasic=True, acceptdigest=False, defaultdigest=False" % wdcName

        # Instantiate DAV resource provider objects for every share
        self.providerMap = {}
        for (share, provider) in provider_mapping.items():
            # Make sure share starts with, or is, '/'
            share = "/" + share.strip("/")

            # We allow a simple string a 'provider'. In this case we interpret
            # it as a file system root folder that is published.
            if isinstance(provider, basestring):
                provider = FilesystemProvider(provider)

            assert isinstance(provider, DAVProvider)

            provider.setSharePath(share)
            if mount_path:
                provider.setMountPath(mount_path)

            # TODO: someday we may want to configure different lock/prop managers per provider
            provider.setLockManager(locksManager)
            provider.setPropManager(propsManager)

            self.providerMap[share] = provider

        if self._verbose >= 2:
            print "Using lock manager: %s" % locksManager
            print "Using property manager: %s" % propsManager
            print "Using domain controller: %s" % domainController
            print "Registered DAV providers:"
            for share, provider in self.providerMap.items():
                hint = ""
                if isDefaultDC and not user_mapping.get(share):
                    hint = " Anonymous!"
                print "  Share '%s': %s%s" % (share, provider, hint)

        # If the default DC is used, emit a warning for anonymous realms
        if isDefaultDC and self._verbose >= 1:
            for share in self.providerMap:
                if not user_mapping.get(share):
                    # TODO: we should only warn here, if --no-auth is not given
                    print "WARNING: share '%s' will allow anonymous access." % share

        # Define WSGI application stack
        application = RequestResolver()
        application = WsgiDavDirBrowser(application)
        application = HTTPAuthenticator(application,
                                        domainController,
                                        authacceptbasic,
                                        authacceptdigest,
                                        authdefaultdigest)
        application = ErrorPrinter(application,
                                   server_descriptor=response_trailer,
                                   catchall=False)

        application = WsgiDavDebugFilter(application)

        self._application = application

    def __call__(self, environ, start_response):

        util.log("SCRIPT_NAME='%s', PATH_INFO='%s'" % (environ.get("SCRIPT_NAME"), environ.get("PATH_INFO")))

        # We unquote PATH_INFO here, although this should already be done by
        # the server.
        path = urllib.unquote(environ["PATH_INFO"])

        # Always adding these values to environ:
        environ["wsgidav.config"] = self.config
        environ["wsgidav.provider"] = None
        environ["wsgidav.verbose"] = self._verbose

        ## Find DAV provider that matches the share

        # sorting share list by reverse length
        shareList = self.providerMap.keys()
        shareList.sort(key=len, reverse=True)

        share = None
        for r in shareList:
            # @@: Case sensitivity should be an option of some sort here;
            #     os.path.normpath might give the preferred case for a filename.
            if r == "/":
                share = r
                break
            elif path.upper() == r.upper() or path.upper().startswith(r.upper()+"/"):
                share = r
                break

        provider = self.providerMap.get(share)

        # Note: we call the next app, even if provider is None, because OPTIONS
        #       must still be handled.
        #       All other requests will result in '404 Not Found'
        environ["wsgidav.provider"] = provider

        # TODO: test with multi-level realms: 'aa/bb'
        # TODO: test security: url contains '..'

        # Transform SCRIPT_NAME and PATH_INFO
        # (Since path and share are unquoted, this also fixes quoted values.)
        if share == "/" or not share:
            environ["PATH_INFO"] = path
        else:
            environ["SCRIPT_NAME"] += share
            environ["PATH_INFO"] = path[len(share):]
#        util.log("--> SCRIPT_NAME='%s', PATH_INFO='%s'" % (environ.get("SCRIPT_NAME"), environ.get("PATH_INFO")))

        # See http://mail.python.org/pipermail/web-sig/2007-January/002475.html
        # for some clarification about SCRIPT_NAME/PATH_INFO format
        # SCRIPT_NAME starts with '/' or is empty
        assert environ["SCRIPT_NAME"] == "" or environ["SCRIPT_NAME"].startswith("/")
        # SCRIPT_NAME must not have a trailing '/'
        assert environ["SCRIPT_NAME"] in ("", "/") or not environ["SCRIPT_NAME"].endswith("/")
        # PATH_INFO starts with '/'
        assert environ["PATH_INFO"] == "" or environ["PATH_INFO"].startswith("/")

        start_time = time.time()
        def _start_response_wrapper(status, response_headers, exc_info=None):
            # Log request
            if self._verbose >= 1:
                threadInfo = ""
                userInfo = environ.get("http_authenticator.username")
                if not userInfo:
                    userInfo = "(anonymous)"
                if self._verbose >= 1:
                    threadInfo = "<%s> " % threading._get_ident()
                extra = []
                if "HTTP_DESTINATION" in environ:
                    extra.append('dest="%s"' % environ.get("HTTP_DESTINATION"))
                if environ.get("CONTENT_LENGTH", "") != "":
                    extra.append("length=%s" % environ.get("CONTENT_LENGTH"))
                if "HTTP_DEPTH" in environ:
                    extra.append("depth=%s" % environ.get("HTTP_DEPTH"))
                if "HTTP_RANGE" in environ:
                    extra.append("range=%s" % environ.get("HTTP_RANGE"))
                if "HTTP_OVERWRITE" in environ:
                    extra.append("overwrite=%s" % environ.get("HTTP_OVERWRITE"))
#                if "HTTP_EXPECT" in environ:
#                    extra.append('expect="%s"' % environ.get("HTTP_EXPECT"))
                if self._verbose >= 2 and "HTTP_USER_AGENT" in environ:
                    extra.append('agent="%s"' % environ.get("HTTP_USER_AGENT"))
                if self._verbose >= 1:
                    extra.append('elap=%.3fsec' % (time.time() - start_time))
                extra = ", ".join(extra)

#               This is the CherryPy format:
#                127.0.0.1 - - [08/Jul/2009:17:25:23] "GET /loginPrompt?redirect=/renderActionList%3Frelation%3Dpersonal%26key%3D%26filter%3DprivateSchedule&reason=0 HTTP/1.1" 200 1944 "http://127.0.0.1:8002/command?id=CMD_Schedule" "Mozilla/5.0 (Windows; U; Windows NT 6.0; de; rv:1.9.1) Gecko/20090624 Firefox/3.5"
                print >>sys.stderr, '%s - %s - [%s] "%s" %s -> %s' % (
                                        threadInfo + environ.get("REMOTE_ADDR",""),
                                        userInfo,
                                        util.getLogTime(),
                                        environ.get("REQUEST_METHOD") + " " + environ.get("PATH_INFO", ""),
                                        extra,
                                        status,
                                        # response Content-Length
                                        # referer
                                     )
            return start_response(status, response_headers, exc_info)

        # Call next middleware
#        for v in self._application(environ, start_response):
        for v in self._application(environ, _start_response_wrapper):
            util.debug("sc", "WsgiDAVApp: yield start")
            yield v
            util.debug("sc", "WsgiDAVApp: yield end")
        return
<commit_msg>Fix issue 22<commit_after># -*- coding: iso-8859-1 -*-

"""
wsgidav_app
===========

:Author: Martin Wendt, moogle(at)wwwendt.de
:Author: Ho Chun Wei, fuzzybr80(at)gmail.com (author of original PyFileServer)
:Copyright: Lesser GNU Public License, see LICENSE file attached with package

WSGI container, that handles the HTTP requests. This object is passed to the
WSGI server and represents our WsgiDAV application to the outside.

Configuration
-------------

provider_mapping
    Type: dictionary, default: {}
    {shareName: DAVProvider,
    }
user_mapping
    Type: dictionary, default: {}
host
    Type: str, default: 'localhost'
port
    Type: int, default: 8080
ext_servers
    Type: string list
enable_loggers
    List
propsmanager
    Default: None (use property_manager.PropertyManager)
locksmanager
    Default: None (use lock_manager.LockManager)
domaincontroller
    Default: None (use domain_controller.WsgiDAVDomainController(user_mapping))
verbose
    Type: int, default: 2
    0 no output (excepting application exceptions)
    1 - show single line request summaries (for HTTP logging)
    2 - show additional events
    3 - show full request/response header info (HTTP Logging) request body and GET response bodies not shown

# HTTP Authentication Options
"acceptbasic": True,    # Allow basic authentication, True or False
"acceptdigest": True,   # Allow digest authentication, True or False
"defaultdigest": True,  # True (default digest) or False (default basic)

# Organizational Information - printed as a footer on html output
"response_trailer": None,

See DEVELOPERS.txt_ for more information about the WsgiDAV architecture.

.. _DEVELOPERS.txt: http://wiki.wsgidav-dev.googlecode.com/hg/DEVELOPERS.html
"""
from fs_dav_provider import FilesystemProvider
from wsgidav.dir_browser import WsgiDavDirBrowser
from wsgidav.dav_provider import DAVProvider
import time
import sys
import threading
import urllib
import util
from error_printer import ErrorPrinter
from debug_filter import WsgiDavDebugFilter
from http_authenticator import HTTPAuthenticator
from request_resolver import RequestResolver
from domain_controller import WsgiDAVDomainController
from property_manager import PropertyManager
from lock_manager import LockManager

__docformat__ = "reStructuredText"

# Use these settings, if config file does not define them (or is totally missing)
DEFAULT_CONFIG = {
    "mount_path": None,  # Application root, e.g. <mount_path>/<share_name>/<res_path>
    "provider_mapping": {},
    "host": "localhost",
    "port": 8080,
    "ext_servers": [
#                   "paste",
#                   "cherrypy",
#                   "wsgiref",
                   "wsgidav",
                   ],

    "enable_loggers": [
                      ],

    "propsmanager": None,  # None: use property_manager.PropertyManager
    "locksmanager": None,  # None: use lock_manager.LockManager

    # HTTP Authentication Options
    "user_mapping": {},       # dictionary of dictionaries
    "domaincontroller": None, # None: domain_controller.WsgiDAVDomainController(user_mapping)
    "acceptbasic": True,      # Allow basic authentication, True or False
    "acceptdigest": True,     # Allow digest authentication, True or False
    "defaultdigest": True,    # True (default digest) or False (default basic)

    # Verbose Output
    "verbose": 2,        # 0 - no output (excepting application exceptions)
                         # 1 - show single line request summaries (for HTTP logging)
                         # 2 - show additional events
                         # 3 - show full request/response header info (HTTP Logging)
                         #     request body and GET response bodies not shown

    # Organizational Information - printed as a footer on html output
    "response_trailer": None,
}

def _checkConfig(config):
    mandatoryFields = ["provider_mapping",
                       ]
    for field in mandatoryFields:
        if not field in config:
            raise ValueError("Invalid configuration: missing required field '%s'" % field)

class WsgiDAVApp(object):

    def __init__(self, config):
        self.config = config

        util.initLogging(config["verbose"], config.get("enable_loggers", []))

        util.log("Default encoding: %s (file system: %s)" % (sys.getdefaultencoding(), sys.getfilesystemencoding()))

        # Evaluate configuration and set defaults
        _checkConfig(config)
        provider_mapping = self.config["provider_mapping"]
        response_trailer = config.get("response_trailer", "")
        self._verbose = config.get("verbose", 2)

        locksManager = config.get("locksmanager")
        if not locksManager:
            locksManager = LockManager()

        propsManager = config.get("propsmanager")
        if not propsManager:
            propsManager = PropertyManager()

        mount_path = config.get("mount_path")

        user_mapping = self.config.get("user_mapping", {})
        domainController = config.get("domaincontroller") or WsgiDAVDomainController(user_mapping)
        isDefaultDC = isinstance(domainController, WsgiDAVDomainController)

        # authentication fields
        authacceptbasic = config.get("acceptbasic", True)
        authacceptdigest = config.get("acceptdigest", True)
        authdefaultdigest = config.get("defaultdigest", True)

        # Check configuration for NTDomainController
        # We don't use 'isinstance', because include would fail on non-windows boxes.
        wdcName = "NTDomainController"
        if domainController.__class__.__name__ == wdcName:
            if authacceptdigest or authdefaultdigest or not authacceptbasic:
                print >>sys.stderr, "WARNING: %s requires basic authentication.\n\tSet acceptbasic=True, acceptdigest=False, defaultdigest=False" % wdcName

        # Instantiate DAV resource provider objects for every share
        self.providerMap = {}
        for (share, provider) in provider_mapping.items():
            # Make sure share starts with, or is, '/'
            share = "/" + share.strip("/")

            # We allow a simple string a 'provider'. In this case we interpret
            # it as a file system root folder that is published.
            if isinstance(provider, basestring):
                provider = FilesystemProvider(provider)

            assert isinstance(provider, DAVProvider)

            provider.setSharePath(share)
            if mount_path:
                provider.setMountPath(mount_path)

            # TODO: someday we may want to configure different lock/prop managers per provider
            provider.setLockManager(locksManager)
            provider.setPropManager(propsManager)

            self.providerMap[share] = provider

        if self._verbose >= 2:
            print "Using lock manager: %s" % locksManager
            print "Using property manager: %s" % propsManager
            print "Using domain controller: %s" % domainController
            print "Registered DAV providers:"
            for share, provider in self.providerMap.items():
                hint = ""
                if isDefaultDC and not user_mapping.get(share):
                    hint = " Anonymous!"
                print "  Share '%s': %s%s" % (share, provider, hint)

        # If the default DC is used, emit a warning for anonymous realms
        if isDefaultDC and self._verbose >= 1:
            for share in self.providerMap:
                if not user_mapping.get(share):
                    # TODO: we should only warn here, if --no-auth is not given
                    print "WARNING: share '%s' will allow anonymous access." % share

        # Define WSGI application stack
        application = RequestResolver()
        application = WsgiDavDirBrowser(application)
        application = HTTPAuthenticator(application,
                                        domainController,
                                        authacceptbasic,
                                        authacceptdigest,
                                        authdefaultdigest)
        application = ErrorPrinter(application,
                                   server_descriptor=response_trailer,
                                   catchall=False)

        application = WsgiDavDebugFilter(application)

        self._application = application

    def __call__(self, environ, start_response):

        util.log("SCRIPT_NAME='%s', PATH_INFO='%s'" % (environ.get("SCRIPT_NAME"), environ.get("PATH_INFO")))

        # We unquote PATH_INFO here, although this should already be done by
        # the server.
        path = urllib.unquote(environ["PATH_INFO"])
        # issue 22: Pylons sends root as u'/'
        if isinstance(path, unicode):
            util.log("Got unicode PATH_INFO: %r" % path)
            path = path.encode("utf8")

        # Always adding these values to environ:
        environ["wsgidav.config"] = self.config
        environ["wsgidav.provider"] = None
        environ["wsgidav.verbose"] = self._verbose

        ## Find DAV provider that matches the share

        # sorting share list by reverse length
        shareList = self.providerMap.keys()
        shareList.sort(key=len, reverse=True)

        share = None
        for r in shareList:
            # @@: Case sensitivity should be an option of some sort here;
            #     os.path.normpath might give the preferred case for a filename.
            if r == "/":
                share = r
                break
            elif path.upper() == r.upper() or path.upper().startswith(r.upper()+"/"):
                share = r
                break

        provider = self.providerMap.get(share)

        # Note: we call the next app, even if provider is None, because OPTIONS
        #       must still be handled.
        #       All other requests will result in '404 Not Found'
        environ["wsgidav.provider"] = provider

        # TODO: test with multi-level realms: 'aa/bb'
        # TODO: test security: url contains '..'

        # Transform SCRIPT_NAME and PATH_INFO
        # (Since path and share are unquoted, this also fixes quoted values.)
        if share == "/" or not share:
            environ["PATH_INFO"] = path
        else:
            environ["SCRIPT_NAME"] += share
            environ["PATH_INFO"] = path[len(share):]
#        util.log("--> SCRIPT_NAME='%s', PATH_INFO='%s'" % (environ.get("SCRIPT_NAME"), environ.get("PATH_INFO")))

        # See http://mail.python.org/pipermail/web-sig/2007-January/002475.html
        # for some clarification about SCRIPT_NAME/PATH_INFO format
        # SCRIPT_NAME starts with '/' or is empty
        assert environ["SCRIPT_NAME"] == "" or environ["SCRIPT_NAME"].startswith("/")
        # SCRIPT_NAME must not have a trailing '/'
        assert environ["SCRIPT_NAME"] in ("", "/") or not environ["SCRIPT_NAME"].endswith("/")
        # PATH_INFO starts with '/'
        assert environ["PATH_INFO"] == "" or environ["PATH_INFO"].startswith("/")

        start_time = time.time()
        def _start_response_wrapper(status, response_headers, exc_info=None):
            # Log request
            if self._verbose >= 1:
                threadInfo = ""
                userInfo = environ.get("http_authenticator.username")
                if not userInfo:
                    userInfo = "(anonymous)"
                if self._verbose >= 1:
                    threadInfo = "<%s> " % threading._get_ident()
                extra = []
                if "HTTP_DESTINATION" in environ:
                    extra.append('dest="%s"' % environ.get("HTTP_DESTINATION"))
                if environ.get("CONTENT_LENGTH", "") != "":
                    extra.append("length=%s" % environ.get("CONTENT_LENGTH"))
                if "HTTP_DEPTH" in environ:
                    extra.append("depth=%s" % environ.get("HTTP_DEPTH"))
                if "HTTP_RANGE" in environ:
                    extra.append("range=%s" % environ.get("HTTP_RANGE"))
                if "HTTP_OVERWRITE" in environ:
                    extra.append("overwrite=%s" % environ.get("HTTP_OVERWRITE"))
#                if "HTTP_EXPECT" in environ:
#                    extra.append('expect="%s"' % environ.get("HTTP_EXPECT"))
                if self._verbose >= 2 and "HTTP_USER_AGENT" in environ:
                    extra.append('agent="%s"' % environ.get("HTTP_USER_AGENT"))
                if self._verbose >= 1:
                    extra.append('elap=%.3fsec' % (time.time() - start_time))
                extra = ", ".join(extra)

#               This is the CherryPy format:
#                127.0.0.1 - - [08/Jul/2009:17:25:23] "GET /loginPrompt?redirect=/renderActionList%3Frelation%3Dpersonal%26key%3D%26filter%3DprivateSchedule&reason=0 HTTP/1.1" 200 1944 "http://127.0.0.1:8002/command?id=CMD_Schedule" "Mozilla/5.0 (Windows; U; Windows NT 6.0; de; rv:1.9.1) Gecko/20090624 Firefox/3.5"
                print >>sys.stderr, '%s - %s - [%s] "%s" %s -> %s' % (
                                        threadInfo + environ.get("REMOTE_ADDR",""),
                                        userInfo,
                                        util.getLogTime(),
                                        environ.get("REQUEST_METHOD") + " " + environ.get("PATH_INFO", ""),
                                        extra,
                                        status,
                                        # response Content-Length
                                        # referer
                                     )
            return start_response(status, response_headers, exc_info)

        # Call next middleware
#        for v in self._application(environ, start_response):
        for v in self._application(environ, _start_response_wrapper):
            util.debug("sc", "WsgiDAVApp: yield start")
            yield v

        return
<|endoftext|>


<commit_before>import base64
from django.conf import settings
from github import Github

from wtl.wtgithub.models import Repository
from wtl.wtlib.models import Project, Library, LibraryVersion, Language
from wtl.wtparser.parser import get_parser_for_filename

class WorkerError(BaseException):
    """
    Base class for all worker exceptions.
    """

class CantFindParserError(WorkerError):
    """
    Raised when can't find requirements file in given repository.
    """

class ParseError(WorkerError):
    """
    Raised when parser can't parse requirements file.
    """

class GithubWorker(object):
    github = None

    def __init__(self, github=None):
        super(GithubWorker, self).__init__()
        if github is None:
            github = Github(getattr(settings, 'WTGITHUB_USERNAME', None),
                            getattr(settings, 'WTGITHUB_PASSWORD', None))
        self.github = github

    def _get_repo(self, full_name):
        """
        Fetches GitHub repository information.
        """
        return self.github.get_repo(full_name)

    def _get_or_create_repository(self, rep):
        """
        Creates (if does not exist) and returns:
        `wtgithub.models.Repository` and `wtlib.models.Project`.
        """
        try:
            repository = Repository.objects.get(name=rep.name,
                                                owner=rep.owner._identity)
        except Repository.DoesNotExist:
            repository = Repository(name=rep.name, owner=rep.owner._identity)
        repository.starsCount = rep.watchers
        repository.description = rep.description
        repository.save()
        try:
            project = repository.project
        except Project.DoesNotExist:
            project = Project.objects.create(github=repository, name=rep.name)
        return repository, project

    def _get_parser_for_repository(self, rep):
        """
        Analyzes repository tree to find corresponding requirements parser.
        Currently analyzes only repository root and returns first matching
        parser. Raises `CantFindParserError` if can't determine parser for
        given repository.
        """
        tree = rep.get_git_tree('master')
        for node in tree.tree:
            if node.type != 'blob':
                continue
            parser = get_parser_for_filename(node.path)
            if parser is not None:
                return node.sha, parser
        raise CantFindParserError()

    def _parse_requirements(self, rep, blob_sha, parser):
        """
        Parses requirements file in given repository with given parser
        Raises `ParseError` if parse fails.
        """
        blob = rep.get_git_blob(blob_sha)
        if blob.encoding == 'utf-8':
            content = blob.content
        elif blob.encoding == 'base64':
            try:
                content = base64.b64decode(blob.content).decode('utf-8')
            except:
                raise ParseError('Error decoding blob')
        else:
            raise ParseError('Unknown blob encoding')
        try:
            return parser.parse(content)
        except:
            raise ParseError()

    def _save_parsed_requirements(self, project, parsed):
        """
        Saves parsed requirements to database.
        """
        language = Language.objects.get(name=parsed['language'])
        for package_dict in parsed['packages']:
            library, _ = Library.objects.get_or_create(
                language=language, name=package_dict['name'])
            version, _ = LibraryVersion.objects.get_or_create(
                library=library, version=package_dict['version'],
                version_special=package_dict['version_special'])
            version.save()
            project.libraries.add(version)
        self._update_user_counts(project)

    def _update_user_counts(self, project):
        LibraryVersion.update_totals_by_project(project)

    def analyze_repo(self, full_name):
        rep = self._get_repo(full_name)
        repository, project = self._get_or_create_repository(rep)
        requirements_blob_sha, parser = self._get_parser_for_repository(rep)
        parsed = self._parse_requirements(rep, requirements_blob_sha, parser)
        self._save_parsed_requirements(project, parsed)
        return repository, project
<commit_msg>Extracted BaseGithubWorker class from GithubWorker<commit_after>import base64
from django.conf import settings
from wtl.wtgithub.github import Github

from wtl.wtgithub.models import Repository
from wtl.wtlib.models import Project, Library, LibraryVersion, Language
from wtl.wtparser.parser import get_parser_for_filename

class WorkerError(BaseException):
    """
    Base class for all worker exceptions.
    """

class CantFindParserError(WorkerError):
    """
    Raised when can't find requirements file in given repository.
    """

class ParseError(WorkerError):
    """
    Raised when parser can't parse requirements file.
    """

class BaseGithubWorker(object):
    github = None

    def __init__(self, github=None):
        super(BaseGithubWorker, self).__init__()
        if github is None:
            github = Github(getattr(settings, 'WTGITHUB_USERNAME', None),
                            getattr(settings, 'WTGITHUB_PASSWORD', None),
                            per_page=getattr(settings, 'WTGITHUB_PER_PAGE', 30))
        self.github = github

class GithubWorker(BaseGithubWorker):
    def _get_repo(self, full_name):
        """
        Fetches GitHub repository information.
        """
        return self.github.get_repo(full_name)

    def _get_or_create_repository(self, rep):
        """
        Creates (if does not exist) and returns:
        `wtgithub.models.Repository` and `wtlib.models.Project`.
        """
        try:
            repository = Repository.objects.get(name=rep.name,
                                                owner=rep.owner._identity)
        except Repository.DoesNotExist:
            repository = Repository(name=rep.name, owner=rep.owner._identity)
        repository.starsCount = rep.watchers
        repository.description = rep.description
        repository.save()
        try:
            project = repository.project
        except Project.DoesNotExist:
            project = Project.objects.create(github=repository, name=rep.name)
        return repository, project

    def _get_parser_for_repository(self, rep):
        """
        Analyzes repository tree to find corresponding requirements parser.
        Currently analyzes only repository root and returns first matching
        parser. Raises `CantFindParserError` if can't determine parser for
        given repository.
        """
        tree = rep.get_git_tree('master')
        for node in tree.tree:
            if node.type != 'blob':
                continue
            parser = get_parser_for_filename(node.path)
            if parser is not None:
                return node.sha, parser
        raise CantFindParserError()

    def _parse_requirements(self, rep, blob_sha, parser):
        """
        Parses requirements file in given repository with given parser
        Raises `ParseError` if parse fails.
        """
        blob = rep.get_git_blob(blob_sha)
        if blob.encoding == 'utf-8':
            content = blob.content
        elif blob.encoding == 'base64':
            try:
                content = base64.b64decode(blob.content).decode('utf-8')
            except:
                raise ParseError('Error decoding blob')
        else:
            raise ParseError('Unknown blob encoding')
        try:
            return parser.parse(content)
        except:
            raise ParseError()

    def _save_parsed_requirements(self, project, parsed):
        """
        Saves parsed requirements to database.
        """
        language = Language.objects.get(name=parsed['language'])
        for package_dict in parsed['packages']:
            library, _ = Library.objects.get_or_create(
                language=language, name=package_dict['name'])
            version, _ = LibraryVersion.objects.get_or_create(
                library=library, version=package_dict['version'],
                version_special=package_dict['version_special'])
            version.save()
            project.libraries.add(version)
        self._update_user_counts(project)

    def _update_user_counts(self, project):
        LibraryVersion.update_totals_by_project(project)

    def analyze_repo(self, full_name):
        rep = self._get_repo(full_name)
        repository, project = self._get_or_create_repository(rep)
        requirements_blob_sha, parser = self._get_parser_for_repository(rep)
        parsed = self._parse_requirements(rep, requirements_blob_sha, parser)
        self._save_parsed_requirements(project, parsed)
        return repository, project
<|endoftext|>