<commit_before>import os #used to work on an entire directory of xml files

#opens a file and reads its lines into a list, one item per line.
def getFileLinesList(filepath):
    theFile = open(filepath)
    listOfLines = theFile.readlines()
    theFile.close()
    return listOfLines

def qsort(lineItemList):
    """Modified Quicksort using list comprehensions"""
    if len(lineItemList) == 0:
        return []
    else:
        if (len(lineItemList) == 1):
            return lineItemList
        else:
            skipIndex = len(lineItemList)/2
            pivot = lineItemList[skipIndex][0]
            skippedItem = lineItemList.pop(skipIndex)
            lesser = qsort([x for x in lineItemList if x[0] < pivot])
            greater = qsort([x for x in lineItemList if x[0] >= pivot])
            return lesser + [skippedItem] + greater

#helper function I wrote to easily extract data from a string.
#'line' is the string to extract from, 'firstDelimiter' is a string that
#identifies where to start looking, and 'secondDelimiter' is the char
#(or string) that signifies the end of the text that will be extracted.
#e.g. to extract '576' from 'top="33" left="576" blahblah':
#line = 'top="33" left="576" blahblah'
#firstDelim = 'left="'   (technically more than needed, but might as well make it readable)
#secondDelim = '"'
def getTextBetweenDelimiters(line, firstDelimiter, secondDelimiter):
    beginIndex = line.find(firstDelimiter) + len(firstDelimiter)
    if beginIndex == -1: #basic check
        return -1
    restOfLine = line[beginIndex:]
    nextDelimiterIndex = restOfLine.find(secondDelimiter)
    if nextDelimiterIndex == -1:
        return line[beginIndex:] #shouldn't really happen
    endIndex = beginIndex + nextDelimiterIndex
    return line[beginIndex:endIndex]

#This function goes through the list of lines read from the file and partially
#parses the text of each line, saving info in either a field list or a data list.
#The relevant information from each line is the numbers assigned to top and left,
#as well as the actual text data represented in the line.
#Finally, the two lists are sorted with respect to the top coordinates.
def createLineIndex(listOfLines):
    lineDataItemList = [] #to store text data and top/left coords in better structure
    lineFieldItemList = []

    for line in listOfLines:

        if ('<text' in line):

            #extracts the digits of the left coordinate as a string
            lineLeft = int(getTextBetweenDelimiters(line, 'left="', '"'))

            if (lineLeft < 60):
                continue
            else:
                #extracts the digits of the top coordinate as a string
                lineTop = int(getTextBetweenDelimiters(line, 'top="', '"'))

                if (line.find('<b>') != -1):
                    lineData = getTextBetweenDelimiters(line, '<b>', '<')
                    dataMarker = True
                elif (line.find('<i>') != -1):
                    lineData = getTextBetweenDelimiters(line, '<i>', '<')
                    dataMarker = True
                else:
                    lineData = getTextBetweenDelimiters(line, '>', '<')
                    dataMarker = False

                if (lineData == -1):
                    lineData = "IRREGULARLY FORMATTED TEXT LINE IN XML FILE"
                    dataMarker = False

                lineItem = [lineTop, lineLeft, lineData]
                if (dataMarker):
                    lineDataItemList.append(lineItem)
                else:
                    lineFieldItemList.append(lineItem)

    sortedDataItemList = qsort(lineDataItemList)
    sortedFieldItemList = qsort(lineFieldItemList)

    return sortedDataItemList, sortedFieldItemList

def processSortedLists(dlist, flist, sectionNameCoordList):
    """Separates list items into chunks that are representative of sections of the pdf.
    sectionNameCoordList is a list of two item lists of the form [yCoordofSectionBreak, "name of section"]"""
    kvpsMaster = []

    for coord in sectionNameCoordList:
        tempDlist = []
        tempFlist = []
        while (dlist[0][0] < coord[0]):
            tempDlist.append(dlist.pop(0))
        while (flist[0][0] < coord[0]):
            tempFlist.append(flist.pop(0))

        #we now have tempDlist and tempFlist containing all the data and fields within a certain
        #section of the pdf. they are passed to another function to try to pair appropriate values.

        listOfMatchedValues = pairFieldandData(tempDlist, tempFlist, coord[1]) #coord[1] is the name of the section; used for state

        kvpsMaster.extend(listOfMatchedValues)

    return kvpsMaster

def pairFieldandData(dlistChunk, flistChunk, state):

    dlistNoChecks, flistNoChecks, kvps = stripCheckmarkData(dlistChunk, flistChunk, state)

    unmatchedData = []

    if (state == "AGENCY_INFO"):

        while (len(flistNoChecks) > 0):
            #print flistNoChecks[0][2] + ' @ ' + str(flistNoChecks[0][0]) + ', ' + str(flistNoChecks[0][1]) + ' compared to '
            for y in range(len(dlistNoChecks)):
                #print dlistNoChecks[y][2] + ' @ ' + str(dlistNoChecks[y][0]) + ', ' + str(dlistNoChecks[y][1])
                if ((abs(flistNoChecks[0][1] - dlistNoChecks[y][1]) < 10)
                    and abs((flistNoChecks[0][0]+20) - dlistNoChecks[y][0]) < 10):
                    #print "Match Found"
                    kvps.append([flistNoChecks[0][2], dlistNoChecks[y][2]])
                    flistNoChecks.pop(0)
                    dlistNoChecks.pop(y)
                    break
                if (y == len(dlistNoChecks)-1):
                    #print "No Match Found"
                    kvps.append([flistNoChecks[0][2], 'NULL'])
                    flistNoChecks.pop(0)
            #print "--------------------------------------------"

        #these two loops fix a one-off technicality instance of weirdness in the forms we're dealing with
        #where two separate fields are represented together; here they are separated.
        for x in kvps:
            if (x[0] == 'ORI'):
                dateTimeArrested = x[1][-17:]
                x[1] = x[1][:-18]
        for x in kvps:
            if (x[0] == 'Date/Time Arrested'):
                x[1] = dateTimeArrested

    elif (state == "ARRESTEE_INFO"):

        #remove these first because the field is split into two lines and oddly formatted (it is handled on its own afterwards)
        flistNoChecks = removeFromFieldList(flistNoChecks, 'Country of')
        flistNoChecks = removeFromFieldList(flistNoChecks, 'Citizenship')
        kvps.append(["Country of Citizenship", "NULL"])

        #occupation is often a problem spot that doesn't get matched so we match it specifically before going general
        flistNoChecks = removeFromFieldList(flistNoChecks, 'Occupation')
        for x in range(len(dlistNoChecks)):
            if (dlistNoChecks[x][0] < 180 or dlistNoChecks[x][0] > 210):
                continue
            elif (dlistNoChecks[x][1] < 500 or dlistNoChecks[x][1] > 720):
                continue
            else:
                kvps.append(["Occupation", dlistNoChecks[x][2]])
                dlistNoChecks.pop(x)
                break

        while (len(flistNoChecks) > 0):
            #print flistNoChecks[0][2] + ' @ ' + str(flistNoChecks[0][0]) + ', ' + str(flistNoChecks[0][1]) + ' compared to '
            for y in range(len(dlistNoChecks)):
                #print dlistNoChecks[y][2] + ' @ ' + str(dlistNoChecks[y][0]) + ', ' + str(dlistNoChecks[y][1])
                if ((abs(flistNoChecks[0][1] - dlistNoChecks[y][1]) < 12)
                    and abs((flistNoChecks[0][0]+22) - dlistNoChecks[y][0]) < 10):
                    #print "Match Found"
                    kvps.append([flistNoChecks[0][2], dlistNoChecks[y][2]])
                    flistNoChecks.pop(0)
                    dlistNoChecks.pop(y)
                    break
                if (y == len(dlistNoChecks)-1):
                    #print "No Match Found"
                    kvps.append([flistNoChecks[0][2], 'NULL'])
                    flistNoChecks.pop(0)
            #print "--------------------------------------------"

        #these two are oddly formatted and probably won't have matched correctly
        stateofbirth = 'NULL'
        citizenship = 'NULL'

        #so assuming they haven't... find them where they usually are:
        for d in dlistNoChecks:
            if (abs(d[0] - 158) < 6):
                if (abs(d[1] - 765) < 10):
                    stateofbirth = d[2]
                elif (abs(d[1] - 790) < 10):
                    citizenship = d[2]
            else:
                unmatchedData.append(d)

        for x in kvps:
            if (x[0] == 'Country of Citizenship'):
                x[1] = citizenship
            if (x[0] == 'Place of Birth' and x[1] == 'NULL'):
                x[1] = stateofbirth

    elif (state == "ARREST_INFO"):

        while (len(flistNoChecks) > 0):
            #print flistNoChecks[0][2] + ' @ ' + str(flistNoChecks[0][0]) + ', ' + str(flistNoChecks[0][1]) + ' compared to '
            for y in range(len(dlistNoChecks)):
                #print dlistNoChecks[y][2] + ' @ ' + str(dlistNoChecks[y][0]) + ', ' + str(dlistNoChecks[y][1])
                if ((abs(flistNoChecks[0][1] - dlistNoChecks[y][1]) < 12)
                    and abs((flistNoChecks[0][0]+15) - dlistNoChecks[y][0]) < 10):
                    #print "Match Found"
                    kvps.append([flistNoChecks[0][2], dlistNoChecks[y][2]])
                    flistNoChecks.pop(0)
                    dlistNoChecks.pop(y)
                    break
                if (y == len(dlistNoChecks)-1):
                    #print "No Match Found"
                    kvps.append([flistNoChecks[0][2], 'NULL'])
                    flistNoChecks.pop(0)
            #print "--------------------------------------------"

        for d in dlistNoChecks:
            unmatchedData.append(d)
##    elif (state == "VEH_INFO"):
##
##    elif (state == "BOND"):
##
##    elif (state == "DRUGS"):
##
##    elif (state == "COMP"):
##
##    elif (state == "NARRATIVE"):
##
##    elif (state == "STATUS"):

    for x in unmatchedData:
        print x

    return kvps

def removeFromFieldList(flist, field):
    for x in range(len(flist)):
        if (flist[x][2] == field):
            flist.pop(x)
            break
    return flist

def removeMultipleFromFieldList(flist, fieldsToRemove):
    for field in fieldsToRemove:
        flist = removeFromFieldList(flist, field)
    return flist

def stripCheckmarkData(dlistChunk, flistChunk, state):
    moe = 0 #margin of error; number of pixels by which something can be off its hardcoded position
    kvps = [] #key-value pairs
    savedIndices = [] #to save indices of data that's been matched to pop that data item at the end;
                        #can't do it when the match happens or the loop gets messed up and items may be skipped

    if (state == "AGENCY_INFO"):
        moe = 6
        for x in range(len(dlistChunk)):
            if (dlistChunk[x][2] == 'X'):
                if (abs(dlistChunk[x][1] - 79) < moe):
                    if (abs(dlistChunk[x][0] - 106) < moe):
                        kvps.append(["Prints Taken", True])
                        flistChunk = removeFromFieldList(flistChunk, 'Prints')
                        savedIndices.append(x)
                    elif (abs(dlistChunk[x][0] - 118) < moe):
                        kvps.append(["Photos Taken", True])
                        flistChunk = removeFromFieldList(flistChunk, 'Photos')
                        savedIndices.append(x)
                else: #something went wrong; there are checks but not in the right place
                    kvps.append(["ChkMkError", False])

        savedIndices.reverse()
        for x in savedIndices:
            dlistChunk.pop(x)
        flistChunk = removeFromFieldList(flistChunk, 'Taken')

    elif (state == "ARRESTEE_INFO"):
        moe = 6

        for x in range(len(dlistChunk)):
            if (dlistChunk[x][2] == 'X'):
                if (dlistChunk[x][0] < 210):
                    if (abs(dlistChunk[x][1] - 735) < moe):
                        if (abs(dlistChunk[x][0] - 184) < moe):
                            kvps.append(["Residency Status", "Resident"])
                            flistChunk = removeMultipleFromFieldList(flistChunk, ['Resident', 'Non-Resident', 'Unknown'])
                            savedIndices.append(x)
                        elif (abs(dlistChunk[x][0] - 198) < moe):
                            kvps.append(["Residency Status", "Non-Resident"])
                            flistChunk = removeMultipleFromFieldList(flistChunk, ['Resident', 'Non-Resident', 'Unknown'])
                            savedIndices.append(x)
                    elif (abs(dlistChunk[x][1] - 815) < moe):
                        if (abs(dlistChunk[x][0] - 184) < moe):
                            kvps.append(["Residency Status", "Unknown"])
                            flistChunk = removeMultipleFromFieldList(flistChunk, ['Resident', 'Non-Resident', 'Unknown'])
                            savedIndices.append(x)
                    else: #something went wrong; there are checks but not in the right place
                        kvps.append(["ChkMkError", False])
                else:
                    if (abs(dlistChunk[x][0] - 262) < moe):
                        if (abs(dlistChunk[x][1] - 775) < moe):
                            kvps.append(["Consumed Drug/Alcohol", "Yes"])
                            flistChunk = removeMultipleFromFieldList(flistChunk, ['Yes', 'No', 'Unk'])
                            savedIndices.append(x)
                        elif (abs(dlistChunk[x][1] - 815) < moe):
                            kvps.append(["Consumed Drug/Alcohol", "No"])
                            flistChunk = removeMultipleFromFieldList(flistChunk, ['Yes', 'No', 'Unk'])
                            savedIndices.append(x)
                        elif (abs(dlistChunk[x][1] - 851) < moe):
                            kvps.append(["Consumed Drug/Alcohol", "Unknown"])
                            flistChunk = removeMultipleFromFieldList(flistChunk, ['Yes', 'No', 'Unk'])
                            savedIndices.append(x)

        savedIndices.reverse()
        for x in savedIndices:
            dlistChunk.pop(x)

    elif (state == "ARREST_INFO"):
        chargeInfo = [False, False, False] #used to keep track of which charge numbers (1-3) have charges and which are null

        for x in range(len(dlistChunk)):
            if (dlistChunk[x][2] == 'X'):
                if (dlistChunk[x][0] < 380):
                    moe = 6
                    if (abs(dlistChunk[x][0] - 355) < moe):
                        if (abs(dlistChunk[x][1] - 278) < moe):
                            kvps.append(["Method of Arrest", "On-View"])
                            flistChunk = removeMultipleFromFieldList(flistChunk, ['On-View', 'Criminal Summons', 'Order for Arrest', 'Citation', 'Warrant'])
                            savedIndices.append(x)
                        elif (abs(dlistChunk[x][1] - 356) < moe):
                            kvps.append(["Method of Arrest", "Criminal Summons"])
                            flistChunk = removeMultipleFromFieldList(flistChunk, ['On-View', 'Criminal Summons', 'Order for Arrest', 'Citation', 'Warrant'])
                            savedIndices.append(x)
                    elif (abs(dlistChunk[x][0] - 370) < moe):
                        if (abs(dlistChunk[x][1] - 278) < moe):
                            kvps.append(["Method of Arrest", "Order for Arrest"])
                            flistChunk = removeMultipleFromFieldList(flistChunk, ['On-View', 'Criminal Summons', 'Order for Arrest', 'Citation', 'Warrant'])
                            savedIndices.append(x)
                        elif (abs(dlistChunk[x][1] - 375) < moe):
                            kvps.append(["Method of Arrest", "Citation"])
                            flistChunk = removeMultipleFromFieldList(flistChunk, ['On-View', 'Criminal Summons', 'Order for Arrest', 'Citation', 'Warrant'])
                            savedIndices.append(x)
                        elif (abs(dlistChunk[x][1] - 446) < moe):
                            kvps.append(["Method of Arrest", "Warrant"])
                            flistChunk = removeMultipleFromFieldList(flistChunk, ['On-View', 'Criminal Summons', 'Order for Arrest', 'Citation', 'Warrant'])
                            savedIndices.append(x)
                else:
                    moe = 6
                    if (abs(dlistChunk[x][1] - 328) < moe):
                        moe = 3
                        if (abs(dlistChunk[x][0] - 395) < moe):
                            kvps.append(["Charge 1 Type", "Felony"])
                            chargeInfo[0] = True
                            savedIndices.append(x)
                        elif (abs(dlistChunk[x][0] - 406) < moe):
                            kvps.append(["Charge 1 Type", "Misdemeanor"])
                            chargeInfo[0] = True
                            savedIndices.append(x)
                        elif (abs(dlistChunk[x][0] - 433) < moe):
                            kvps.append(["Charge 2 Type", "Felony"])
                            chargeInfo[1] = True
                            savedIndices.append(x)
                        elif (abs(dlistChunk[x][0] - 447) < moe):
                            kvps.append(["Charge 2 Type", "Misdemeanor"])
                            chargeInfo[1] = True
                            savedIndices.append(x)
                        elif (abs(dlistChunk[x][0] - 473) < moe):
                            kvps.append(["Charge 3 Type", "Felony"])
                            chargeInfo[2] = True
                            savedIndices.append(x)
                        elif (abs(dlistChunk[x][0] - 486) < moe):
                            kvps.append(["Charge 3 Type", "Misdemeanor"])
                            chargeInfo[2] = True
                            savedIndices.append(x)

        for x in range(len(chargeInfo)):
            if (chargeInfo[x] == False):
                kvps.append(["Charge " + str(x+1) + " Type", "NULL"])

        flistChunk = removeMultipleFromFieldList(flistChunk, ['Fel', 'Misd', 'Fel', 'Misd', 'Fel', 'Misd'])

        savedIndices.reverse()
        for x in savedIndices:
            dlistChunk.pop(x)

##    elif (state == "VEH_INFO"):
##
##    elif (state == "BOND"):
##
##    elif (state == "DRUGS"):
##
##    elif (state == "COMP"):
##
##    elif (state == "NARRATIVE"):
##
##    elif (state == "STATUS"):

    return dlistChunk, flistChunk, kvps

#The next two functions are convenience functions I wrote so I wouldn't have
#to repetitively enter parameters when calling the functions in IDLE.
#Absolute file paths are more reliable.
#
#'filepath' is path to xml file
def getLineIndexFromFile(filepath):
    LOL = getFileLinesList(filepath)
    sortedDList, sortedFList = createLineIndex(LOL)
    return sortedDList, sortedFList

def getKVPMasterList(sortedDlist, sortedFlist, sectionNameCoordList):
    return processSortedLists(sortedDlist, sortedFlist, sectionNameCoordList)

#analagous usage but directoryPath is path to directory folder rather than file
def doItAllMultipleTimes(directoryPath):
    allData = []
    sncl = [[130, 'AGENCY_INFO']]
    for filename in os.listdir(directoryPath):
        dlist, flist = getLineIndexFromFile(directoryPath+filename)
        kvps = getKVPMasterList(dlist, flist, sncl)
        allData.append(kvps)
    return allData

#sncl = [[130, 'AGENCY_INFO'], [350, 'ARRESTEE_INFO'], [502, 'ARREST-INFO']]
#

#if you set 'path' to wherever your directory of (already converted!) xml files
#is located, the program should work by running this module and then entering the following commands:
#
#path = [your path here]
#dlist, flist = getLineIndexFromFile(path)
#sncl = [[130, 'AGENCY_INFO'], [350, 'ARRESTEE_INFO'], [502, 'ARREST-INFO']]
#kvps = getKVPMasterList(dlist, flist, sncl)
#for x in kvps:
#    print x

#this can be ignored, I use it bc of a file naming quirk that comes up when
#I try to use the pdftohtml unix util in a loop that goes through a whole
#directory. It would convert a pdf to 'example.pdf.xml', so I got rid of '.pdf'
def renamingScript(prependpath):
    for filename in os.listdir(prependpath):
        os.rename(prependpath+filename, prependpath+filename[:-8]+filename[-4:])
<commit_msg>remove old version of scraper<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import json
import ligneous

logger = ligneous.log("SkeletonDB")

def load(location="skeleton.db", fsave=True, verbose=True):
    '''Return a SkeletonDB object. location is the path to the json file.'''
    return Database(location, fsave, verbose)

class Database(object):

    def __init__(self, location="skeleton.db", fsave=True, verbose=True):
        '''Creates a database object and loads the data from the location path.
        If the file does not exist it will be created on the first update.'''
        self.verbose = verbose
        self.load(location, fsave)

    def load(self, location, fsave):
        '''Loads, reloads or changes the path to the db file.'''
        location = os.path.expanduser(location)
        self.location = location
        self.fsave = fsave
        if os.path.exists(location):
            self._loaddb()
        else:
            self.db = {}

    def dump(self):
        '''Force dump memory db to file.'''
        self._dumpdb(True)

    def set(self, key, value):
        '''Set the (string,int,whatever) value of a key'''
        self.db[key] = value
        if self.verbose:
            logger.info("insert")
        self._dumpdb(self.fsave)

    def get(self, key):
        '''Get the value of a key'''
        try:
            return self.db[key]
        except KeyError:
            return None

    def keys(self,key=None):
        '''Returns list containing keys of database or dict entry'''
        if key:
            return list(self.db[key].keys())
        return list(self.db.keys())

    def reload(self):
        '''Reloads database'''
        self._loaddb(self.fsave)

    def exists(self,key):
        '''Checks if key exists in database'''
        if key not in self.db:
            return False
        else:
            return True

    def remove(self, key):
        '''Delete a key'''
        del self.db[key]
        self._dumpdb(self.fsave)

    def _deldb(self):
        '''Delete everything from the database'''
        self.db= {}
        self._dumpdb(self.fsave)
        return True

    def _loaddb(self):
        '''Load or reload the json info from the file'''
        if self.verbose:
            logger.info("load from database")
        self.db = json.load(open(self.location))

    def _dumpdb(self, forced):
        '''Write/save the json dump into the file'''
        if forced:
            if self.verbose:
                logger.info("commit")
            json.dump(self.db, open(self.location, 'w'))

<commit_msg>key value database for caching<commit_after>#!/usr/bin/env python

import os
import json
import ligneous

logger = ligneous.log("database")

def load(location="skeleton.db", fsave=True, verbose=True):
    '''Return a SkeletonDB object. location is the path to the json file.'''
    return Database(location, fsave, verbose)

class Database(object):

    def __init__(self, location="skeleton.db", fsave=True, verbose=True):
        '''Creates a database object and loads the data from the location path.
        If the file does not exist it will be created on the first update.'''
        self.verbose = verbose
        self.load(location, fsave)

    def load(self, location, fsave):
        '''Loads, reloads or changes the path to the db file.'''
        location = os.path.expanduser(location)
        self.location = location
        self.fsave = fsave
        if os.path.exists(location):
            self._loaddb()
        else:
            self.db = {}

    def dump(self):
        '''Force dump memory db to file.'''
        self._dumpdb(True)

    def set(self, key, value):
        '''Set the (string,int,whatever) value of a key'''
        self.db[key] = value
        if self.verbose:
            logger.info("insert")
        self._dumpdb(self.fsave)

    def get(self, key):
        '''Get the value of a key'''
        try:
            return self.db[key]
        except KeyError:
            return None

    def keys(self,key=None):
        '''Returns list containing keys of database or dict entry'''
        if key:
            return list(self.db[key].keys())
        return list(self.db.keys())

    def reload(self):
        '''Reloads database'''
        self._loaddb(self.fsave)

    def exists(self,key):
        '''Checks if key exists in database'''
        if key not in self.db:
            return False
        else:
            return True

    def remove(self, key):
        '''Delete a key'''
        del self.db[key]
        self._dumpdb(self.fsave)

    def _deldb(self):
        '''Delete everything from the database'''
        self.db= {}
        self._dumpdb(self.fsave)
        return True

    def _loaddb(self):
        '''Load or reload the json info from the file'''
        if self.verbose:
            logger.info("load from database")
        self.db = json.load(open(self.location))

    def _dumpdb(self, forced):
        '''Write/save the json dump into the file'''
        if forced:
            if self.verbose:
                logger.info("commit")
            json.dump(self.db, open(self.location, 'w'))

<|endoftext|>


<commit_before>#-*- encoding: utf-8 -*-
from __future__ import unicode_literals

import chstrings

import os

global_config = dict(
    # Approximate maximum length for a snippet
    snippet_max_size = 420,

    # Approximate minimum length for a snippet
    snippet_min_size = 100,
)

# Most (all?) Wikipedias support these English settings in addition
# to the localized ones, so make sure to handle them!

EN_WIKILINK_PREFIX_BLACKLIST = [
    'File:',
    'Image:',
    'Media:'
]

EN_CITATION_NEEDED_TEMPLATES = [
    'Citation needed',
    'cn',
]

EN_TAGS_BLACKLIST = [
    'math',
]

EN_TEMPLATES_BLACKLIST = [
    'lang',
]

lang_code_to_config = dict(
    en = dict(
        # A friendly name for the language
        lang_name = 'English',
        # The direction of the language, either ltr or rtl
        lang_dir = 'ltr',
        # The database to use on Tools Labs
        database = 'enwiki_p',
        # The domain for Wikipedia in this language
        wikipedia_domain = 'en.wikipedia.org',

        # A link to an introductory article about adding citations
        beginners_link = 'https://en.wikipedia.org/wiki/Help:Referencing_for_beginners',
        # A human-readable title for the article specified above. This gets
        # interpolated into the localizable string 'beginners_hint'
        beginners_link_title = 'Referencing for beginners',

        # Some Wikipedias have specific policies for adding citations to the
        # lead section of an article. This should be a link to that policy.
        lead_section_policy_link = "https://en.wikipedia.org/wiki/Wikipedia:CITELEAD",
        # A human-readable title for the link above. This gets interpolated
        # into the localizable string 'lead_section_hint'
        lead_section_policy_link_title = "WP:CITELEAD",

        # The name of the category containing articles lacking
        # citations, without the 'Category:' prefix
        citation_needed_category = 'All_articles_with_unsourced_statements',

        # The names of templates that mark statements lacking
        # citations. The first letter is case-insensitive. When
        # adding a new language, this may help:
        # https://www.wikidata.org/wiki/Q5312535
        citation_needed_templates = EN_CITATION_NEEDED_TEMPLATES,

        # For consistency, we don't actually display each of the templates
        # listed in `citation_needed_templates` in the user interface; instead
        # we just replace them with one common name. For example, in the English
        # Wikipedia, we use the iconic [citation needed]. This should basically
        # match what the user would see on Wikipedia, minus the brackets.
        citation_needed_template_name = 'citation needed',

        # Wikilinks having these prefixes will be omitted
        # entirely from the output; others will get replaced
        # by their titles.
        wikilink_prefix_blacklist = EN_WIKILINK_PREFIX_BLACKLIST,

        # A set of tags that we know for sure we can't handle, so
        # snippets containing them will be dropped.
        tags_blacklist = EN_TAGS_BLACKLIST,

        # A set of templates that we know for sure we can't handle, so
        # snippets containing them will be dropped.
        templates_blacklist = EN_TAGS_BLACKLIST,

        # The name of the category for hidden categories. Categories
        # belonging to this category are typically used for maintenance
        # and should not show up on Citation Hunt.
        hidden_category = 'Hidden_categories',

        # Citation Hunt will ignore categories if their names match one
        # of these regular expressions.
        category_name_regexps_blacklist = [
            '^Articles',
            '^Pages ',
            ' stubs$',
            '.*[0-9]+.*',
        ],

        # The maximum number of categories to use
        max_categories = 5000,
    ),

    fr = dict(
        lang_name = 'Français',
        lang_dir = 'ltr',
        database = 'frwiki_p',
        wikipedia_domain = 'fr.wikipedia.org',
        citation_needed_category = 'Article_à_référence_nécessaire',
        beginners_link = 'https://fr.wikipedia.org/wiki/Aide:Pr%C3%A9sentez_vos_sources',
        beginners_link_title = 'Aide:Source',
        lead_section_policy_link = "https://fr.wikipedia.org/wiki/WP:INTRO",
        lead_section_policy_link_title = "WP:INTRO",

        # Looks like there are many other interesting templates:
        # https://fr.wikipedia.org/wiki/Aide:Référence_nécessaire
        citation_needed_templates = EN_CITATION_NEEDED_TEMPLATES + [
            'Inédit',
            'Référence nécessaire',
            'Référence souhaitée',
            'ref nec',
            'ref sou',
            'refnec',
            'refsou',
        ],
        citation_needed_template_name = 'réf. nécessaire',

        wikilink_prefix_blacklist = EN_WIKILINK_PREFIX_BLACKLIST + [
            'Fichier:',
        ],

        tags_blacklist = EN_TAGS_BLACKLIST,

        templates_blacklist = EN_TEMPLATES_BLACKLIST,

        hidden_category = 'Catégorie_cachée',

        category_name_regexps_blacklist = [
            '.*[0-9]+.*',
        ],

        max_categories = 1000,
    ),

    it = dict(
        lang_name = 'Italiano',
        lang_dir = 'ltr',
        database = 'itwiki_p',
        wikipedia_domain = 'it.wikipedia.org',
        citation_needed_category = 'Informazioni_senza_fonte',
        beginners_link = 'https://it.wikipedia.org/wiki/Aiuto:Uso_delle_fonti',
        beginners_link_title = 'Aiuto:Uso_delle_fonti',
        lead_section_policy_link = '',
        lead_section_policy_link_title = '',

        citation_needed_templates = EN_CITATION_NEEDED_TEMPLATES + [
            'Citazione necessaria',
            'Senza fonte',
        ],
        citation_needed_template_name = 'senza fonte',

        wikilink_prefix_blacklist = EN_WIKILINK_PREFIX_BLACKLIST,

        tags_blacklist = EN_TAGS_BLACKLIST,

        templates_blacklist = EN_TEMPLATES_BLACKLIST,

        hidden_category = 'Categorie_nascoste',

        category_name_regexps_blacklist = [
            '.*[0-9]+.*',
        ],

        max_categories = 1000,
    ),

    pl = dict(
        lang_name = 'Polski',
        lang_dir = 'ltr',
        database = 'plwiki_p',
        wikipedia_domain = 'pl.wikipedia.org',
        citation_needed_category = 'Artykuły_wymagające_uzupełnienia_źródeł',
        beginners_link = 'https://pl.wikipedia.org/wiki/Pomoc:Przypisy',
        beginners_link_title = 'Pomoc:Przypisy',
        lead_section_policy_link = '',
        lead_section_policy_link_title = '',

        citation_needed_templates = EN_CITATION_NEEDED_TEMPLATES + [
            'fakt',
        ],
        citation_needed_template_name = 'potrzebne źródło',

        wikilink_prefix_blacklist = EN_WIKILINK_PREFIX_BLACKLIST,

        tags_blacklist = EN_TAGS_BLACKLIST,

        templates_blacklist = EN_TEMPLATES_BLACKLIST,

        hidden_category = 'Ukryte_kategorie',

        category_name_regexps_blacklist = [
            '.*[0-9]+.*',
        ],

        max_categories = 1000,
    ),

    ca = dict(
        lang_name = 'Català',
        lang_dir = 'ltr',
        database = 'cawiki_p',
        wikipedia_domain = 'ca.wikipedia.org',
        citation_needed_category = 'Articles_amb_referències_puntuals_demanades',
        beginners_link = 'https://ca.wikipedia.org/wiki/Viquip%C3%A8dia:Guia_per_referenciar',
        beginners_link_title = 'Viquipèdia:Guia per referenciar',
        lead_section_policy_link = '',
        lead_section_policy_link_title = '',

        citation_needed_templates = EN_CITATION_NEEDED_TEMPLATES + [
            'Citació necessària',
            'CC',
            'CN',
        ],
        citation_needed_template_name = 'cal citació',

        wikilink_prefix_blacklist = EN_WIKILINK_PREFIX_BLACKLIST,

        tags_blacklist = EN_TAGS_BLACKLIST,

        templates_blacklist = EN_TEMPLATES_BLACKLIST,

        hidden_category = 'Categories_ocultes',

        category_name_regexps_blacklist = [
            '.*[0-9]+.*',
        ],

        max_categories = 100,
    ),
)

# In py3: types.SimpleNamespace
class Config(object):
    def __init__(self, **kwargs):
        self.__dict__.update(kwargs)

def get_localized_config(lang_code = None):
    if lang_code is None:
        lang_code = os.getenv('CH_LANG')
    cfg = Config(lang_code = lang_code,
        **dict(global_config, **lang_code_to_config[lang_code]))
    cfg.strings = chstrings.get_localized_strings(cfg, lang_code)
    cfg.lang_code_to_config = lang_code_to_config
    return cfg
<commit_msg>Switch the beginners' link for English.<commit_after>#-*- encoding: utf-8 -*-
from __future__ import unicode_literals

import chstrings

import os

global_config = dict(
    # Approximate maximum length for a snippet
    snippet_max_size = 420,

    # Approximate minimum length for a snippet
    snippet_min_size = 100,
)

# Most (all?) Wikipedias support these English settings in addition
# to the localized ones, so make sure to handle them!

EN_WIKILINK_PREFIX_BLACKLIST = [
    'File:',
    'Image:',
    'Media:'
]

EN_CITATION_NEEDED_TEMPLATES = [
    'Citation needed',
    'cn',
]

EN_TAGS_BLACKLIST = [
    'math',
]

EN_TEMPLATES_BLACKLIST = [
    'lang',
]

lang_code_to_config = dict(
    en = dict(
        # A friendly name for the language
        lang_name = 'English',
        # The direction of the language, either ltr or rtl
        lang_dir = 'ltr',
        # The database to use on Tools Labs
        database = 'enwiki_p',
        # The domain for Wikipedia in this language
        wikipedia_domain = 'en.wikipedia.org',

        # A link to an introductory article about adding citations
        beginners_link = 'https://en.wikipedia.org/wiki/Help:Introduction_to_referencing_with_VisualEditor/1',
        # A human-readable title for the article specified above. This gets
        # interpolated into the localizable string 'beginners_hint'
        beginners_link_title = 'Introduction to referencing with VisualEditor',

        # Some Wikipedias have specific policies for adding citations to the
        # lead section of an article. This should be a link to that policy.
        lead_section_policy_link = "https://en.wikipedia.org/wiki/Wikipedia:CITELEAD",
        # A human-readable title for the link above. This gets interpolated
        # into the localizable string 'lead_section_hint'
        lead_section_policy_link_title = "WP:CITELEAD",

        # The name of the category containing articles lacking
        # citations, without the 'Category:' prefix
        citation_needed_category = 'All_articles_with_unsourced_statements',

        # The names of templates that mark statements lacking
        # citations. The first letter is case-insensitive. When
        # adding a new language, this may help:
        # https://www.wikidata.org/wiki/Q5312535
        citation_needed_templates = EN_CITATION_NEEDED_TEMPLATES,

        # For consistency, we don't actually display each of the templates
        # listed in `citation_needed_templates` in the user interface; instead
        # we just replace them with one common name. For example, in the English
        # Wikipedia, we use the iconic [citation needed]. This should basically
        # match what the user would see on Wikipedia, minus the brackets.
        citation_needed_template_name = 'citation needed',

        # Wikilinks having these prefixes will be omitted
        # entirely from the output; others will get replaced
        # by their titles.
        wikilink_prefix_blacklist = EN_WIKILINK_PREFIX_BLACKLIST,

        # A set of tags that we know for sure we can't handle, so
        # snippets containing them will be dropped.
        tags_blacklist = EN_TAGS_BLACKLIST,

        # A set of templates that we know for sure we can't handle, so
        # snippets containing them will be dropped.
        templates_blacklist = EN_TAGS_BLACKLIST,

        # The name of the category for hidden categories. Categories
        # belonging to this category are typically used for maintenance
        # and should not show up on Citation Hunt.
        hidden_category = 'Hidden_categories',

        # Citation Hunt will ignore categories if their names match one
        # of these regular expressions.
        category_name_regexps_blacklist = [
            '^Articles',
            '^Pages ',
            ' stubs$',
            '.*[0-9]+.*',
        ],

        # The maximum number of categories to use
        max_categories = 5000,
    ),

    fr = dict(
        lang_name = 'Français',
        lang_dir = 'ltr',
        database = 'frwiki_p',
        wikipedia_domain = 'fr.wikipedia.org',
        citation_needed_category = 'Article_à_référence_nécessaire',
        beginners_link = 'https://fr.wikipedia.org/wiki/Aide:Pr%C3%A9sentez_vos_sources',
        beginners_link_title = 'Aide:Source',
        lead_section_policy_link = "https://fr.wikipedia.org/wiki/WP:INTRO",
        lead_section_policy_link_title = "WP:INTRO",

        # Looks like there are many other interesting templates:
        # https://fr.wikipedia.org/wiki/Aide:Référence_nécessaire
        citation_needed_templates = EN_CITATION_NEEDED_TEMPLATES + [
            'Inédit',
            'Référence nécessaire',
            'Référence souhaitée',
            'ref nec',
            'ref sou',
            'refnec',
            'refsou',
        ],
        citation_needed_template_name = 'réf. nécessaire',

        wikilink_prefix_blacklist = EN_WIKILINK_PREFIX_BLACKLIST + [
            'Fichier:',
        ],

        tags_blacklist = EN_TAGS_BLACKLIST,

        templates_blacklist = EN_TEMPLATES_BLACKLIST,

        hidden_category = 'Catégorie_cachée',

        category_name_regexps_blacklist = [
            '.*[0-9]+.*',
        ],

        max_categories = 1000,
    ),

    it = dict(
        lang_name = 'Italiano',
        lang_dir = 'ltr',
        database = 'itwiki_p',
        wikipedia_domain = 'it.wikipedia.org',
        citation_needed_category = 'Informazioni_senza_fonte',
        beginners_link = 'https://it.wikipedia.org/wiki/Aiuto:Uso_delle_fonti',
        beginners_link_title = 'Aiuto:Uso_delle_fonti',
        lead_section_policy_link = '',
        lead_section_policy_link_title = '',

        citation_needed_templates = EN_CITATION_NEEDED_TEMPLATES + [
            'Citazione necessaria',
            'Senza fonte',
        ],
        citation_needed_template_name = 'senza fonte',

        wikilink_prefix_blacklist = EN_WIKILINK_PREFIX_BLACKLIST,

        tags_blacklist = EN_TAGS_BLACKLIST,

        templates_blacklist = EN_TEMPLATES_BLACKLIST,

        hidden_category = 'Categorie_nascoste',

        category_name_regexps_blacklist = [
            '.*[0-9]+.*',
        ],

        max_categories = 1000,
    ),

    pl = dict(
        lang_name = 'Polski',
        lang_dir = 'ltr',
        database = 'plwiki_p',
        wikipedia_domain = 'pl.wikipedia.org',
        citation_needed_category = 'Artykuły_wymagające_uzupełnienia_źródeł',
        beginners_link = 'https://pl.wikipedia.org/wiki/Pomoc:Przypisy',
        beginners_link_title = 'Pomoc:Przypisy',
        lead_section_policy_link = '',
        lead_section_policy_link_title = '',

        citation_needed_templates = EN_CITATION_NEEDED_TEMPLATES + [
            'fakt',
        ],
        citation_needed_template_name = 'potrzebne źródło',

        wikilink_prefix_blacklist = EN_WIKILINK_PREFIX_BLACKLIST,

        tags_blacklist = EN_TAGS_BLACKLIST,

        templates_blacklist = EN_TEMPLATES_BLACKLIST,

        hidden_category = 'Ukryte_kategorie',

        category_name_regexps_blacklist = [
            '.*[0-9]+.*',
        ],

        max_categories = 1000,
    ),

    ca = dict(
        lang_name = 'Català',
        lang_dir = 'ltr',
        database = 'cawiki_p',
        wikipedia_domain = 'ca.wikipedia.org',
        citation_needed_category = 'Articles_amb_referències_puntuals_demanades',
        beginners_link = 'https://ca.wikipedia.org/wiki/Viquip%C3%A8dia:Guia_per_referenciar',
        beginners_link_title = 'Viquipèdia:Guia per referenciar',
        lead_section_policy_link = '',
        lead_section_policy_link_title = '',

        citation_needed_templates = EN_CITATION_NEEDED_TEMPLATES + [
            'Citació necessària',
            'CC',
            'CN',
        ],
        citation_needed_template_name = 'cal citació',

        wikilink_prefix_blacklist = EN_WIKILINK_PREFIX_BLACKLIST,

        tags_blacklist = EN_TAGS_BLACKLIST,

        templates_blacklist = EN_TEMPLATES_BLACKLIST,

        hidden_category = 'Categories_ocultes',

        category_name_regexps_blacklist = [
            '.*[0-9]+.*',
        ],

        max_categories = 100,
    ),
)

# In py3: types.SimpleNamespace
class Config(object):
    def __init__(self, **kwargs):
        self.__dict__.update(kwargs)

def get_localized_config(lang_code = None):
    if lang_code is None:
        lang_code = os.getenv('CH_LANG')
    cfg = Config(lang_code = lang_code,
        **dict(global_config, **lang_code_to_config[lang_code]))
    cfg.strings = chstrings.get_localized_strings(cfg, lang_code)
    cfg.lang_code_to_config = lang_code_to_config
    return cfg
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import os
# Belonging to the path of  sqlite development database.
basedir = os.path.abspath(os.path.dirname(__file__))
engine = 'sqlite'  # sqlite, Mysql, postgres and so on.
db_name = 'fudcon.db'
db_user = ''
db_password = ''
db_server = 'localhost'  # Usually, localhost

class Config(object):
    DEBUG = False
    TESTING = False
    CSRF_ENABLED = True
    SQLALCHEMY_ECHO = True
    FAS_OPENID_ENDPOINT = 'http://id.fedoraproject.org/'
    FAS_CHECK_CERT = True
    FUDCON_NAME = 'Fudcon Managua'
    ADMIN_GROUP = ['provenpackager']
    ITEMS_PER_PAGE = 10
    UPLOADS_FOLDER = os.path.realpath('.') + '/fudcon/static/uploads'
    ALLOWED_EXTENSIONS = set(['jpeg', 'jpg', 'gif', 'png',
                             'txt', 'pdf'])

class ProductionConfig(Config):
    SQLALCHEMY_DATABASE_URI = '{0}://{1}:{2}@{3}/{4}'.format(engine,
                                                             db_user,
                                                             db_password,
                                                             db_server,
                                                             db_name)
    DATABASE_CONNECT_OPTIONS = {}
    SECRET_KEY = 'key'
    CSRF_SESSION_KEY = "somethingimpossibletoguess"

class DevelopmentConfig(Config):
    DEBUG = True
    SQLALCHEMY_DATABASE_URI = engine + ':///' + os.path.join(basedir,
                                                             db_name)
    DATABASE_CONNECT_OPTIONS = {}
    sQLALCHEMY_ECHO = True
    SECRET_KEY = 'This string will be replaced with a proper key in production'
    CSRF_SESSION_KEY = "somethingimpossibletoguess"

class TestingConfig(Config):
    TESTING = True
<commit_msg>Added python-social-auth parameters config<commit_after># -*- coding: utf-8 -*-
import os
# Belonging to the path of  sqlite development database.
basedir = os.path.abspath(os.path.dirname(__file__))
engine = 'sqlite'  # sqlite, Mysql, postgres and so on.
db_name = 'fudcon.db'
db_user = ''
db_password = ''
db_server = 'localhost'  # Usually, localhost

class Config(object):
    DEBUG = False
    TESTING = False
    CSRF_ENABLED = True
    SQLALCHEMY_ECHO = True
    FAS_OPENID_ENDPOINT = 'http://id.fedoraproject.org/'
    FAS_CHECK_CERT = True
    FUDCON_NAME = 'Fudcon Managua'
    ADMIN_GROUP = ['provenpackager']
    ITEMS_PER_PAGE = 10
    UPLOADS_FOLDER = os.path.realpath('.') + '/fudcon/static/uploads'
    ALLOWED_EXTENSIONS = set(['jpeg', 'jpg', 'gif', 'png',
                             'txt', 'pdf'])
    SOCIAL_AUTH_LOGIN_URL = '/login'
    SOCIAL_AUTH_LOGIN_REDIRECT_URL = '/done'
    SOCIAL_AUTH_USER_MODEL = 'fudcon.modules.users.models.User'
    SOCIAL_AUTH_AUTHENTICATION_BACKENDS = (
        'social.backends.google.GoogleOpenId',
        'social.backends.twitter.TwitterOAuth',
        'social.backends.fedora.FedoraOpenId',
        )

class ProductionConfig(Config):
    SQLALCHEMY_DATABASE_URI = '{0}://{1}:{2}@{3}/{4}'.format(engine,
                                                             db_user,
                                                             db_password,
                                                             db_server,
                                                             db_name)
    DATABASE_CONNECT_OPTIONS = {}
    SECRET_KEY = 'key'
    CSRF_SESSION_KEY = "somethingimpossibletoguess"

class DevelopmentConfig(Config):
    DEBUG = True
    SQLALCHEMY_DATABASE_URI = engine + ':///' + os.path.join(basedir,
                                                             db_name)
    DATABASE_CONNECT_OPTIONS = {}
    sQLALCHEMY_ECHO = True
    SECRET_KEY = 'This string will be replaced with a proper key in production'
    CSRF_SESSION_KEY = "somethingimpossibletoguess"

class TestingConfig(Config):
    TESTING = True
<|endoftext|>


<commit_before>import os
from os import path as p
from datetime import date as d

_basedir = p.dirname(__file__)

# configuration
class Content(object):
	site_values = (
		'prometheus', 'Prometheus', 'Reuben Cummings',
		'http://reubano.github.com', 'home')

	topnav_values = [
		('event', 'Events', 'hermes.get', 'event'),
		('transaction', 'Transactions', 'cronus.transaction', None),
		('worth', 'Net Worth', 'apollo.worth', 'USD'),
		('about', 'About', 'about', None),
		('api', 'API', 'api', None)]

	hero_values = (
		'Prometheus: a global asset allocation tool', 'Prometheus is'
		' a full featured web app that tells you how your stock portfolio has '
		' performed over time, gives insight into how to optimize your asset '
		' allocation, and monitors your portfolio for rebalancing or performance'
		' enhancing opportunities.', 'about')

	sub_unit_values = [
		(
			'Events', 'See all your stocks events in one convenient'
			' location. Track stock splits, dividend payments,'
			' mergers and more!', 'hermes.get', 'event'),
		(
			'Prices', 'Update your stock prices with the click of a button! '
			'Automatically grap the latest pricing information from Yahoo or '
			'Google.', 'hermes.get', 'price'),
		(
			'Net Worth', 'See how the value of your portfolio changed over '
			'time with these sleek interactive charts! Instantly see how  '
			'dividends impact your return.', 'apollo.worth', 'USD')]

	heroku_app = site_values[0]
	mkd_values = [('about', 'about.md'), ('api', 'api.md')]
	total_site_values = site_values + (
		d.today().strftime("%Y"), 12 / len(sub_unit_values))

	site_keys = (
		'id', 'caption', 'author', 'author_url', 'location', 'date', 'sub_span')
	topnav_keys = ('id', 'caption', 'location', 'table')
	hero_keys = ('heading', 'subheading', 'location')
	sub_unit_keys = ('heading', 'text', 'location', 'table')
	mkd_keys = ('id', 'file')

	SITE = dict(zip(site_keys, total_site_values))
	TOPNAV = [dict(zip(topnav_keys, values)) for values in topnav_values]
	HERO = dict(zip(hero_keys, hero_values))
	SUB_UNITS = [dict(zip(sub_unit_keys, values)) for values in sub_unit_values]
	MKD_PAGES = [dict(zip(mkd_keys, values)) for values in mkd_values]
	MKD_FOLDER = 'markdown'

class Config(Content):
	DEBUG = False
	ADMINS = frozenset(['reubano@gmail.com'])
	TESTING = False
	HOST = '127.0.0.1'
	PORT = int(os.environ.get('PORT', 5000))
# 	SERVER_NAME = '%s.herokuapp.com' % Content.heroku_app
	SECRET_KEY = os.environ.get('SECRET_KEY', 'key')
	CSRF_SESSION_KEY = os.environ.get('CSRF_SESSION_KEY', 'key')
	RECAPTCHA_PUBLIC_KEY = os.environ.get('RECAPTCHA_PUBLIC_KEY', '')
	RECAPTCHA_PRIVATE_KEY = os.environ.get('RECAPTCHA_PRIVATE_KEY', '')
	BOOTSTRAP_USE_MINIFIED = True
	BOOTSTRAP_USE_CDN = False
	BOOTSTRAP_FONTAWESOME = False
	BOOTSTRAP_HTML5_SHIM = True
	BOOTSTRAP_GOOGLE_ANALYTICS_ACCOUNT = os.environ.get(
		'GOOGLE_ANALYTICS_ID', '')

	CSRF_ENABLED = True
	RECAPTCHA_USE_SSL = False
	RECAPTCHA_OPTIONS = {'theme': 'white'}
	API_METHODS = ['GET', 'POST', 'DELETE', 'PATCH', 'PUT']
	API_ALLOW_FUNCTIONS = True
	API_ALLOW_PATCH_MANY = True

class Production(Config):
	defaultdb = 'postgres://reubano@localhost/app'
	SQLALCHEMY_DATABASE_URI = os.environ.get('DATABASE_URL', defaultdb)
	HOST = '0.0.0.0'
	BOOTSTRAP_USE_CDN = True
	BOOTSTRAP_FONTAWESOME = True

class Development(Config):
	SQLALCHEMY_DATABASE_URI = 'sqlite:///%s' % p.join(_basedir, 'app.db')
	DEBUG = True

class Test(Config):
	SQLALCHEMY_DATABASE_URI = 'sqlite:///:memory:'
	TESTING = True
<commit_msg>Change home page column<commit_after>import os
from os import path as p
from datetime import date as d

_basedir = p.dirname(__file__)

# configuration
class Content(object):
	site_values = (
		'prometheus', 'Prometheus', 'Reuben Cummings',
		'http://reubano.github.com', 'home')

	topnav_values = [
		('event', 'Events', 'hermes.get', 'event'),
		('transaction', 'Transactions', 'cronus.transaction', None),
		('worth', 'Net Worth', 'apollo.worth', 'USD'),
		('about', 'About', 'about', None),
		('api', 'API', 'api', None)]

	hero_values = (
		'Prometheus: a global asset allocation tool', 'Prometheus is'
		' a full featured web app that tells you how your stock portfolio has '
		' performed over time, gives insight into how to optimize your asset '
		' allocation, and monitors your portfolio for rebalancing or performance'
		' enhancing opportunities.', 'about')

	sub_unit_values = [
		(
			'Events', 'See all your stocks events in one convenient'
			' location. Track stock splits, dividend payments,'
			' mergers and more!', 'hermes.get', 'event'),
		(
			'Transactions', 'Add your stock transactions with the click of a '
			'button! CSV file upload feature coming soon.',
			'cronus.transaction', None),
		(
			'Net Worth', 'See how the value of your portfolio changed over '
			'time with these sleek interactive charts! Instantly see how  '
			'dividends impact your return.', 'apollo.worth', 'USD')]

	heroku_app = site_values[0]
	mkd_values = [('about', 'about.md'), ('api', 'api.md')]
	total_site_values = site_values + (
		d.today().strftime("%Y"), 12 / len(sub_unit_values))

	site_keys = (
		'id', 'caption', 'author', 'author_url', 'location', 'date', 'sub_span')
	topnav_keys = ('id', 'caption', 'location', 'table')
	hero_keys = ('heading', 'subheading', 'location')
	sub_unit_keys = ('heading', 'text', 'location', 'table')
	mkd_keys = ('id', 'file')

	SITE = dict(zip(site_keys, total_site_values))
	TOPNAV = [dict(zip(topnav_keys, values)) for values in topnav_values]
	HERO = dict(zip(hero_keys, hero_values))
	SUB_UNITS = [dict(zip(sub_unit_keys, values)) for values in sub_unit_values]
	MKD_PAGES = [dict(zip(mkd_keys, values)) for values in mkd_values]
	MKD_FOLDER = 'markdown'

class Config(Content):
	DEBUG = False
	ADMINS = frozenset(['reubano@gmail.com'])
	TESTING = False
	HOST = '127.0.0.1'
	PORT = int(os.environ.get('PORT', 5000))
# 	SERVER_NAME = '%s.herokuapp.com' % Content.heroku_app
	SECRET_KEY = os.environ.get('SECRET_KEY', 'key')
	CSRF_SESSION_KEY = os.environ.get('CSRF_SESSION_KEY', 'key')
	RECAPTCHA_PUBLIC_KEY = os.environ.get('RECAPTCHA_PUBLIC_KEY', '')
	RECAPTCHA_PRIVATE_KEY = os.environ.get('RECAPTCHA_PRIVATE_KEY', '')
	BOOTSTRAP_USE_MINIFIED = True
	BOOTSTRAP_USE_CDN = False
	BOOTSTRAP_FONTAWESOME = False
	BOOTSTRAP_HTML5_SHIM = True
	BOOTSTRAP_GOOGLE_ANALYTICS_ACCOUNT = os.environ.get(
		'GOOGLE_ANALYTICS_ID', '')

	CSRF_ENABLED = True
	RECAPTCHA_USE_SSL = False
	RECAPTCHA_OPTIONS = {'theme': 'white'}
	API_METHODS = ['GET', 'POST', 'DELETE', 'PATCH', 'PUT']
	API_ALLOW_FUNCTIONS = True
	API_ALLOW_PATCH_MANY = True

class Production(Config):
	defaultdb = 'postgres://reubano@localhost/app'
	SQLALCHEMY_DATABASE_URI = os.environ.get('DATABASE_URL', defaultdb)
	HOST = '0.0.0.0'
	BOOTSTRAP_USE_CDN = True
	BOOTSTRAP_FONTAWESOME = True

class Development(Config):
	SQLALCHEMY_DATABASE_URI = 'sqlite:///%s' % p.join(_basedir, 'app.db')
	DEBUG = True

class Test(Config):
	SQLALCHEMY_DATABASE_URI = 'sqlite:///:memory:'
	TESTING = True
<|endoftext|>


<commit_before># The MIT License
#
# Copyright (c) 2010 Jeffrey Jenkins
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

from mongoalchemy.exceptions import BadValueException

class BadQueryException(Exception):
    ''' Raised when a method would result in a query which is not well-formed.
    '''
    pass

class FreeFormField(object):
    has_subfields = True
    no_real_attributes = True
    def __init__(self, name=None):
        self.__name = name
        self.db_field = name
    def __getattr__(self, name):
        return FreeFormField(name=name)
    def __getitem__(self, name):
        return getattr(self, name)
    @classmethod
    def wrap_value(self, value):
        return value
    def subfields(self):
        return FreeFormField(name=None)
    def is_valid_wrap(*args):
        return True
    is_valid_unwrap = is_valid_wrap
    __contains__ = is_valid_wrap

class FreeFormDoc(object):
    def __init__(self, name):
        self.__name = name
    def __getattr__(self, name):
        return QueryField(FreeFormField(name))
    @classmethod
    def unwrap(cls, value, *args, **kwargs):
        return value
    def get_collection_name(self):
        return self.__name
    def get_indexes(self):
        return []
    mongo_id = FreeFormField(name='_id')

Q = FreeFormDoc('')

class QueryField(object):
    def __init__(self, type, parent=None):
        self.__type = type
        self.__parent = parent
        self.__cached_id_value = None
        self.__matched_index = False

    @property
    def __cached_id(self):
        if self.__cached_id_value == None:
            self.__cached_id_value = str(self)
        return self.__cached_id_value

    def _get_parent(self):
        return self.__parent

    def get_type(self):
        ''' Returns the underlying :class:`mongoalchemy.fields.Field` '''
        return self.__type

    def matched_index(self):
        ''' Represents the matched array index on a query with objects inside
            of a list.  In the MongoDB docs, this is the ``$`` operator '''
        self.__matched_index = True
        return self

    def __getattr__(self, name):
        if not self.__type.no_real_attributes and hasattr(self.__type, name):
            return getattr(self.__type, name)

        if not self.__type.has_subfields:
            raise AttributeError(name)

        fields = self.__type.subfields()
        if name not in fields:
            raise BadQueryException('%s is not a field in %s' % (name, self.__type.sub_type()))
        return QueryField(fields[name], parent=self)

    def get_absolute_name(self):
        res = []
        current = self

        while type(current) != type(None):
            if current.__matched_index:
                res.append('$')
            res.append(current.get_type().db_field)
            current = current._get_parent()
        return '.'.join(reversed(res))

    def in_(self, *values):
        ''' A query to check if this query field is one of the values
            in ``values``.  Produces a MongoDB ``$in`` expression.
        '''
        return QueryExpression({
            self : { '$in' : [self.get_type().wrap_value(value) for value in values] }
        })

    def nin(self, *values):
        ''' A query to check if this query field is not one of the values
            in ``values``.  Produces a MongoDB ``$nin`` expression.
        '''
        return QueryExpression({
            self : { '$nin' : [self.get_type().wrap_value(value) for value in values] }
        })

    def __str__(self):
        return self.get_absolute_name()

    def __repr__(self):
        return 'QueryField(%s)' % str(self)

    def __hash__(self):
        return hash(self.__cached_id)

    def __eq__(self, value):
        return self.eq_(value)
    def eq_(self, value):
        ''' Creates a query expression where ``this field == value``

            .. note:: The prefered usage is via an operator: ``User.name == value``
        '''
        if isinstance(value, QueryField):
            return self.__cached_id == value.__cached_id
        return QueryExpression({ self : self.get_type().wrap_value(value) })

    def __lt__(self, value):
        return self.lt_(value)
    def lt_(self, value):
        ''' Creates a query expression where ``this field < value``

            .. note:: The prefered usage is via an operator: ``User.name < value``
        '''
        return self.__comparator('$lt', value)

    def __le__(self, value):
        return self.le_(value)
    def le_(self, value):
        ''' Creates a query expression where ``this field <= value``

            .. note:: The prefered usage is via an operator: ``User.name <= value``
        '''
        return self.__comparator('$lte', value)

    def __ne__(self, value):
        return self.ne_(value)
    def ne_(self, value):
        ''' Creates a query expression where ``this field != value``

            .. note:: The prefered usage is via an operator: ``User.name != value``
        '''
        if isinstance(value, QueryField):
            return self.__cached_id != value.__cached_id
        return self.__comparator('$ne', value)

    def __gt__(self, value):
        return self.gt_(value)
    def gt_(self, value):
        ''' Creates a query expression where ``this field > value``

            .. note:: The prefered usage is via an operator: ``User.name > value``
        '''
        return self.__comparator('$gt', value)

    def __ge__(self, value):
        return self.ge_(value)
    def ge_(self, value):
        ''' Creates a query expression where ``this field >= value``

            .. note:: The prefered usage is via an operator: ``User.name >= value``
        '''
        return self.__comparator('$gte', value)

    def __comparator(self, op, value):
        return QueryExpression({
            self.get_absolute_name() : {
                op : self.get_type().wrap(value)
            }
        })

class QueryExpression(object):
    ''' A QueryExpression wraps a dictionary representing a query to perform
        on a mongo collection.  The

        .. note:: There is no ``and_`` expression because multiple expressions
            can be specified to a single call of :func:`Query.filter`
    '''
    def __init__(self, obj):
        self.obj = obj
    def not_(self):
        '''Negates this instance's query expression using MongoDB's ``$not``
            operator

            **Example**: ``(User.name == 'Jeff').not_()``

            .. note:: Another usage is via an operator, but parens are needed
                to get past precedence issues: ``~ (User.name == 'Jeff')``
            '''
        ret_obj = {}
        for k, v in self.obj.iteritems():
            if not isinstance(v, dict):
                ret_obj[k] = {'$ne' : v }
                continue
            num_ops = len([x for x in v if x[0] == '$'])
            if num_ops != len(v) and num_ops != 0:
                raise BadQueryException('$ operator used in field name')

            if num_ops == 0:
                ret_obj[k] = {'$ne' : v }
                continue

            for op, value in v.iteritems():
                k_dict = ret_obj.setdefault(k, {})
                not_dict = k_dict.setdefault('$not', {})
                not_dict[op] = value

        return QueryExpression(ret_obj)

    def __invert__(self):
        return self.not_()

    def __or__(self, expression):
        return self.or_(expression)

    def or_(self, expression):
        ''' Adds the given expression to this instance's MongoDB ``$or``
            expression, starting a new one if one does not exst

            **Example**: ``(User.name == 'Jeff').or_(User.name == 'Jack')``

            .. note:: The prefered usageis via an operator: ``User.name == 'Jeff' | User.name == 'Jack'``

            '''

        if '$or' in self.obj:
            self.obj['$or'].append(expression.obj)
            return self
        self.obj = {
            '$or' : [self.obj, expression.obj]
        }
        return self
<commit_msg>Use the query field as the key for query expressions with comparators<commit_after># The MIT License
#
# Copyright (c) 2010 Jeffrey Jenkins
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

from mongoalchemy.exceptions import BadValueException

class BadQueryException(Exception):
    ''' Raised when a method would result in a query which is not well-formed.
    '''
    pass

class FreeFormField(object):
    has_subfields = True
    no_real_attributes = True
    def __init__(self, name=None):
        self.__name = name
        self.db_field = name
    def __getattr__(self, name):
        return FreeFormField(name=name)
    def __getitem__(self, name):
        return getattr(self, name)
    @classmethod
    def wrap_value(self, value):
        return value
    def subfields(self):
        return FreeFormField(name=None)
    def is_valid_wrap(*args):
        return True
    is_valid_unwrap = is_valid_wrap
    __contains__ = is_valid_wrap

class FreeFormDoc(object):
    def __init__(self, name):
        self.__name = name
    def __getattr__(self, name):
        return QueryField(FreeFormField(name))
    @classmethod
    def unwrap(cls, value, *args, **kwargs):
        return value
    def get_collection_name(self):
        return self.__name
    def get_indexes(self):
        return []
    mongo_id = FreeFormField(name='_id')

Q = FreeFormDoc('')

class QueryField(object):
    def __init__(self, type, parent=None):
        self.__type = type
        self.__parent = parent
        self.__cached_id_value = None
        self.__matched_index = False

    @property
    def __cached_id(self):
        if self.__cached_id_value == None:
            self.__cached_id_value = str(self)
        return self.__cached_id_value

    def _get_parent(self):
        return self.__parent

    def get_type(self):
        ''' Returns the underlying :class:`mongoalchemy.fields.Field` '''
        return self.__type

    def matched_index(self):
        ''' Represents the matched array index on a query with objects inside
            of a list.  In the MongoDB docs, this is the ``$`` operator '''
        self.__matched_index = True
        return self

    def __getattr__(self, name):
        if not self.__type.no_real_attributes and hasattr(self.__type, name):
            return getattr(self.__type, name)

        if not self.__type.has_subfields:
            raise AttributeError(name)

        fields = self.__type.subfields()
        if name not in fields:
            raise BadQueryException('%s is not a field in %s' % (name, self.__type.sub_type()))
        return QueryField(fields[name], parent=self)

    def get_absolute_name(self):
        res = []
        current = self

        while type(current) != type(None):
            if current.__matched_index:
                res.append('$')
            res.append(current.get_type().db_field)
            current = current._get_parent()
        return '.'.join(reversed(res))

    def in_(self, *values):
        ''' A query to check if this query field is one of the values
            in ``values``.  Produces a MongoDB ``$in`` expression.
        '''
        return QueryExpression({
            self : { '$in' : [self.get_type().wrap_value(value) for value in values] }
        })

    def nin(self, *values):
        ''' A query to check if this query field is not one of the values
            in ``values``.  Produces a MongoDB ``$nin`` expression.
        '''
        return QueryExpression({
            self : { '$nin' : [self.get_type().wrap_value(value) for value in values] }
        })

    def __str__(self):
        return self.get_absolute_name()

    def __repr__(self):
        return 'QueryField(%s)' % str(self)

    def __hash__(self):
        return hash(self.__cached_id)

    def __eq__(self, value):
        return self.eq_(value)
    def eq_(self, value):
        ''' Creates a query expression where ``this field == value``

            .. note:: The prefered usage is via an operator: ``User.name == value``
        '''
        if isinstance(value, QueryField):
            return self.__cached_id == value.__cached_id
        return QueryExpression({ self : self.get_type().wrap_value(value) })

    def __lt__(self, value):
        return self.lt_(value)
    def lt_(self, value):
        ''' Creates a query expression where ``this field < value``

            .. note:: The prefered usage is via an operator: ``User.name < value``
        '''
        return self.__comparator('$lt', value)

    def __le__(self, value):
        return self.le_(value)
    def le_(self, value):
        ''' Creates a query expression where ``this field <= value``

            .. note:: The prefered usage is via an operator: ``User.name <= value``
        '''
        return self.__comparator('$lte', value)

    def __ne__(self, value):
        return self.ne_(value)
    def ne_(self, value):
        ''' Creates a query expression where ``this field != value``

            .. note:: The prefered usage is via an operator: ``User.name != value``
        '''
        if isinstance(value, QueryField):
            return self.__cached_id != value.__cached_id
        return self.__comparator('$ne', value)

    def __gt__(self, value):
        return self.gt_(value)
    def gt_(self, value):
        ''' Creates a query expression where ``this field > value``

            .. note:: The prefered usage is via an operator: ``User.name > value``
        '''
        return self.__comparator('$gt', value)

    def __ge__(self, value):
        return self.ge_(value)
    def ge_(self, value):
        ''' Creates a query expression where ``this field >= value``

            .. note:: The prefered usage is via an operator: ``User.name >= value``
        '''
        return self.__comparator('$gte', value)

    def __comparator(self, op, value):
        return QueryExpression({
            self : {
                op : self.get_type().wrap(value)
            }
        })

class QueryExpression(object):
    ''' A QueryExpression wraps a dictionary representing a query to perform
        on a mongo collection.  The

        .. note:: There is no ``and_`` expression because multiple expressions
            can be specified to a single call of :func:`Query.filter`
    '''
    def __init__(self, obj):
        self.obj = obj
    def not_(self):
        '''Negates this instance's query expression using MongoDB's ``$not``
            operator

            **Example**: ``(User.name == 'Jeff').not_()``

            .. note:: Another usage is via an operator, but parens are needed
                to get past precedence issues: ``~ (User.name == 'Jeff')``
            '''
        ret_obj = {}
        for k, v in self.obj.iteritems():
            if not isinstance(v, dict):
                ret_obj[k] = {'$ne' : v }
                continue
            num_ops = len([x for x in v if x[0] == '$'])
            if num_ops != len(v) and num_ops != 0:
                raise BadQueryException('$ operator used in field name')

            if num_ops == 0:
                ret_obj[k] = {'$ne' : v }
                continue

            for op, value in v.iteritems():
                k_dict = ret_obj.setdefault(k, {})
                not_dict = k_dict.setdefault('$not', {})
                not_dict[op] = value

        return QueryExpression(ret_obj)

    def __invert__(self):
        return self.not_()

    def __or__(self, expression):
        return self.or_(expression)

    def or_(self, expression):
        ''' Adds the given expression to this instance's MongoDB ``$or``
            expression, starting a new one if one does not exst

            **Example**: ``(User.name == 'Jeff').or_(User.name == 'Jack')``

            .. note:: The prefered usageis via an operator: ``User.name == 'Jeff' | User.name == 'Jack'``

            '''

        if '$or' in self.obj:
            self.obj['$or'].append(expression.obj)
            return self
        self.obj = {
            '$or' : [self.obj, expression.obj]
        }
        return self
<|endoftext|>