<commit_before>import re
import sys
import textwrap
import phox_utilities
from dateutil import parser

MAX_LEDE = 4   # number of sentences to output
# this is relatively high because we are only looking for sentences that
# will have subject and object
MIN_SENTLENGTH = 100
MAX_SENTLENGTH = 512

MAX_URLLENGTH = 192   # temporary to accommodate TABARI input limits

newsourcestem = 'newsources.'

sources = {
    'csmonitor.com': 'CSM',
    'bbc.co.uk': 'BBC',
    'reuters.com': 'REU',
    'xinhuanet.com': 'XIN',
    'www.upi.com': 'UPI',
    'nytimes.com': 'NYT',
    'todayszaman.com': 'TZA',
    'hosted2.ap.org': 'APP',
    'theguardian.com': 'GUA',
    'todayszaman.com': 'TZA',
    'insightcrime.org': 'ISC',
    'france24.com': 'FRA',
    'yahoo.com': 'YAH',
    'allafrica.com': 'ALA',
    'voanews.com': 'VOA',
    'aljazeera.com': 'AJZ',
    'AlAkhbarEnglish': 'AKB',
    'usatoday.com': 'USA',
    'latimes.com': 'LAT',
    'foxnews.com': 'FOX',
    'IRINnews.org': 'IRI',
    'rfi.fr': 'RFI',
    'cnn.com': 'CNN',
    'abcnews': 'ABC',
    'wsj.com': 'WSJ',
    'nydailynews.com': 'NYD',
    'washingtonpost.com': 'WAS',
    'chicagotribune.com': 'CHT',
    'miamiherald.com':'MIA',
    'thestar.com.my':'STA',
    'al-monitor.com':'MON',
    'un.org':'UNN',
    'pakistantelegraph.com':'PAK',
    'africaleader.com':'AFR',
    'theuknews.com':'UKN',
    'indiagazette.com':'ING',
    'mexicostar.com':'MES',
    'japanherald.com':'JAH',
    'chinanationalnews.com':'CHN',
    'jpost.com':'JPO',
    'middleeaststar.com':'MEA',
    'beijingbulletin.com':'BEB',
    'arabherald.com':'ARH',
    'russiaherald.com':'RUH',
    'iraqsun.com':'IRA',
    'malaysiasun.com':'MAL',
    'thehindu.com':'HIN',
    'shanghaisun.com':'SHA',
    'dawn.com':'DAW',
    'singaporestar.com':'SNG',
    'southeastasiapost.com':'SEA',
    'afghanistansun.com':'AFG',
    'australianherald.com':'AUS',
    'thenational.ae':'NAT',
    'europesun.com':'EUS',
    'greekherald.com':'GKH',
    'irishsun.com':'IRS',
    'iranherald.com':'IAH',
    'parisguardian.com':'PAG',
    'brazilsun.com':'BZS',
    'asiabulletin.com':'ASB',
    'nigeriasun.com':'NIG',
    'haitisun.com':'HAI',
    'philippinetimes.com':'PHI',
    'venezuelastar.com':'VEN',
    'caribbeanherald.com':'CAR',
    'zimbabwestar.com':'ZIM',
    'torontotelegraph.com':'TOR',
    'bangladeshsun.com':'BAN',
    'sydneysun.com':'SYD',
    'menafn.com':'MEN',
    'thelocal.se':'TLO',
    'nepalnational.com':'NEP',
    'kenyastar.com':'KEN',
    'sierraleonetimes.com':'SIE',
    'hurriyetdailynews.com':'HDN',
    'pajhwok.com':'PAJ',
    'thelocal.it':'LIT',
    'thelocal.fr':'LFR',
    'thelocal.es':'LES',
    'thelocal.ch':'LCH',
    'ipsnews.net':'IPS',
    'jamaicantimes.com':'JAM',
    'argentinastar.com':'ARG',
    'srilankasource.com':'SRI',
    'centralasiatimes.com':'CEN',
    'thelocal.no':'LNO',
    'theeastafrican.co.ke':'ESF',
    'taiwansun.com':'TAI',
    'cambodiantimes.com':'CAM',
    'payvand.com':'PAY',
    'trinidadtimes.com':'TRD',
    'yementimes.com':'YEM',
    'hongkongherald.com':'HNK'}

def get_date(field):
    """
    Function to extract date from story header in scraper_results.

    Parameters
    ------
    field: String
            Story header.

    Returns
    -------

    date : String
            Date string in the form YYMMDD
    """
    if 'csmonitor.com' in field[-1]:
        csmdate = field[-1].split('/20')[1].split('/')
        date = csmdate[0] + csmdate[1]
    elif field[1]:
        try:
        date_obj = parser.parse(field[1])
            date = str(date_obj)[2:4] + str(date_obj)[5:7] + str(date_obj)[8:10]
    except:
        date = '000000'
    else:
        date = '000000'

    return date

def write_record(source, sourcecount, thisdate, thisURL, story, fout):
    """
    Function to write TABARI-formatted story record.

    Parameters
    ------
    source: String
            Story news source.

    sourcecount: Dictionary
            Dictionary of counts of stories by source

    thisdate: String
            Date string in the form YYMMDD

    thisURL: String
            String of URL to story at source

    story: String
            News article story

    fout: file
            File for TABARI-formatted output
    """
    if source in sourcecount:  # count of the stories by source
        sourcecount[source] += 1
    else:
        sourcecount[source] = 1

    sentlist = sentence_segmenter(story)

    nsent = 1
    for sent in sentlist:
        if sent[0] != '"':  # skip sentences beginning with quotes
            print thisdate, source, thisURL
           # print >> fout, thisdate + ' ' + source + '-' + \
           #     str(sourcecount[source]).zfill(4) + '-' + \
           #     str(nsent) + ' ' + thisURL  # + '\n'
        print >> fout, thisdate + ' ' + thisURL
            lines = textwrap.wrap(sent, 80)
            for txt in lines:
                print >> fout, txt

            print >> fout, '\n'

        nsent += 1
        if nsent > MAX_LEDE:
            break

def sentence_segmenter(paragr):
    """
    Function to break a string 'paragraph' into a list of sentences based on the following rules:

    1. Look for terminal [.,?,!] followed by a space and [A-Z]
    2. If ., check against abbreviation list ABBREV_LIST: Get the string between the . and the
       previous blank, lower-case it, and see if it is in the list. Also check for single-
       letter initials. If true, continue search for terminal punctuation
    3. Extend selection to balance (...) and "...". Reapply termination rules
    4. Add to sentlist if the length of the string is between MIN_SENTLENGTH and MAX_SENTLENGTH
    5. Returns sentlist

    Parameters
    ------
    paragr: String

    """
#   ka = 0
#   print '\nSentSeg-Mk1'
# sentence termination pattern used in sentence_segmenter(paragr)
    terpat = re.compile('[\.\?!]\s+[A-Z\"]')

    # source: LbjNerTagger1.11.release/Data/KnownLists/known_title.lst from
    # University of Illinois with editing
    ABBREV_LIST = [
        'mrs.', 'ms.', 'mr.', 'dr.', 'gov.', 'sr.', 'rev.', 'r.n.', 'pres.',
        'treas.', 'sect.', 'maj.', 'ph.d.', 'ed. psy.', 'proc.', 'fr.', 'asst.', 'p.f.c.', 'prof.',
        'admr.', 'engr.', 'mgr.', 'supt.', 'admin.', 'assoc.', 'voc.', 'hon.', 'm.d.', 'dpty.',
        'sec.', 'capt.', 'c.e.o.', 'c.f.o.', 'c.i.o.', 'c.o.o.', 'c.p.a.', 'c.n.a.', 'acct.',
        'llc.', 'inc.', 'dir.', 'esq.', 'lt.', 'd.d.', 'ed.', 'revd.', 'psy.d.', 'v.p.',
        'senr.', 'gen.', 'prov.', 'cmdr.', 'sgt.', 'sen.', 'col.', 'lieut.', 'cpl.', 'pfc.',
        'k.p.h.', 'cent.', 'deg.', 'doz.', 'Fahr.', 'Cel.', 'F.', 'C.', 'K.', 'ft.', 'fur.',
        'gal.', 'gr.', 'in.', 'kg.', 'km.', 'kw.', 'l.', 'lat.', 'lb.', 'lb per sq in.',
        'long.', 'mg.', 'mm.,, m.p.g.', 'm.p.h.', 'cc.', 'qr.', 'qt.', 'sq.', 't.', 'vol.',
        'w.', 'wt.']

    sentlist = []
    searchstart = 0  # controls skipping over non-terminal conditions
    terloc = terpat.search(paragr)
    while terloc:
#       print 'Mk2-0:', paragr[:terloc.start()+2]
        isok = True
        if paragr[terloc.start()] == '.':
            if (paragr[terloc.start() - 1].isupper() and
                    paragr[terloc.start() - 2] == ' '):
                        isok = False      # single initials
            else:
                # check abbreviations
                loc = paragr.rfind(' ', 0, terloc.start() - 1)
                if loc > 0:
#                   print 'SentSeg-Mk1: checking',paragr[loc+1:terloc.start()+1]
                    if paragr[loc + 1:terloc.start() + 1].lower() in ABBREV_LIST:
#                       print 'SentSeg-Mk2: found',paragr[loc+1:terloc.start()+1]
                        isok = False
        if paragr[:terloc.start()].count('(') != paragr[:terloc.start()].count(')'):
#           print 'SentSeg-Mk2: unbalanced ()'
            isok = False
        if paragr[:terloc.start()].count('"') % 2 != 0  :
#           print 'SentSeg-Mk2: unbalanced ""'
            isok = False
        if isok:
            if (len(paragr[:terloc.start()]) > MIN_SENTLENGTH and
                    len(paragr[:terloc.start()]) < MAX_SENTLENGTH):
                sentlist.append(paragr[:terloc.start() + 2])
#               print 'SentSeg-Mk3: added',paragr[:terloc.start()+2]
            paragr = paragr[terloc.end() - 1:]
            searchstart = 0
        else:
            searchstart = terloc.start() + 2

#       print 'SentSeg-Mk4:',paragr[:64]
#       print '            ',paragr[searchstart:searchstart+64]
        terloc = terpat.search(paragr, searchstart)
#       ka += 1
#       if ka > 16: sys.exit()

    # add final sentence
    if (len(paragr) > MIN_SENTLENGTH and len(paragr) < MAX_SENTLENGTH):
        sentlist.append(paragr)

    return sentlist

def get_source(field):
    """
    Function to extract source from URL in story header.

    Parameters
    ------
    field: String
            Story header.

    Returns
    -------

    source : String
            3 char source code from sources dictionary
    """
    source_url = field[-1].strip()
    sourced = source_url.split('http://')[-1]

    try:
        key = [i for i, s in enumerate(sources.keys()) if s in sourced]
        if key:
            source = sources[sources.keys()[key[0]]]
        else:
            source = '999'
    except AttributeError:
        source = '999'

    return source

def get_story(story_all):
    """
    Function to extract story text without date and source line.

    Parameters
    ------
    story_all: String
            All story text.

    Returns

    -------

    story : String
            3 char source code from sources dictionary
    """

    if '(Reuters)' in story_all:
        story = story_all[story_all.find('(Reuters)') + 12:]
    elif '(IANS)' in story_all:
        story = story_all[story_all.find('(IANS)') + 7:]
    elif '(ANI)' in story_all:
        story = story_all[story_all.find('(ANI)') + 7:]
    elif '(Xinhua) -- ' in story_all:
        story = story_all[story_all.find('(Xinhua) -- ') + 12:]
    elif '(UPI) -- ' in story_all:
        story = story_all[story_all.find('(UPI) -- ') + 9:]
    if bool(re.search("\xe2\x80\x93", story_all[0:32])):
        story = story_all.split("\xe2\x80\x93 ", 1)[1]
    else:
        story = story_all

    return story

def main(thisday):
    """
    Main function to parse scraper_results to TABARI-formatted output.
    """
    server_list, file_list =  phox_utilities.parse_config('PHOX_config.ini')
    scraperfilename = file_list.scraper_stem + thisday + '.txt'
    print "Mongo: Scraper file name:", scraperfilename

    recordfilename = file_list.recordfile_stem + thisday + '.txt'
    print "Mongo: Record file name:", recordfilename

    newsourcefile = newsourcestem + thisday + '.txt'
    print "Mongo: New Sources file name:", newsourcefile

    try:
        fin = open(scraperfilename, 'r')
    except IOError:
        print scraperfilename
    phox_utilities.do_RuntimeError(
            'Could not find the scraper file for',
            thisday)

    finlist = fin.readlines()
    fout = open(recordfilename, 'w')
    newout = open(newsourcefile, 'w')
    sourcecount = {}

    #storyno = 1
    #csno = 1

    for line in range(0, len(finlist)):
        if '\thttp' in finlist[line]:
            field = finlist[line].split('\t')
        thisURL = field[2][:-1]
            # temporary to accommodate TABARI input limits
            thisURL = thisURL[:MAX_URLLENGTH]

            thisstory = get_story(finlist[line + 1])
            thisdate = get_date(field)
            thissource = get_source(field)

            if thissource == '999':
                # Adds sources not included in sources dictionary to
                # 'newsource_results_20..' file output
                print >> newout, thisURL

            write_record(
                thissource,
                sourcecount,
                thisdate,
                thisURL,
                thisstory,
                fout)

    fin.close()
    fout.close()
    print "Finished"

if __name__ == '__main__':
    if len(sys.argv) > 2:  # initializations for stand-alone tests
        phox_utilities.init_logger('test_pipeline.log')
        logger = phox_utilities.logger  # get a local copy for the pipeline
        # initialize the various phox_utilities globals
        phox_utilities.parse_config('test_config.ini')

    if len(sys.argv) > 1:
        thisday = sys.argv[1]
    else:
        phox_utilities.do_RuntimeError('No date suffix in Mongo.formatter.py')

    main(thisday)
<commit_msg>some tabs<commit_after>import re
import sys
import textwrap
import phox_utilities
from dateutil import parser

MAX_LEDE = 4   # number of sentences to output
# this is relatively high because we are only looking for sentences that
# will have subject and object
MIN_SENTLENGTH = 100
MAX_SENTLENGTH = 512

MAX_URLLENGTH = 192   # temporary to accommodate TABARI input limits

newsourcestem = 'newsources.'

sources = {
    'csmonitor.com': 'CSM',
    'bbc.co.uk': 'BBC',
    'reuters.com': 'REU',
    'xinhuanet.com': 'XIN',
    'www.upi.com': 'UPI',
    'nytimes.com': 'NYT',
    'todayszaman.com': 'TZA',
    'hosted2.ap.org': 'APP',
    'theguardian.com': 'GUA',
    'todayszaman.com': 'TZA',
    'insightcrime.org': 'ISC',
    'france24.com': 'FRA',
    'yahoo.com': 'YAH',
    'allafrica.com': 'ALA',
    'voanews.com': 'VOA',
    'aljazeera.com': 'AJZ',
    'AlAkhbarEnglish': 'AKB',
    'usatoday.com': 'USA',
    'latimes.com': 'LAT',
    'foxnews.com': 'FOX',
    'IRINnews.org': 'IRI',
    'rfi.fr': 'RFI',
    'cnn.com': 'CNN',
    'abcnews': 'ABC',
    'wsj.com': 'WSJ',
    'nydailynews.com': 'NYD',
    'washingtonpost.com': 'WAS',
    'chicagotribune.com': 'CHT',
    'miamiherald.com':'MIA',
    'thestar.com.my':'STA',
    'al-monitor.com':'MON',
    'un.org':'UNN',
    'pakistantelegraph.com':'PAK',
    'africaleader.com':'AFR',
    'theuknews.com':'UKN',
    'indiagazette.com':'ING',
    'mexicostar.com':'MES',
    'japanherald.com':'JAH',
    'chinanationalnews.com':'CHN',
    'jpost.com':'JPO',
    'middleeaststar.com':'MEA',
    'beijingbulletin.com':'BEB',
    'arabherald.com':'ARH',
    'russiaherald.com':'RUH',
    'iraqsun.com':'IRA',
    'malaysiasun.com':'MAL',
    'thehindu.com':'HIN',
    'shanghaisun.com':'SHA',
    'dawn.com':'DAW',
    'singaporestar.com':'SNG',
    'southeastasiapost.com':'SEA',
    'afghanistansun.com':'AFG',
    'australianherald.com':'AUS',
    'thenational.ae':'NAT',
    'europesun.com':'EUS',
    'greekherald.com':'GKH',
    'irishsun.com':'IRS',
    'iranherald.com':'IAH',
    'parisguardian.com':'PAG',
    'brazilsun.com':'BZS',
    'asiabulletin.com':'ASB',
    'nigeriasun.com':'NIG',
    'haitisun.com':'HAI',
    'philippinetimes.com':'PHI',
    'venezuelastar.com':'VEN',
    'caribbeanherald.com':'CAR',
    'zimbabwestar.com':'ZIM',
    'torontotelegraph.com':'TOR',
    'bangladeshsun.com':'BAN',
    'sydneysun.com':'SYD',
    'menafn.com':'MEN',
    'thelocal.se':'TLO',
    'nepalnational.com':'NEP',
    'kenyastar.com':'KEN',
    'sierraleonetimes.com':'SIE',
    'hurriyetdailynews.com':'HDN',
    'pajhwok.com':'PAJ',
    'thelocal.it':'LIT',
    'thelocal.fr':'LFR',
    'thelocal.es':'LES',
    'thelocal.ch':'LCH',
    'ipsnews.net':'IPS',
    'jamaicantimes.com':'JAM',
    'argentinastar.com':'ARG',
    'srilankasource.com':'SRI',
    'centralasiatimes.com':'CEN',
    'thelocal.no':'LNO',
    'theeastafrican.co.ke':'ESF',
    'taiwansun.com':'TAI',
    'cambodiantimes.com':'CAM',
    'payvand.com':'PAY',
    'trinidadtimes.com':'TRD',
    'yementimes.com':'YEM',
    'hongkongherald.com':'HNK'}

def get_date(field):
    """
    Function to extract date from story header in scraper_results.

    Parameters
    ------
    field: String
            Story header.

    Returns
    -------

    date : String
            Date string in the form YYMMDD
    """
    if 'csmonitor.com' in field[-1]:
        csmdate = field[-1].split('/20')[1].split('/')
        date = csmdate[0] + csmdate[1]
    elif field[1]:
        try:
            date_obj = parser.parse(field[1])
            date = str(date_obj)[2:4] + str(date_obj)[5:7] + str(date_obj)[8:10]
        except:
            date = '000000'
    else:
        date = '000000'

    return date

def write_record(source, sourcecount, thisdate, thisURL, story, fout):
    """
    Function to write TABARI-formatted story record.

    Parameters
    ------
    source: String
            Story news source.

    sourcecount: Dictionary
            Dictionary of counts of stories by source

    thisdate: String
            Date string in the form YYMMDD

    thisURL: String
            String of URL to story at source

    story: String
            News article story

    fout: file
            File for TABARI-formatted output
    """
    if source in sourcecount:  # count of the stories by source
        sourcecount[source] += 1
    else:
        sourcecount[source] = 1

    sentlist = sentence_segmenter(story)

    nsent = 1
    for sent in sentlist:
        if sent[0] != '"':  # skip sentences beginning with quotes
            print thisdate, source, thisURL
           # print >> fout, thisdate + ' ' + source + '-' + \
           #     str(sourcecount[source]).zfill(4) + '-' + \
           #     str(nsent) + ' ' + thisURL  # + '\n'
            print >> fout, thisdate + ' ' + thisURL
            lines = textwrap.wrap(sent, 80)
            for txt in lines:
                print >> fout, txt

            print >> fout, '\n'

        nsent += 1
        if nsent > MAX_LEDE:
            break

def sentence_segmenter(paragr):
    """
    Function to break a string 'paragraph' into a list of sentences based on the following rules:

    1. Look for terminal [.,?,!] followed by a space and [A-Z]
    2. If ., check against abbreviation list ABBREV_LIST: Get the string between the . and the
       previous blank, lower-case it, and see if it is in the list. Also check for single-
       letter initials. If true, continue search for terminal punctuation
    3. Extend selection to balance (...) and "...". Reapply termination rules
    4. Add to sentlist if the length of the string is between MIN_SENTLENGTH and MAX_SENTLENGTH
    5. Returns sentlist

    Parameters
    ------
    paragr: String

    """
#   ka = 0
#   print '\nSentSeg-Mk1'
# sentence termination pattern used in sentence_segmenter(paragr)
    terpat = re.compile('[\.\?!]\s+[A-Z\"]')

    # source: LbjNerTagger1.11.release/Data/KnownLists/known_title.lst from
    # University of Illinois with editing
    ABBREV_LIST = [
        'mrs.', 'ms.', 'mr.', 'dr.', 'gov.', 'sr.', 'rev.', 'r.n.', 'pres.',
        'treas.', 'sect.', 'maj.', 'ph.d.', 'ed. psy.', 'proc.', 'fr.', 'asst.', 'p.f.c.', 'prof.',
        'admr.', 'engr.', 'mgr.', 'supt.', 'admin.', 'assoc.', 'voc.', 'hon.', 'm.d.', 'dpty.',
        'sec.', 'capt.', 'c.e.o.', 'c.f.o.', 'c.i.o.', 'c.o.o.', 'c.p.a.', 'c.n.a.', 'acct.',
        'llc.', 'inc.', 'dir.', 'esq.', 'lt.', 'd.d.', 'ed.', 'revd.', 'psy.d.', 'v.p.',
        'senr.', 'gen.', 'prov.', 'cmdr.', 'sgt.', 'sen.', 'col.', 'lieut.', 'cpl.', 'pfc.',
        'k.p.h.', 'cent.', 'deg.', 'doz.', 'Fahr.', 'Cel.', 'F.', 'C.', 'K.', 'ft.', 'fur.',
        'gal.', 'gr.', 'in.', 'kg.', 'km.', 'kw.', 'l.', 'lat.', 'lb.', 'lb per sq in.',
        'long.', 'mg.', 'mm.,, m.p.g.', 'm.p.h.', 'cc.', 'qr.', 'qt.', 'sq.', 't.', 'vol.',
        'w.', 'wt.']

    sentlist = []
    searchstart = 0  # controls skipping over non-terminal conditions
    terloc = terpat.search(paragr)
    while terloc:
#       print 'Mk2-0:', paragr[:terloc.start()+2]
        isok = True
        if paragr[terloc.start()] == '.':
            if (paragr[terloc.start() - 1].isupper() and
                    paragr[terloc.start() - 2] == ' '):
                        isok = False      # single initials
            else:
                # check abbreviations
                loc = paragr.rfind(' ', 0, terloc.start() - 1)
                if loc > 0:
#                   print 'SentSeg-Mk1: checking',paragr[loc+1:terloc.start()+1]
                    if paragr[loc + 1:terloc.start() + 1].lower() in ABBREV_LIST:
#                       print 'SentSeg-Mk2: found',paragr[loc+1:terloc.start()+1]
                        isok = False
        if paragr[:terloc.start()].count('(') != paragr[:terloc.start()].count(')'):
#           print 'SentSeg-Mk2: unbalanced ()'
            isok = False
        if paragr[:terloc.start()].count('"') % 2 != 0  :
#           print 'SentSeg-Mk2: unbalanced ""'
            isok = False
        if isok:
            if (len(paragr[:terloc.start()]) > MIN_SENTLENGTH and
                    len(paragr[:terloc.start()]) < MAX_SENTLENGTH):
                sentlist.append(paragr[:terloc.start() + 2])
#               print 'SentSeg-Mk3: added',paragr[:terloc.start()+2]
            paragr = paragr[terloc.end() - 1:]
            searchstart = 0
        else:
            searchstart = terloc.start() + 2

#       print 'SentSeg-Mk4:',paragr[:64]
#       print '            ',paragr[searchstart:searchstart+64]
        terloc = terpat.search(paragr, searchstart)
#       ka += 1
#       if ka > 16: sys.exit()

    # add final sentence
    if (len(paragr) > MIN_SENTLENGTH and len(paragr) < MAX_SENTLENGTH):
        sentlist.append(paragr)

    return sentlist

def get_source(field):
    """
    Function to extract source from URL in story header.

    Parameters
    ------
    field: String
            Story header.

    Returns
    -------

    source : String
            3 char source code from sources dictionary
    """
    source_url = field[-1].strip()
    sourced = source_url.split('http://')[-1]

    try:
        key = [i for i, s in enumerate(sources.keys()) if s in sourced]
        if key:
            source = sources[sources.keys()[key[0]]]
        else:
            source = '999'
    except AttributeError:
        source = '999'

    return source

def get_story(story_all):
    """
    Function to extract story text without date and source line.

    Parameters
    ------
    story_all: String
            All story text.

    Returns

    -------

    story : String
            3 char source code from sources dictionary
    """

    if '(Reuters)' in story_all:
        story = story_all[story_all.find('(Reuters)') + 12:]
    elif '(IANS)' in story_all:
        story = story_all[story_all.find('(IANS)') + 7:]
    elif '(ANI)' in story_all:
        story = story_all[story_all.find('(ANI)') + 7:]
    elif '(Xinhua) -- ' in story_all:
        story = story_all[story_all.find('(Xinhua) -- ') + 12:]
    elif '(UPI) -- ' in story_all:
        story = story_all[story_all.find('(UPI) -- ') + 9:]
    if bool(re.search("\xe2\x80\x93", story_all[0:32])):
        story = story_all.split("\xe2\x80\x93 ", 1)[1]
    else:
        story = story_all

    return story

def main(thisday):
    """
    Main function to parse scraper_results to TABARI-formatted output.
    """
    server_list, file_list =  phox_utilities.parse_config('PHOX_config.ini')
    scraperfilename = file_list.scraper_stem + thisday + '.txt'
    print "Mongo: Scraper file name:", scraperfilename

    recordfilename = file_list.recordfile_stem + thisday + '.txt'
    print "Mongo: Record file name:", recordfilename

    newsourcefile = newsourcestem + thisday + '.txt'
    print "Mongo: New Sources file name:", newsourcefile

    try:
        fin = open(scraperfilename, 'r')
    except IOError:
        print scraperfilename
        phox_utilities.do_RuntimeError(
            'Could not find the scraper file for',
            thisday)

    finlist = fin.readlines()
    fout = open(recordfilename, 'w')
    newout = open(newsourcefile, 'w')
    sourcecount = {}

    #storyno = 1
    #csno = 1

    for line in range(0, len(finlist)):
        if '\thttp' in finlist[line]:
            field = finlist[line].split('\t')
            thisURL = field[2][:-1]
            # temporary to accommodate TABARI input limits
            thisURL = thisURL[:MAX_URLLENGTH]

            thisstory = get_story(finlist[line + 1])
            thisdate = get_date(field)
            thissource = get_source(field)

            if thissource == '999':
                # Adds sources not included in sources dictionary to
                # 'newsource_results_20..' file output
                print >> newout, thisURL

            write_record(
                thissource,
                sourcecount,
                thisdate,
                thisURL,
                thisstory,
                fout)

    fin.close()
    fout.close()
    print "Finished"

if __name__ == '__main__':
    if len(sys.argv) > 2:  # initializations for stand-alone tests
        phox_utilities.init_logger('test_pipeline.log')
        logger = phox_utilities.logger  # get a local copy for the pipeline
        # initialize the various phox_utilities globals
        phox_utilities.parse_config('test_config.ini')

    if len(sys.argv) > 1:
        thisday = sys.argv[1]
    else:
        phox_utilities.do_RuntimeError('No date suffix in Mongo.formatter.py')

    main(thisday)
<|endoftext|>


<commit_before>"""
    TODO move permission checks to the dispatch view thingee
"""

from datetime import datetime

from django.conf import settings
from django.core.urlresolvers import reverse
from django.forms import widgets
from django.forms.widgets import DateTimeInput, CheckboxInput
from django.http import HttpResponseForbidden, HttpResponseRedirect
from django.views.generic.edit import DeletionMixin
from django.views.generic import DetailView
from django.views.generic import ListView
from django.views.generic import TemplateView
from django.views.generic.edit import FormView

from mongoengine.fields import EmbeddedDocumentField, ListField

from mongonaut.forms import DocumentListForm
from mongonaut.forms import DocumentDetailForm
from mongonaut.forms import document_detail_form_factory
from mongonaut.mixins import MongonautViewMixin
from mongonaut.utils import is_valid_object_id

class IndexView(ListView, MongonautViewMixin):

    template_name = "mongonaut/index.html"
    queryset = []

    def get_queryset(self):
        return self.get_mongoadmins()

class AppListView(ListView, MongonautViewMixin):
    """ :args: <app_label> """

    template_name = "mongonaut/app_list.html"

class DocumentListView(FormView, MongonautViewMixin):
    """ :args: <app_label> <document_name>

        TODO - Make a generic document fetcher method
    """
    form_class = DocumentListForm
    success_url = '/'
    template_name = "mongonaut/document_list.html"

    #def dispatch(self, *args, **kwargs):
    #    self.set_mongoadmin()
    #    self.set_permissions()
    #    return super(DocumentListView, self).dispatch(*args, **kwargs)

    def get_qset(self, queryset, q):
        if self.mongoadmin.search_fields and q:
            params = {}
            for field in self.mongoadmin.search_fields:
                if field == 'id':
                    # check to make sure this is a valid ID, otherwise we just continue
                    if is_valid_object_id(q):
                        return queryset.filter(id=q)
                    continue
                search_key = "{field}__icontains".format(field=field)
                params[search_key] = q

            queryset = queryset.filter(**params)
        return queryset

    def get_queryset(self):
        if hasattr(self, "queryset") and self.queryset:
            return self.queryset

        self.set_mongonaut_base()
        self.set_mongoadmin()
        self.document = getattr(self.models, self.document_name)
        queryset = self.document.objects.all()

        # search. move this to get_queryset
        # search. move this to get_queryset
        q = self.request.GET.get('q')
        query = self.get_qset(queryset, q)

        ### Start pagination
        ### Note:
        ###    Didn't use the Paginator in Django cause mongoengine querysets are
        ###    not the same as Django ORM querysets and it broke.
        # Make sure page request is an int. If not, deliver first page.
        try:
            page = int(self.request.GET.get('page', '1'))
        except ValueError:
            page = 1
        self.page = page

        self.documents_per_page = 25
        self.total_pages = max(queryset.count() / self.documents_per_page, 1)
        start = (self.page -1) * self.documents_per_page
        end = self.page * self.documents_per_page
        self.previous_page_number = page - 1
        self.next_page_number = page + 1
        try:
            queryset = queryset[start:end]
        except Exception as e:
            print e

        self.queryset = queryset
        return queryset

    def get_initial(self):
        self.query = self.get_queryset()
        mongo_ids = {'mongo_id':[unicode(x.id) for x in self.query]}
        return mongo_ids

    def get_context_data(self, **kwargs):
        context = super(DocumentListView, self).get_context_data(**kwargs)
        context = self.set_permissions_in_context(context)
        if not context['has_view_permission']:
            return HttpResponseForbidden("You do not have permissions to view this content.")

        context['object_list'] = self.get_queryset()
        context['document'] = self.document
        context['app_label'] = self.app_label
        context['document_name'] = self.document_name

        # pagination bits
        context['page'] = self.page
        context['documents_per_page'] = self.documents_per_page
        context['previous_page_number'] = self.previous_page_number
        context['next_page_number'] = self.next_page_number
        context['total_pages'] = self.total_pages

        # Part of upcoming list view form functionality
        if self.queryset.count():
            context['keys'] = ['id',]

            # Show those items for which we've got list_fields on the mongoadmin
            for key in [x for x in self.document._fields.keys() if x != 'id' and x in self.mongoadmin.list_fields]:

                # TODO - Figure out why this EmbeddedDocumentField and ListField breaks this view
                # Note - This is the challenge part, right? :)
                if isinstance(self.document._fields[key], EmbeddedDocumentField):
                    continue
                if isinstance(self.document._fields[key], ListField):
                    continue
                context['keys'].append(key)

        if self.mongoadmin.search_fields:
            context['search_field'] = True

        return context

    def post(self, request, *args, **kwargs):
        # TODO - make sure to check the rights of the poster
        #self.get_queryset() # TODO - write something that grabs the document class better
        form_class = self.get_form_class()
        form = self.get_form(form_class)
        mongo_ids = self.get_initial()['mongo_id']
        for form_mongo_id in form.data.getlist('mongo_id'):
            for mongo_id in mongo_ids:
                if form_mongo_id == mongo_id:
                    self.document.objects.get(id=mongo_id).delete()

        return self.form_invalid(form)

class DocumentDetailView(TemplateView, MongonautViewMixin):
    """ :args: <app_label> <document_name> <id> """
    template_name = "mongonaut/document_detail.html"

    def get_context_data(self, **kwargs):
        context = super(DocumentDetailView, self).get_context_data(**kwargs)
        self.set_mongoadmin()
        context = self.set_permissions_in_context(context)
        if not context['has_view_permission']:
            return HttpResponseForbidden("You do not have permissions to view this content.")
        self.document_type = getattr(self.models, self.document_name)
        self.ident = self.kwargs.get('id')
        self.document = self.document_type.objects.get(id=self.ident)

        context['document'] = self.document
        context['app_label'] = self.app_label
        context['document_name'] = self.document_name
        context['keys'] = ['id',]
        for key in sorted([x for x in self.document._fields.keys() if x != 'id']):
            # TODO - Figure out why this EmbeddedDocumentField and ListField breaks this view
            # Note - This is the challenge part, right? :)
            if isinstance(self.document._fields[key], EmbeddedDocumentField):
                continue
            if isinstance(self.document._fields[key], ListField):
                continue
            context['keys'].append(key)

        return context

class DocumentDetailEditFormView(FormView, MongonautViewMixin):
    """ :args: <app_label> <document_name> <id> """#

    template_name = "mongonaut/document_edit_form.html"
    form_class = DocumentDetailForm
    success_url = '/'

    def get_success_url(self):
        self.set_mongonaut_base()
        return reverse('document_detail_edit_form', kwargs={'app_label':self.app_label,'document_name':self.document_name,'id':self.kwargs.get('id')})

    def get_context_data(self, **kwargs):
        context = super(DocumentDetailEditFormView, self).get_context_data(**kwargs)
        self.set_mongoadmin()
        context = self.set_permissions_in_context(context)
        if not context['has_edit_permission']:
            return HttpResponseForbidden("You do not have permissions to edit this content.")
        self.document_type = getattr(self.models, self.document_name)
        self.ident = self.kwargs.get('id')
        self.document = self.document_type.objects.get(id=self.ident)

        context['document'] = self.document
        context['app_label'] = self.app_label
        context['document_name'] = self.document_name

        return context

    def get_form(self, DocumentDetailForm):
        self.set_mongoadmin()
        context = self.set_permissions_in_context({})
        if not context['has_edit_permission']:
            return HttpResponseForbidden("You do not have permissions to edit this content.")

        self.document_type = getattr(self.models, self.document_name)
        self.ident = self.kwargs.get('id')
        self.document = self.document_type.objects.get(id=self.ident)
        self.form = DocumentDetailForm()

        self.form = document_detail_form_factory(form=self.form, document_type=self.document_type, initial=self.document)
        if self.request.method == 'POST':
            self.form.data = self.request.POST
            self.form.is_bound = True
            if self.form.is_valid():
                for key, field in self.form.fields.items():
                    if 'readonly' in field.widget.attrs:
                        # For _id or things specified as such
                        continue

                    if isinstance(field.widget, DateTimeInput):
                        format = field.widget.format
                        setattr(self.document, key, datetime.strptime(self.request.POST[key], format))
                        continue

                    if isinstance(field.widget, CheckboxInput):
                        if key in self.request.POST:
                            setattr(self.document, key, True)
                        else:
                            setattr(self.document, key, False)
                        continue

                    if isinstance(field.widget, widgets.Select):
                        # supporting reference fields!
                        # TODO - support ListFields
                        value = field.mongofield.document_type.objects.get(id=self.request.POST[key])
                        setattr(self.document, key, value)
                        continue

                    # for strings
                    setattr(self.document, key, self.request.POST[key])

                self.document.save()
                # TODO add message for save

        return self.form

class DocumentDetailAddFormView(FormView, MongonautViewMixin):
    """ :args: <app_label> <document_name> <id> """#

    template_name = "mongonaut/document_add_form.html"
    form_class = DocumentDetailForm
    success_url = '/'

    def get_success_url(self):
        self.set_mongonaut_base()
        return reverse('document_detail', kwargs={'app_label':self.app_label,'document_name':self.document_name,'id':str(self.document.id)})

    def get_context_data(self, **kwargs):
        """ TODO - possibly inherit this from DocumentDetailEditFormView. This is same thing minus:
            self.ident = self.kwargs.get('id')
            self.document = self.document_type.objects.get(id=self.ident)
        """
        context = super(DocumentDetailAddFormView, self).get_context_data(**kwargs)
        self.set_mongoadmin()
        context = self.set_permissions_in_context(context)
        if not context['has_add_permission']:
            return HttpResponseForbidden("You do not have permissions to edit view content.")
        self.document_type = getattr(self.models, self.document_name)

        context['app_label'] = self.app_label
        context['document_name'] = self.document_name
        return context

    def get_form(self, DocumentDetailForm):
        self.set_mongonaut_base()
        self.document_type = getattr(self.models, self.document_name)
        self.form = DocumentDetailForm()
        self.form = document_detail_form_factory(form=self.form, document_type=self.document_type)
        if self.request.method == 'POST':
            self.form.data = self.request.POST
            self.form.is_bound = True
            if self.form.is_valid():
                self.document = self.document_type()
                for key, field in self.form.fields.items():
                    if 'readonly' in field.widget.attrs:
                        # For _id
                        # TODO - make the ones below work
                        # for ReferenceField - like <class 'articles.models.User'> on Blog
                        # For ListField - like 'field': <mongoengine.fields.StringField object at 0x101b51810>,
                        # For EmbeddedDocumentField
                        continue

                    if isinstance(field.widget, DateTimeInput):
                        format = field.widget.format
                        setattr(self.document, key, datetime.strptime(self.request.POST[key], format))
                        continue

                    if isinstance(field.widget, CheckboxInput):
                        if key in self.request.POST:
                            setattr(self.document, key, True)
                        else:
                            setattr(self.document, key, False)
                        continue

                    # for strings
                    setattr(self.document, key, self.request.POST[key])
                self.document.save()
                # TODO add message for save

        return self.form

class DocumentDeleteView(TemplateView, DeletionMixin, MongonautViewMixin):
    """ :args: <app_label> <document_name> <id>

        TODO - implement a GET view for confirmation
    """

    success_url = "/"
    template_name = "mongonaut/document_delete.html"

    def get_success_url(self):
        self.set_mongonaut_base()
        return reverse('document_list', kwargs={'app_label':self.app_label,'document_name':self.document_name})

    def get_object(self):
        self.set_mongoadmin()
        self.document_type = getattr(self.models, self.document_name)
        self.ident = self.kwargs.get('id')
        self.document = self.document_type.objects.get(id=self.ident)
        return self.document

class EmbeddedDocumentDetailView(DetailView, MongonautViewMixin):
    """ :args: <app_label> <document_name> <id> <???> """
    template_name = "mongonaut/embedded_document_detail.html"<commit_msg>Fix the reference field in the add form #8<commit_after>"""
    TODO move permission checks to the dispatch view thingee
"""

from datetime import datetime

from django.conf import settings
from django.core.urlresolvers import reverse
from django.forms import widgets
from django.forms.widgets import DateTimeInput, CheckboxInput
from django.http import HttpResponseForbidden, HttpResponseRedirect
from django.views.generic.edit import DeletionMixin
from django.views.generic import DetailView
from django.views.generic import ListView
from django.views.generic import TemplateView
from django.views.generic.edit import FormView

from mongoengine.fields import EmbeddedDocumentField, ListField

from mongonaut.forms import DocumentListForm
from mongonaut.forms import DocumentDetailForm
from mongonaut.forms import document_detail_form_factory
from mongonaut.mixins import MongonautViewMixin
from mongonaut.utils import is_valid_object_id

class IndexView(ListView, MongonautViewMixin):

    template_name = "mongonaut/index.html"
    queryset = []

    def get_queryset(self):
        return self.get_mongoadmins()

class AppListView(ListView, MongonautViewMixin):
    """ :args: <app_label> """

    template_name = "mongonaut/app_list.html"

class DocumentListView(FormView, MongonautViewMixin):
    """ :args: <app_label> <document_name>

        TODO - Make a generic document fetcher method
    """
    form_class = DocumentListForm
    success_url = '/'
    template_name = "mongonaut/document_list.html"

    #def dispatch(self, *args, **kwargs):
    #    self.set_mongoadmin()
    #    self.set_permissions()
    #    return super(DocumentListView, self).dispatch(*args, **kwargs)

    def get_qset(self, queryset, q):
        if self.mongoadmin.search_fields and q:
            params = {}
            for field in self.mongoadmin.search_fields:
                if field == 'id':
                    # check to make sure this is a valid ID, otherwise we just continue
                    if is_valid_object_id(q):
                        return queryset.filter(id=q)
                    continue
                search_key = "{field}__icontains".format(field=field)
                params[search_key] = q

            queryset = queryset.filter(**params)
        return queryset

    def get_queryset(self):
        if hasattr(self, "queryset") and self.queryset:
            return self.queryset

        self.set_mongonaut_base()
        self.set_mongoadmin()
        self.document = getattr(self.models, self.document_name)
        queryset = self.document.objects.all()

        # search. move this to get_queryset
        # search. move this to get_queryset
        q = self.request.GET.get('q')
        query = self.get_qset(queryset, q)

        ### Start pagination
        ### Note:
        ###    Didn't use the Paginator in Django cause mongoengine querysets are
        ###    not the same as Django ORM querysets and it broke.
        # Make sure page request is an int. If not, deliver first page.
        try:
            page = int(self.request.GET.get('page', '1'))
        except ValueError:
            page = 1
        self.page = page

        self.documents_per_page = 25
        self.total_pages = max(queryset.count() / self.documents_per_page, 1)
        start = (self.page -1) * self.documents_per_page
        end = self.page * self.documents_per_page
        self.previous_page_number = page - 1
        self.next_page_number = page + 1
        try:
            queryset = queryset[start:end]
        except Exception as e:
            print e

        self.queryset = queryset
        return queryset

    def get_initial(self):
        self.query = self.get_queryset()
        mongo_ids = {'mongo_id':[unicode(x.id) for x in self.query]}
        return mongo_ids

    def get_context_data(self, **kwargs):
        context = super(DocumentListView, self).get_context_data(**kwargs)
        context = self.set_permissions_in_context(context)
        if not context['has_view_permission']:
            return HttpResponseForbidden("You do not have permissions to view this content.")

        context['object_list'] = self.get_queryset()
        context['document'] = self.document
        context['app_label'] = self.app_label
        context['document_name'] = self.document_name

        # pagination bits
        context['page'] = self.page
        context['documents_per_page'] = self.documents_per_page
        context['previous_page_number'] = self.previous_page_number
        context['next_page_number'] = self.next_page_number
        context['total_pages'] = self.total_pages

        # Part of upcoming list view form functionality
        if self.queryset.count():
            context['keys'] = ['id',]

            # Show those items for which we've got list_fields on the mongoadmin
            for key in [x for x in self.document._fields.keys() if x != 'id' and x in self.mongoadmin.list_fields]:

                # TODO - Figure out why this EmbeddedDocumentField and ListField breaks this view
                # Note - This is the challenge part, right? :)
                if isinstance(self.document._fields[key], EmbeddedDocumentField):
                    continue
                if isinstance(self.document._fields[key], ListField):
                    continue
                context['keys'].append(key)

        if self.mongoadmin.search_fields:
            context['search_field'] = True

        return context

    def post(self, request, *args, **kwargs):
        # TODO - make sure to check the rights of the poster
        #self.get_queryset() # TODO - write something that grabs the document class better
        form_class = self.get_form_class()
        form = self.get_form(form_class)
        mongo_ids = self.get_initial()['mongo_id']
        for form_mongo_id in form.data.getlist('mongo_id'):
            for mongo_id in mongo_ids:
                if form_mongo_id == mongo_id:
                    self.document.objects.get(id=mongo_id).delete()

        return self.form_invalid(form)

class DocumentDetailView(TemplateView, MongonautViewMixin):
    """ :args: <app_label> <document_name> <id> """
    template_name = "mongonaut/document_detail.html"

    def get_context_data(self, **kwargs):
        context = super(DocumentDetailView, self).get_context_data(**kwargs)
        self.set_mongoadmin()
        context = self.set_permissions_in_context(context)
        if not context['has_view_permission']:
            return HttpResponseForbidden("You do not have permissions to view this content.")
        self.document_type = getattr(self.models, self.document_name)
        self.ident = self.kwargs.get('id')
        self.document = self.document_type.objects.get(id=self.ident)

        context['document'] = self.document
        context['app_label'] = self.app_label
        context['document_name'] = self.document_name
        context['keys'] = ['id',]
        for key in sorted([x for x in self.document._fields.keys() if x != 'id']):
            # TODO - Figure out why this EmbeddedDocumentField and ListField breaks this view
            # Note - This is the challenge part, right? :)
            if isinstance(self.document._fields[key], EmbeddedDocumentField):
                continue
            if isinstance(self.document._fields[key], ListField):
                continue
            context['keys'].append(key)

        return context

class DocumentDetailEditFormView(FormView, MongonautViewMixin):
    """ :args: <app_label> <document_name> <id> """#

    template_name = "mongonaut/document_edit_form.html"
    form_class = DocumentDetailForm
    success_url = '/'

    def get_success_url(self):
        self.set_mongonaut_base()
        return reverse('document_detail_edit_form', kwargs={'app_label':self.app_label,'document_name':self.document_name,'id':self.kwargs.get('id')})

    def get_context_data(self, **kwargs):
        context = super(DocumentDetailEditFormView, self).get_context_data(**kwargs)
        self.set_mongoadmin()
        context = self.set_permissions_in_context(context)
        if not context['has_edit_permission']:
            return HttpResponseForbidden("You do not have permissions to edit this content.")
        self.document_type = getattr(self.models, self.document_name)
        self.ident = self.kwargs.get('id')
        self.document = self.document_type.objects.get(id=self.ident)

        context['document'] = self.document
        context['app_label'] = self.app_label
        context['document_name'] = self.document_name

        return context

    def get_form(self, DocumentDetailForm):
        self.set_mongoadmin()
        context = self.set_permissions_in_context({})
        if not context['has_edit_permission']:
            return HttpResponseForbidden("You do not have permissions to edit this content.")

        self.document_type = getattr(self.models, self.document_name)
        self.ident = self.kwargs.get('id')
        self.document = self.document_type.objects.get(id=self.ident)
        self.form = DocumentDetailForm()

        self.form = document_detail_form_factory(form=self.form, document_type=self.document_type, initial=self.document)
        if self.request.method == 'POST':
            self.form.data = self.request.POST
            self.form.is_bound = True
            if self.form.is_valid():
                for key, field in self.form.fields.items():
                    if 'readonly' in field.widget.attrs:
                        # For _id or things specified as such
                        continue

                    if isinstance(field.widget, DateTimeInput):
                        format = field.widget.format
                        setattr(self.document, key, datetime.strptime(self.request.POST[key], format))
                        continue

                    if isinstance(field.widget, CheckboxInput):
                        if key in self.request.POST:
                            setattr(self.document, key, True)
                        else:
                            setattr(self.document, key, False)
                        continue

                    if isinstance(field.widget, widgets.Select):
                        # supporting reference fields!
                        value = field.mongofield.document_type.objects.get(id=self.request.POST[key])
                        setattr(self.document, key, value)
                        continue

                    # for strings
                    setattr(self.document, key, self.request.POST[key])

                self.document.save()
                # TODO add message for save

        return self.form

class DocumentDetailAddFormView(FormView, MongonautViewMixin):
    """ :args: <app_label> <document_name> <id> """#

    template_name = "mongonaut/document_add_form.html"
    form_class = DocumentDetailForm
    success_url = '/'

    def get_success_url(self):
        self.set_mongonaut_base()
        return reverse('document_detail', kwargs={'app_label':self.app_label,'document_name':self.document_name,'id':str(self.document.id)})

    def get_context_data(self, **kwargs):
        """ TODO - possibly inherit this from DocumentDetailEditFormView. This is same thing minus:
            self.ident = self.kwargs.get('id')
            self.document = self.document_type.objects.get(id=self.ident)
        """
        context = super(DocumentDetailAddFormView, self).get_context_data(**kwargs)
        self.set_mongoadmin()
        context = self.set_permissions_in_context(context)
        if not context['has_add_permission']:
            return HttpResponseForbidden("You do not have permissions to edit view content.")
        self.document_type = getattr(self.models, self.document_name)

        context['app_label'] = self.app_label
        context['document_name'] = self.document_name
        return context

    def get_form(self, DocumentDetailForm):
        self.set_mongonaut_base()
        self.document_type = getattr(self.models, self.document_name)
        self.form = DocumentDetailForm()
        self.form = document_detail_form_factory(form=self.form, document_type=self.document_type)
        if self.request.method == 'POST':
            self.form.data = self.request.POST
            self.form.is_bound = True
            if self.form.is_valid():
                self.document = self.document_type()
                for key, field in self.form.fields.items():
                    if 'readonly' in field.widget.attrs:
                        # For _id
                        continue

                    if isinstance(field.widget, DateTimeInput):
                        format = field.widget.format
                        setattr(self.document, key, datetime.strptime(self.request.POST[key], format))
                        continue

                    if isinstance(field.widget, CheckboxInput):
                        if key in self.request.POST:
                            setattr(self.document, key, True)
                        else:
                            setattr(self.document, key, False)
                        continue

                    if isinstance(field.widget, widgets.Select):
                        # supporting reference fields!
                        value = field.mongofield.document_type.objects.get(id=self.request.POST[key])
                        setattr(self.document, key, value)
                        continue

                    # for strings
                    setattr(self.document, key, self.request.POST[key])
                self.document.save()
                # TODO add message for save

        return self.form

class DocumentDeleteView(TemplateView, DeletionMixin, MongonautViewMixin):
    """ :args: <app_label> <document_name> <id>

        TODO - implement a GET view for confirmation
    """

    success_url = "/"
    template_name = "mongonaut/document_delete.html"

    def get_success_url(self):
        self.set_mongonaut_base()
        return reverse('document_list', kwargs={'app_label':self.app_label,'document_name':self.document_name})

    def get_object(self):
        self.set_mongoadmin()
        self.document_type = getattr(self.models, self.document_name)
        self.ident = self.kwargs.get('id')
        self.document = self.document_type.objects.get(id=self.ident)
        return self.document

class EmbeddedDocumentDetailView(DetailView, MongonautViewMixin):
    """ :args: <app_label> <document_name> <id> <???> """
    template_name = "mongonaut/embedded_document_detail.html"<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
#The MIT License (MIT)
#
#Copyright (c) <2013-2014> <Colin Duquesnoy and others, see AUTHORS.txt>
#
#Permission is hereby granted, free of charge, to any person obtaining a copy
#of this software and associated documentation files (the "Software"), to deal
#in the Software without restriction, including without limitation the rights
#to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#copies of the Software, and to permit persons to whom the Software is
#furnished to do so, subject to the following conditions:
#
#The above copyright notice and this permission notice shall be included in
#all copies or substantial portions of the Software.
#
#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
#AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
#OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
#THE SOFTWARE.
#
import pyqode.core
from pyqode.core import logger
from pyqode.core import get_server, start_server, Worker
from pyqode.python.modes.code_completion import iconFromType
from pyqode.qt import QtCore

class Definition(object):
    """
    A definition object defines a symbol definition in a python source code:
        - import
        - variable
        - class
        - method/function
    """
    def __init__(self, name, icon, line, column, full_name):
        #: Icon resource name associated with the definition, can be None
        self.icon = icon
        #: Definition name (name of the class, method, variable)
        if hasattr(name, "string"):
            name = name.string
        self.name = name
        #: The line of the definition in the current editor text
        self.line = line
        #: The column of the definition in the current editor text
        self.column = column
        #: Symbol name + parent name (for methods and class variables)
        self.full_name = full_name
        #: Possible list of children (only classes have children)
        self.children = []
        if self.full_name == "":
            self.full_name = self.name

    def add_child(self, definition):
        """
        Adds a child definition
        """
        self.children.append(definition)

    def __repr__(self):
        return 'Definition(%r, %r, %r, %r)' % (self.name, self.icon,
                                               self.line, self.column)

    def __eq__(self, other):
        if len(self.children) != len(other.children):
            return False
        for self_child, other_child in zip(self.children, other.children):
            if self_child != other_child:
                return False
        return (self.name == other.name and self.full_name == other.full_name
                and self.line == other.line and self.column == other.column)

def _compare_definitions(a, b):
    """
    Compare two definition list.

    Returns True if they are different.
    """
    if len(a) != len(b):
        return True
    else:
        # compare every definition. If one is different, break
        for def_a, def_b in zip(a, b):
            if def_a != def_b:
                return True
        return False

class DefinedNamesWorker(Worker):
    """
    Subprocess worker that analyses the document using *jedi.defined_names()*.
    """
    _slot = "jedi"

    def __init__(self, code, path, encoding):
        self.code = code
        self.path = path
        self.encoding = encoding

    def __call__(self, client, caller_id, **kwargs):
        import jedi
        ret_val = []
        toplvl_definitions = jedi.defined_names(self.code, self.path,
                                                self.encoding)
        for d in toplvl_definitions:
            d_line, d_column = d.start_pos
            definition = Definition(d.name, iconFromType(d.name, d.type),
                                    d_line, d_column, d.full_name)
            if d.type.upper() == "IMPORT":
                definition.name = definition.full_name
            if d.type == "class":
                sub_definitions = d.defined_names()
                for sub_d in sub_definitions:
                    icon = iconFromType(sub_d.name, sub_d.type)
                    line, column = sub_d.start_pos
                    sub_definition = Definition(
                        sub_d.name, icon, line, column, sub_d.full_name)
                    if sub_definition.full_name == "":
                        sub_definition.full_name = sub_d.name
                    definition.add_child(sub_definition)
            ret_val.append(definition)

        try:
            old_definitions = self.slotDict["%d_definitions" % caller_id]
        except KeyError:
            old_definitions = []

        if ret_val:
            if not _compare_definitions(ret_val, old_definitions):
                ret_val = None
                logger.debug("No changes detected")
            else:
                self.slotDict["%d_definitions" % caller_id] = ret_val
                logger.debug("Document structure %r" % ret_val)
        return ret_val

class DocumentAnalyserMode(pyqode.core.Mode, QtCore.QObject):
    """
    This mode analyses the structure of a document (a tree of
    :class:`pyqode.python.modes.document_analyser.Definition`.

    :attr:pyqode.python.DocumentAnalyserMode.documentChanged` is emitted
    whenever the document structure changed.

    To keep good performances, the analysis task is run when the application is
    idle for more than 1 second (by default).
    """
    IDENTIFIER = "documentAnalyserMode"
    DESCRIPTION = "Analysis the document structure on the fly"

    #: Signal emitted when the document structure changed.
    documentChanged = QtCore.Signal()

    def __init__(self, delay=1000):
        pyqode.core.Mode.__init__(self)
        QtCore.QObject.__init__(self)
        self._jobRunner = pyqode.core.DelayJobRunner(self, nbThreadsMax=1,
                                                     delay=delay)
        #: The list of results (elements might have children; this is actually a tree).
        self.results = []

    def _onStateChanged(self, state):
        if state:
            self.editor.blockCountChanged.connect(self._onLineCountChanged)
            self.editor.newTextSet.connect(self._runAnalysis)
            try:
                srv = get_server()
                if not srv:
                    srv = start_server()
                srv.signals.workCompleted.connect(self._onWorkCompleted)
            except (TypeError, AttributeError):
                pass
        else:
            self.editor.blockCountChanged.disconnect(self._onLineCountChanged)
            self.editor.newTextSet.disconnect(self._runAnalysis)
            try:
                srv = get_server()
                srv.signals.workCompleted.disconnect(self._onWorkCompleted)
            except (TypeError, AttributeError):
                pass

    def _onLineCountChanged(self, e):
        self._jobRunner.requestJob(self._runAnalysis, False)

    def _runAnalysis(self):
        try:
            srv = get_server()
        except AttributeError:
            pass
        else:
            if self.editor.toPlainText():
                worker = DefinedNamesWorker(self.editor.toPlainText(), "", "")
                srv.requestWork(self, worker)
            else:
                self.results = []
                self.documentChanged.emit()

    def _onWorkCompleted(self, caller_id, worker, results):
        if caller_id == id(self) and isinstance(worker, DefinedNamesWorker):
            if results is not None:
                self.results = results
                logger.debug("Document structure changed")
            else:
                self.results = []
            self.documentChanged.emit()

    @property
    def flattenedResults(self):
        """
        Flattens the document structure tree as a simple sequential list.
        """
        ret_val = []
        for d in self.results:
            ret_val.append(d)
            for sub_d in d.children:
                nd = Definition(sub_d.name, sub_d.icon, sub_d.line,
                                sub_d.column, sub_d.full_name)
                nd.name = "  " + nd.name
                nd.full_name = "  " + nd.full_name
                ret_val.append(nd)
        return ret_val
<commit_msg>Improve robustness<commit_after># -*- coding: utf-8 -*-
#
#The MIT License (MIT)
#
#Copyright (c) <2013-2014> <Colin Duquesnoy and others, see AUTHORS.txt>
#
#Permission is hereby granted, free of charge, to any person obtaining a copy
#of this software and associated documentation files (the "Software"), to deal
#in the Software without restriction, including without limitation the rights
#to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#copies of the Software, and to permit persons to whom the Software is
#furnished to do so, subject to the following conditions:
#
#The above copyright notice and this permission notice shall be included in
#all copies or substantial portions of the Software.
#
#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
#AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
#OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
#THE SOFTWARE.
#
import pyqode.core
from pyqode.core import logger
from pyqode.core import get_server, start_server, Worker
from pyqode.python.modes.code_completion import iconFromType
from pyqode.qt import QtCore

class Definition(object):
    """
    A definition object defines a symbol definition in a python source code:
        - import
        - variable
        - class
        - method/function
    """
    def __init__(self, name, icon, line, column, full_name):
        #: Icon resource name associated with the definition, can be None
        self.icon = icon
        #: Definition name (name of the class, method, variable)
        if hasattr(name, "string"):
            name = name.string
        self.name = name
        #: The line of the definition in the current editor text
        self.line = line
        #: The column of the definition in the current editor text
        self.column = column
        #: Symbol name + parent name (for methods and class variables)
        self.full_name = full_name
        #: Possible list of children (only classes have children)
        self.children = []
        if self.full_name == "":
            self.full_name = self.name

    def add_child(self, definition):
        """
        Adds a child definition
        """
        self.children.append(definition)

    def __repr__(self):
        return 'Definition(%r, %r, %r, %r)' % (self.name, self.icon,
                                               self.line, self.column)

    def __eq__(self, other):
        if len(self.children) != len(other.children):
            return False
        for self_child, other_child in zip(self.children, other.children):
            if self_child != other_child:
                return False
        return (self.name == other.name and self.full_name == other.full_name
                and self.line == other.line and self.column == other.column)

def _compare_definitions(a, b):
    """
    Compare two definition list.

    Returns True if they are different.
    """
    if len(a) != len(b):
        return True
    else:
        # compare every definition. If one is different, break
        for def_a, def_b in zip(a, b):
            if def_a != def_b:
                return True
        return False

class DefinedNamesWorker(Worker):
    """
    Subprocess worker that analyses the document using *jedi.defined_names()*.
    """
    _slot = "jedi"

    def __init__(self, code, path, encoding):
        self.code = code
        self.path = path
        self.encoding = encoding

    def __call__(self, client, caller_id, **kwargs):
        import jedi
        ret_val = []
        toplvl_definitions = jedi.defined_names(self.code, self.path,
                                                self.encoding)
        for d in toplvl_definitions:
            d_line, d_column = d.start_pos
            definition = Definition(d.name, iconFromType(d.name, d.type),
                                    d_line, d_column, d.full_name)
            if d.type.upper() == "IMPORT":
                definition.name = definition.full_name
            if d.type == "class":
                sub_definitions = d.defined_names()
                for sub_d in sub_definitions:
                    icon = iconFromType(sub_d.name, sub_d.type)
                    line, column = sub_d.start_pos
                    sub_definition = Definition(
                        sub_d.name, icon, line, column, sub_d.full_name)
                    if sub_definition.full_name == "":
                        sub_definition.full_name = sub_d.name
                    definition.add_child(sub_definition)
            ret_val.append(definition)

        try:
            old_definitions = self.slotDict["%d_definitions" % caller_id]
        except KeyError:
            old_definitions = []

        if ret_val:
            if not _compare_definitions(ret_val, old_definitions):
                ret_val = None
                logger.debug("No changes detected")
            else:
                self.slotDict["%d_definitions" % caller_id] = ret_val
                logger.debug("Document structure %r" % ret_val)
        return ret_val

class DocumentAnalyserMode(pyqode.core.Mode, QtCore.QObject):
    """
    This mode analyses the structure of a document (a tree of
    :class:`pyqode.python.modes.document_analyser.Definition`.

    :attr:pyqode.python.DocumentAnalyserMode.documentChanged` is emitted
    whenever the document structure changed.

    To keep good performances, the analysis task is run when the application is
    idle for more than 1 second (by default).
    """
    IDENTIFIER = "documentAnalyserMode"
    DESCRIPTION = "Analysis the document structure on the fly"

    #: Signal emitted when the document structure changed.
    documentChanged = QtCore.Signal()

    def __init__(self, delay=1000):
        pyqode.core.Mode.__init__(self)
        QtCore.QObject.__init__(self)
        self._jobRunner = pyqode.core.DelayJobRunner(self, nbThreadsMax=1,
                                                     delay=delay)
        #: The list of results (elements might have children; this is actually a tree).
        self.results = []

    def _onStateChanged(self, state):
        if state:
            self.editor.blockCountChanged.connect(self._onLineCountChanged)
            self.editor.newTextSet.connect(self._runAnalysis)
            try:
                srv = get_server()
                if not srv:
                    srv = start_server()
                srv.signals.workCompleted.connect(self._onWorkCompleted)
            except (TypeError, AttributeError):
                pass
        else:
            self.editor.blockCountChanged.disconnect(self._onLineCountChanged)
            self.editor.newTextSet.disconnect(self._runAnalysis)
            try:
                srv = get_server()
                srv.signals.workCompleted.disconnect(self._onWorkCompleted)
            except (TypeError, AttributeError):
                pass

    def _onLineCountChanged(self, e):
        self._jobRunner.requestJob(self._runAnalysis, False)

    def _runAnalysis(self):
        try:
            srv = get_server()
        except AttributeError:
            pass
        else:
            if not self.editor:
                return
            if self.editor.toPlainText():
                worker = DefinedNamesWorker(self.editor.toPlainText(), "", "")
                srv.requestWork(self, worker)
            else:
                self.results = []
                self.documentChanged.emit()

    def _onWorkCompleted(self, caller_id, worker, results):
        if caller_id == id(self) and isinstance(worker, DefinedNamesWorker):
            if results is not None:
                self.results = results
                logger.debug("Document structure changed")
            else:
                self.results = []
            self.documentChanged.emit()

    @property
    def flattenedResults(self):
        """
        Flattens the document structure tree as a simple sequential list.
        """
        ret_val = []
        for d in self.results:
            ret_val.append(d)
            for sub_d in d.children:
                nd = Definition(sub_d.name, sub_d.icon, sub_d.line,
                                sub_d.column, sub_d.full_name)
                nd.name = "  " + nd.name
                nd.full_name = "  " + nd.full_name
                ret_val.append(nd)
        return ret_val
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
TODO move permission checks to the dispatch view thingee
"""
import math

from django.contrib import messages
from django.core.urlresolvers import reverse
from django.forms import Form
from django.http import HttpResponseForbidden
from django.http import Http404
from django.views.generic.edit import DeletionMixin
from django.views.generic import ListView
from django.views.generic import TemplateView
from django.views.generic.edit import FormView
from mongoengine.fields import EmbeddedDocumentField, ListField

from mongonaut.forms import MongoModelForm
from mongonaut.mixins import MongonautFormViewMixin
from mongonaut.mixins import MongonautViewMixin
from mongonaut.utils import is_valid_object_id

class IndexView(MongonautViewMixin, ListView):
    """Lists all the apps with mongoadmins attached."""

    template_name = "mongonaut/index.html"
    queryset = []
    permission = 'has_view_permission'

    def get_queryset(self):
        return self.get_mongoadmins()

class DocumentListView(MongonautViewMixin, FormView):
    """ :args: <app_label> <document_name>

        TODO - Make a generic document fetcher method
    """
    form_class = Form
    success_url = '/'
    template_name = "mongonaut/document_list.html"
    permission = 'has_view_permission'

    documents_per_page = 25

    #def dispatch(self, *args, **kwargs):
    #    self.set_mongoadmin()
    #    self.set_permissions()
    #    return super(DocumentListView, self).dispatch(*args, **kwargs)

    def get_qset(self, queryset, q):
        if self.mongoadmin.search_fields and q:
            params = {}
            for field in self.mongoadmin.search_fields:
                if field == 'id':
                    # check to make sure this is a valid ID, otherwise we just continue
                    if is_valid_object_id(q):
                        return queryset.filter(pk=q)
                    continue
                search_key = "{field}__icontains".format(field=field)
                params[search_key] = q

            queryset = queryset.filter(**params)
        return queryset

    def get_queryset(self):
        if hasattr(self, "queryset") and self.queryset:
            return self.queryset

        self.set_mongonaut_base()
        self.set_mongoadmin()
        self.document = getattr(self.models, self.document_name)
        queryset = self.document.objects.all()

        if self.mongoadmin.ordering:
            queryset = queryset.order_by(*self.mongoadmin.ordering)

        # search. move this to get_queryset
        # search. move this to get_queryset
        q = self.request.GET.get('q')
        queryset = self.get_qset(queryset, q)

        ### Start pagination
        ### Note:
        ###    Didn't use the Paginator in Django cause mongoengine querysets are
        ###    not the same as Django ORM querysets and it broke.
        # Make sure page request is an int. If not, deliver first page.
        try:
            self.page = int(self.request.GET.get('page', '1'))
        except ValueError:
            self.page = 1

        obj_count = queryset.count()
        self.total_pages = math.ceil(obj_count / self.documents_per_page)

        if self.page < 1:
            self.page = 1

        if self.page > self.total_pages:
            self.page = self.total_pages

        start = (self.page - 1) * self.documents_per_page
        end = self.page * self.documents_per_page

        queryset = queryset[start:end] if obj_count else queryset

        self.queryset = queryset
        return queryset

    def get_initial(self):
        self.query = self.get_queryset()
        mongo_ids = {'mongo_id': [unicode(x.id) for x in self.query]}
        return mongo_ids

    def get_context_data(self, **kwargs):
        context = super(DocumentListView, self).get_context_data(**kwargs)
        context = self.set_permissions_in_context(context)

        if not context['has_view_permission']:
            return HttpResponseForbidden("You do not have permissions to view this content.")

        context['object_list'] = self.get_queryset()

        context['document'] = self.document
        context['app_label'] = self.app_label
        context['document_name'] = self.document_name

        # pagination bits
        context['page'] = self.page
        context['documents_per_page'] = self.documents_per_page

        if self.page > 1:
            previous_page_number = self.page - 1
        else:
            previous_page_number = None

        if self.page < self.total_pages:
            next_page_number = self.page + 1
        else:
            next_page_number = None

        context['previous_page_number'] = previous_page_number
        context['has_previous_page'] = previous_page_number is not None
        context['next_page_number'] = next_page_number
        context['has_next_page'] = next_page_number is not None
        context['total_pages'] = self.total_pages

        # Part of upcoming list view form functionality
        if self.queryset.count():
            context['keys'] = ['id', ]

            # Show those items for which we've got list_fields on the mongoadmin
            for key in [x for x in self.mongoadmin.list_fields if x != 'id' and x in self.document._fields.keys()]:

                # TODO - Figure out why this EmbeddedDocumentField and ListField breaks this view
                # Note - This is the challenge part, right? :)
                if isinstance(self.document._fields[key], EmbeddedDocumentField):
                    continue
                if isinstance(self.document._fields[key], ListField):
                    continue
                context['keys'].append(key)

        if self.mongoadmin.search_fields:
            context['search_field'] = True

        return context

    def post(self, request, *args, **kwargs):
        # TODO - make sure to check the rights of the poster
        #self.get_queryset() # TODO - write something that grabs the document class better
        form_class = self.get_form_class()
        form = self.get_form(form_class)
        mongo_ids = self.get_initial()['mongo_id']
        for form_mongo_id in form.data.getlist('mongo_id'):
            for mongo_id in mongo_ids:
                if form_mongo_id == mongo_id:
                    self.document.objects.get(pk=mongo_id).delete()

        return self.form_invalid(form)

class DocumentDetailView(MongonautViewMixin, TemplateView):
    """ :args: <app_label> <document_name> <id> """
    template_name = "mongonaut/document_detail.html"
    permission = 'has_view_permission'

    def get_context_data(self, **kwargs):
        context = super(DocumentDetailView, self).get_context_data(**kwargs)
        self.set_mongoadmin()
        context = self.set_permissions_in_context(context)
        self.document_type = getattr(self.models, self.document_name)
        self.ident = self.kwargs.get('id')
        self.document = self.document_type.objects.get(pk=self.ident)

        context['document'] = self.document
        context['app_label'] = self.app_label
        context['document_name'] = self.document_name
        context['keys'] = ['id', ]
        context['embedded_documents'] = []
        context['list_fields'] = []
        for key in sorted([x for x in self.document._fields.keys() if x != 'id']):
            # TODO - Figure out why this EmbeddedDocumentField and ListField breaks this view
            # Note - This is the challenge part, right? :)
            if isinstance(self.document._fields[key], EmbeddedDocumentField):
                context['embedded_documents'].append(key)
                continue
            if isinstance(self.document._fields[key], ListField):
                context['list_fields'].append(key)
                continue
            context['keys'].append(key)
        return context

class DocumentEditFormView(MongonautViewMixin, FormView, MongonautFormViewMixin):
    """ :args: <app_label> <document_name> <id> """

    template_name = "mongonaut/document_edit_form.html"
    form_class = Form
    success_url = '/'
    permission = 'has_edit_permission'

    def get_success_url(self):
        self.set_mongonaut_base()
        return reverse('document_detail_edit_form', kwargs={'app_label': self.app_label, 'document_name': self.document_name, 'id': self.kwargs.get('id')})

    def get_context_data(self, **kwargs):
        context = super(DocumentEditFormView, self).get_context_data(**kwargs)
        self.set_mongoadmin()
        context = self.set_permissions_in_context(context)
        self.document_type = getattr(self.models, self.document_name)
        self.ident = self.kwargs.get('id')
        self.document = self.document_type.objects.get(pk=self.ident)

        context['document'] = self.document
        context['app_label'] = self.app_label
        context['document_name'] = self.document_name
        context['form_action'] = reverse('document_detail_edit_form', args=[self.kwargs.get('app_label'),
                                                                            self.kwargs.get('document_name'),
                                                                            self.kwargs.get('id')])

        return context

    def get_form(self, Form):
        self.set_mongoadmin()
        context = self.set_permissions_in_context({})

        if not context['has_edit_permission']:
            return HttpResponseForbidden("You do not have permissions to edit this content.")

        self.document_type = getattr(self.models, self.document_name)
        self.ident = self.kwargs.get('id')
        try:
            self.document = self.document_type.objects.get(pk=self.ident)
        except self.document_type.DoesNotExist:
            raise Http404
        self.form = Form()

        if self.request.method == 'POST':
            self.form = self.process_post_form('Your changes have been saved.')
        else:
            self.form = MongoModelForm(model=self.document_type, instance=self.document).get_form()
        return self.form

class DocumentAddFormView(MongonautViewMixin, FormView, MongonautFormViewMixin):
    """ :args: <app_label> <document_name> <id> """

    template_name = "mongonaut/document_add_form.html"
    form_class = Form
    success_url = '/'
    permission = 'has_add_permission'

    def get_success_url(self):
        self.set_mongonaut_base()
        return reverse('document_detail', kwargs={'app_label': self.app_label, 'document_name': self.document_name, 'id': str(self.new_document.id)})

    def get_context_data(self, **kwargs):
        """ TODO - possibly inherit this from DocumentEditFormView. This is same thing minus:
            self.ident = self.kwargs.get('id')
            self.document = self.document_type.objects.get(pk=self.ident)
        """
        context = super(DocumentAddFormView, self).get_context_data(**kwargs)
        self.set_mongoadmin()
        context = self.set_permissions_in_context(context)
        self.document_type = getattr(self.models, self.document_name)

        context['app_label'] = self.app_label
        context['document_name'] = self.document_name
        context['form_action'] = reverse('document_detail_add_form', args=[self.kwargs.get('app_label'),
                                                                           self.kwargs.get('document_name')])

        return context

    def get_form(self, Form):
        self.set_mongonaut_base()
        self.document_type = getattr(self.models, self.document_name)
        self.form = Form()

        if self.request.method == 'POST':
            self.form = self.process_post_form('Your new document has been added and saved.')
        else:
            self.form = MongoModelForm(model=self.document_type).get_form()
        return self.form

class DocumentDeleteView(DeletionMixin, MongonautViewMixin, TemplateView):
    """ :args: <app_label> <document_name> <id>

        TODO - implement a GET view for confirmation
    """

    success_url = "/"
    template_name = "mongonaut/document_delete.html"

    def get_success_url(self):
        self.set_mongonaut_base()
        messages.add_message(self.request, messages.INFO, 'Your document has been deleted.')
        return reverse('document_list', kwargs={'app_label': self.app_label, 'document_name': self.document_name})

    def get_object(self):
        self.set_mongoadmin()
        self.document_type = getattr(self.models, self.document_name)
        self.ident = self.kwargs.get('id')
        self.document = self.document_type.objects.get(pk=self.ident)
        return self.document
<commit_msg>Documented DocumentListView per #83<commit_after># -*- coding: utf-8 -*-
"""
TODO move permission checks to the dispatch view thingee
"""
import math

from django.contrib import messages
from django.core.urlresolvers import reverse
from django.forms import Form
from django.http import HttpResponseForbidden
from django.http import Http404
from django.utils.functional import cached_property
from django.views.generic.edit import DeletionMixin
from django.views.generic import ListView
from django.views.generic import TemplateView
from django.views.generic.edit import FormView
from mongoengine.fields import EmbeddedDocumentField, ListField

from mongonaut.forms import MongoModelForm
from mongonaut.mixins import MongonautFormViewMixin
from mongonaut.mixins import MongonautViewMixin
from mongonaut.utils import is_valid_object_id

class IndexView(MongonautViewMixin, ListView):
    """Lists all the apps with mongoadmins attached."""

    template_name = "mongonaut/index.html"
    queryset = []
    permission = 'has_view_permission'

    def get_queryset(self):
        return self.get_mongoadmins()

class DocumentListView(MongonautViewMixin, FormView):
    """
        Lists individual mongoengine documents for a model.
        :args: <app_label> <document_name>

        TODO - Make a generic document fetcher method
    """
    form_class = Form
    success_url = '/'
    template_name = "mongonaut/document_list.html"
    permission = 'has_view_permission'

    documents_per_page = 25

    #def dispatch(self, *args, **kwargs):
    #    self.set_mongoadmin()
    #    self.set_permissions()
    #    return super(DocumentListView, self).dispatch(*args, **kwargs)

    def get_qset(self, queryset, q):
        """Performs filtering against the default queryset returned by
            mongoengine.
        """
        if self.mongoadmin.search_fields and q:
            params = {}
            for field in self.mongoadmin.search_fields:
                if field == 'id':
                    # check to make sure this is a valid ID, otherwise we just continue
                    if is_valid_object_id(q):
                        return queryset.filter(pk=q)
                    continue
                search_key = "{field}__icontains".format(field=field)
                params[search_key] = q

            queryset = queryset.filter(**params)
        return queryset

    @cached_property
    def get_queryset(self):
        """Replicates Django CBV `get_queryset()` method, but for MongoEngine.
        """
        self.set_mongonaut_base()
        self.set_mongoadmin()
        self.document = getattr(self.models, self.document_name)
        queryset = self.document.objects.all()

        if self.mongoadmin.ordering:
            queryset = queryset.order_by(*self.mongoadmin.ordering)

        # search. move this to get_queryset
        # search. move this to get_queryset
        q = self.request.GET.get('q')
        queryset = self.get_qset(queryset, q)

        ### Start pagination
        ### Note:
        ###    Can't use Paginator in Django because mongoengine querysets are
        ###    not the same as Django ORM querysets and it broke.
        # Make sure page request is an int. If not, deliver first page.
        try:
            self.page = int(self.request.GET.get('page', '1'))
        except ValueError:
            self.page = 1

        obj_count = queryset.count()
        self.total_pages = math.ceil(obj_count / self.documents_per_page)

        if self.page < 1:
            self.page = 1

        if self.page > self.total_pages:
            self.page = self.total_pages

        start = (self.page - 1) * self.documents_per_page
        end = self.page * self.documents_per_page

        queryset = queryset[start:end] if obj_count else queryset
        return queryset

    def get_initial(self):
        """Used during adding/editing of data."""
        self.query = self.get_queryset()
        mongo_ids = {'mongo_id': [unicode(x.id) for x in self.query]}
        return mongo_ids

    def get_context_data(self, **kwargs):
        """Injects data into the context to replicate CBV ListView."""
        context = super(DocumentListView, self).get_context_data(**kwargs)
        context = self.set_permissions_in_context(context)

        if not context['has_view_permission']:
            return HttpResponseForbidden("You do not have permissions to view this content.")

        context['object_list'] = self.get_queryset()

        context['document'] = self.document
        context['app_label'] = self.app_label
        context['document_name'] = self.document_name

        # pagination bits
        context['page'] = self.page
        context['documents_per_page'] = self.documents_per_page

        if self.page > 1:
            previous_page_number = self.page - 1
        else:
            previous_page_number = None

        if self.page < self.total_pages:
            next_page_number = self.page + 1
        else:
            next_page_number = None

        context['previous_page_number'] = previous_page_number
        context['has_previous_page'] = previous_page_number is not None
        context['next_page_number'] = next_page_number
        context['has_next_page'] = next_page_number is not None
        context['total_pages'] = self.total_pages

        # Part of upcoming list view form functionality
        if self.queryset.count():
            context['keys'] = ['id', ]

            # Show those items for which we've got list_fields on the mongoadmin
            for key in [x for x in self.mongoadmin.list_fields if x != 'id' and x in self.document._fields.keys()]:

                # TODO - Figure out why this EmbeddedDocumentField and ListField breaks this view
                # Note - This is the challenge part, right? :)
                if isinstance(self.document._fields[key], EmbeddedDocumentField):
                    continue
                if isinstance(self.document._fields[key], ListField):
                    continue
                context['keys'].append(key)

        if self.mongoadmin.search_fields:
            context['search_field'] = True

        return context

    def post(self, request, *args, **kwargs):
        """Creates new mongoengine records/"""
        # TODO - make sure to check the rights of the poster
        #self.get_queryset() # TODO - write something that grabs the document class better
        form_class = self.get_form_class()
        form = self.get_form(form_class)
        mongo_ids = self.get_initial()['mongo_id']
        for form_mongo_id in form.data.getlist('mongo_id'):
            for mongo_id in mongo_ids:
                if form_mongo_id == mongo_id:
                    self.document.objects.get(pk=mongo_id).delete()

        return self.form_invalid(form)

class DocumentDetailView(MongonautViewMixin, TemplateView):
    """ :args: <app_label> <document_name> <id> """
    template_name = "mongonaut/document_detail.html"
    permission = 'has_view_permission'

    def get_context_data(self, **kwargs):
        context = super(DocumentDetailView, self).get_context_data(**kwargs)
        self.set_mongoadmin()
        context = self.set_permissions_in_context(context)
        self.document_type = getattr(self.models, self.document_name)
        self.ident = self.kwargs.get('id')
        self.document = self.document_type.objects.get(pk=self.ident)

        context['document'] = self.document
        context['app_label'] = self.app_label
        context['document_name'] = self.document_name
        context['keys'] = ['id', ]
        context['embedded_documents'] = []
        context['list_fields'] = []
        for key in sorted([x for x in self.document._fields.keys() if x != 'id']):
            # TODO - Figure out why this EmbeddedDocumentField and ListField breaks this view
            # Note - This is the challenge part, right? :)
            if isinstance(self.document._fields[key], EmbeddedDocumentField):
                context['embedded_documents'].append(key)
                continue
            if isinstance(self.document._fields[key], ListField):
                context['list_fields'].append(key)
                continue
            context['keys'].append(key)
        return context

class DocumentEditFormView(MongonautViewMixin, FormView, MongonautFormViewMixin):
    """ :args: <app_label> <document_name> <id> """

    template_name = "mongonaut/document_edit_form.html"
    form_class = Form
    success_url = '/'
    permission = 'has_edit_permission'

    def get_success_url(self):
        self.set_mongonaut_base()
        return reverse('document_detail_edit_form', kwargs={'app_label': self.app_label, 'document_name': self.document_name, 'id': self.kwargs.get('id')})

    def get_context_data(self, **kwargs):
        context = super(DocumentEditFormView, self).get_context_data(**kwargs)
        self.set_mongoadmin()
        context = self.set_permissions_in_context(context)
        self.document_type = getattr(self.models, self.document_name)
        self.ident = self.kwargs.get('id')
        self.document = self.document_type.objects.get(pk=self.ident)

        context['document'] = self.document
        context['app_label'] = self.app_label
        context['document_name'] = self.document_name
        context['form_action'] = reverse('document_detail_edit_form', args=[self.kwargs.get('app_label'),
                                                                            self.kwargs.get('document_name'),
                                                                            self.kwargs.get('id')])

        return context

    def get_form(self, Form):
        self.set_mongoadmin()
        context = self.set_permissions_in_context({})

        if not context['has_edit_permission']:
            return HttpResponseForbidden("You do not have permissions to edit this content.")

        self.document_type = getattr(self.models, self.document_name)
        self.ident = self.kwargs.get('id')
        try:
            self.document = self.document_type.objects.get(pk=self.ident)
        except self.document_type.DoesNotExist:
            raise Http404
        self.form = Form()

        if self.request.method == 'POST':
            self.form = self.process_post_form('Your changes have been saved.')
        else:
            self.form = MongoModelForm(model=self.document_type, instance=self.document).get_form()
        return self.form

class DocumentAddFormView(MongonautViewMixin, FormView, MongonautFormViewMixin):
    """ :args: <app_label> <document_name> <id> """

    template_name = "mongonaut/document_add_form.html"
    form_class = Form
    success_url = '/'
    permission = 'has_add_permission'

    def get_success_url(self):
        self.set_mongonaut_base()
        return reverse('document_detail', kwargs={'app_label': self.app_label, 'document_name': self.document_name, 'id': str(self.new_document.id)})

    def get_context_data(self, **kwargs):
        """ TODO - possibly inherit this from DocumentEditFormView. This is same thing minus:
            self.ident = self.kwargs.get('id')
            self.document = self.document_type.objects.get(pk=self.ident)
        """
        context = super(DocumentAddFormView, self).get_context_data(**kwargs)
        self.set_mongoadmin()
        context = self.set_permissions_in_context(context)
        self.document_type = getattr(self.models, self.document_name)

        context['app_label'] = self.app_label
        context['document_name'] = self.document_name
        context['form_action'] = reverse('document_detail_add_form', args=[self.kwargs.get('app_label'),
                                                                           self.kwargs.get('document_name')])

        return context

    def get_form(self, Form):
        self.set_mongonaut_base()
        self.document_type = getattr(self.models, self.document_name)
        self.form = Form()

        if self.request.method == 'POST':
            self.form = self.process_post_form('Your new document has been added and saved.')
        else:
            self.form = MongoModelForm(model=self.document_type).get_form()
        return self.form

class DocumentDeleteView(DeletionMixin, MongonautViewMixin, TemplateView):
    """ :args: <app_label> <document_name> <id>

        TODO - implement a GET view for confirmation
    """

    success_url = "/"
    template_name = "mongonaut/document_delete.html"

    def get_success_url(self):
        self.set_mongonaut_base()
        messages.add_message(self.request, messages.INFO, 'Your document has been deleted.')
        return reverse('document_list', kwargs={'app_label': self.app_label, 'document_name': self.document_name})

    def get_object(self):
        self.set_mongoadmin()
        self.document_type = getattr(self.models, self.document_name)
        self.ident = self.kwargs.get('id')
        self.document = self.document_type.objects.get(pk=self.ident)
        return self.document
<|endoftext|>