<commit_before># -*- coding: utf-8 -*-
from bs4 import BeautifulSoup
import urllib2,urllib
import sys
import csv
from datetime import datetime
import os
from time import strftime
from time import sleep
from time import time

from types import *

# TSE : Taiwan Stock Exchange , 台灣證交所 （上市）
# OTC : Over-the-Counter , 櫃檯中心 （上櫃）
# BSR : Buy Sell Report , 分公司買賣進出表

class DownloadBotBase(object):
    def __init__(self):
        self.name = "Newbie Bot."
        self.date = strftime('%Y%m%d')
        self.lstCode = []

    def __str__(self):
        return self.name

    def Date(self):
        return str(self.date)

    def setDate(self,date):
        self.date = date

    def Code(self):
        return self.lstCode

    def showCode(self):
        print '[%s]' % ', '.join(map(str, self.lstCode ))

    def setCode(self,Code):
        if type(Code) in [ IntType , StringType ]:
            if len(str(Code).strip()) != 4: # 只接受4位數的商品
                return
            self.lstCode.append(str(Code))

        elif type(Code) == ListType:
            self.lstCode += Code

    def Run(self):
        count = 0
        retry = 0
        print 'Total:%d'%len(self.Code())
        lstRetry = []
        while(len(self.Code())):
            Code = self.Code().pop(0)
            count += 1
            filename = "%s_%s.csv"%(Code,self.Date())
            print 'Process:[%s] Download:%d Left:%d retry:%d'%(Code,count,len(self.Code()),retry)
            ret = self.RunImp(Code, filename)

            if None == ret:
                self.setCode(Code)
                if lstRetry.count(Code) > 3:
                    print '%s 下載三次失敗'%(Code)
                    lstRetry = [x for x in lstRetry if x != Code ]
                    continue
                lstRetry.append(Code)
                retry += 1
                count -= 1
                print '********fail*******'
                sleep( 3 ) #有錯誤停3秒
                continue
            print '\tWrite %s Finish...'%(filename)

class DownloadTSEBot(DownloadBotBase):
    def __init__(self):
        super(DownloadTSEBot, self).__init__()
        self.name = "TSE BSR Download Bot."

    def RunImp(self,Code,filename):

        # step 1. GetMaxPage and POST data
        def GetMaxPageNum(Code):
            try:
                base_url = 'http://bsr.twse.com.tw/bshtm/bsMenu.aspx'
                req = urllib2.Request(base_url)
                response = urllib2.urlopen(req)
                html = response.read()
                soup = BeautifulSoup(html)
                __VIEWSTATE  = soup.find(attrs={"id": "__VIEWSTATE"})['value']
                sp_Date  = soup.find(attrs={"id":"sp_Date"}).contents[0]
                print 'Trade date(%s)'%sp_Date
                __EVENTVALIDATION = soup.find(attrs={"id": "__EVENTVALIDATION"})['value']#.get('Value')

                PostDataDict = {'__EVENTTARGET':''
                                , '__EVENTARGUMENT':''
                                ,'HiddenField_page':'PAGE_BS'
                                ,'txtTASKNO':Code
                                ,'hidTASKNO':Code
                                ,'__VIEWSTATE':__VIEWSTATE
                                ,'__EVENTVALIDATION':__EVENTVALIDATION
                                ,'HiddenField_spDate':sp_Date
                                ,'btnOK':'%E6%9F%A5%E8%A9%A2'}

                postData = urllib.urlencode( PostDataDict)
                req = urllib2.Request( base_url , postData)
                response = urllib2.urlopen( req)
                html = response.read()
                soup = BeautifulSoup(html)
                MaxPageNum  = soup.find(attrs={"id":"sp_ListCount"}).contents[0]
                return MaxPageNum
            except Exception,e:
                #print e
                return None

        # step 2. GetRawData
        def GetBSRawData(Code,MaxPageNum):
            try:
                url = 'http://bsr.twse.com.tw/bshtm/bsContent.aspx?StartNumber=%s&FocusIndex=All_%s'%(Code,MaxPageNum)
                req = urllib2.Request(url)
                response = urllib2.urlopen(req)
                html = response.read()
                return html
            except Exception , e:
                return None

        # step 3. RawToCSV
        def BSRawToCSV(BSRaw):
            soup = BeautifulSoup(BSRaw)
            #取得資料表title
            title_contents =  soup.find(attrs={ 'class': 'column_title_1'})
            title_list = title_contents.find_all('td')
            title = [title.get_text().encode('big5') for title in title_list]

            #取得各分公司買賣內容
            stock_info_content = soup.find_all(attrs={'class':['column_value_price_3','column_value_price_2']})
            CSVData = []
            for i in stock_info_content:
                row_list = i.find_all('td')
                row = [row.get_text().strip().encode('big5') for row in row_list]
                if len(row[0]) > 0:
                    #print '[%s]'%row[0]
                    CSVData.append(row)
            #使用序號排序,因為網頁奇偶數沒穿插
            CSVData.sort(key=lambda element: int(element[0]))
            #將Title加入資料首列
            CSVData.insert(0, title)
            return CSVData

        def CSVToFile(CSVData,filename):
            with open('BSR/'+filename, 'wb') as csvfile:
                writer = csv.writer(csvfile,dialect='excel')
                writer.writerows(CSVData)

        self.RawBSR = "TSE"
        MaxPageNum = GetMaxPageNum(Code)
        if None == MaxPageNum:
            return None
        BSRawData = GetBSRawData(Code, MaxPageNum)
        if None == BSRawData:
            return None
        CSVData = BSRawToCSV(BSRawData)
        CSVToFile(CSVData, filename)
        return True

class DownloadOTCBot(DownloadBotBase):
    def __init__(self):
        super(DownloadOTCBot, self).__init__()
        self.name = "OTC BSR Download Bot."

    def RunImp(self,Code,filename):

        def DownloadOTC(Code,filename,otcdate):
            try:
                #otcdate = str(otcyear)+g_date[4:]
                base_url = 'http://www.gretai.org.tw/ch/stock/aftertrading/broker_trading/download_ALLCSV.php'
                PostDataDict = {'curstk':Code
                                , 'fromw':'0'
                                ,'numbern':'100'
                                ,'stk_date':otcdate
                                }

                postData = urllib.urlencode( PostDataDict)
                req = urllib2.Request( base_url , postData)
                response = urllib2.urlopen( req)
                html = response.read()
            except Exception , e:
                return None
            with open('BSR/'+filename, 'wb') as f:
                f.write(html)

            return True

        self.RawBSR = "OTC"
        ret = DownloadOTC(Code,filename,self.getOtcDate())
        if None == ret:
            return None
        return True

    def getOtcDate(self):
        otcyear = int(self.date)/int(10000) - 1911
        otcdate = str(otcyear) + self.date[4:]
        return otcdate

    def setOtcDate(self,date):
        self.date =  str(int(date[0:3]) + 1911) + date[3:] #1020101

def getCodeDict():
    CodeDict = {'TSE' : [] , 'OTC': [] }
    with open('data/smast.dat','r') as f:
        for row in f:
            try:
                code = row[:6].strip()
                row = row.decode('utf-8').encode('cp950')
                if len(code)== 4 : #忽略權證,公司債
                    print row[:13].decode('cp950').encode('utf-8')
                    if row[12] == '0': #TSE_上市
                        CodeDict['TSE'].append(row[:4])
                    if row[12] == '1': #OTC_上櫃
                        CodeDict['OTC'].append(row[:4])
            except IndexError:
                print 'You have an empty row'
        sleep(5)
    return CodeDict

if __name__ == '__main__':
    if not os.path.exists('BSR'):
        os.makedirs('BSR')
    print 'Start...'
    CodeDict = getCodeDict()
    print 'TSE:%d OTC:%d'%(len(CodeDict['TSE']),len(CodeDict['OTC']))

    tradedate = '20131121'

    tStart = time()

    OTC = DownloadOTCBot()
    OTC.setDate(tradedate)
    OTC.setCode(CodeDict['OTC'])
    OTC.Run()

    tEndOTC = time()

    TSE = DownloadTSEBot()
    TSE.setDate(tradedate)
    TSE.setCode(CodeDict['TSE'])
    TSE.Run()
    tEndTSE = time()

    print 'End...TSE(%f) OTC(%f) Total(%f)'%(tEndTSE-tEndOTC,tEndOTC-tStart,tEndTSE-tStart)

<commit_msg>import Threading<commit_after># -*- coding: utf-8 -*-
from bs4 import BeautifulSoup
import urllib2,urllib
import sys
import csv
from datetime import datetime
import os
import threading
from time import strftime
from time import sleep
from time import time

from types import *

# TSE : Taiwan Stock Exchange , 台灣證交所 （上市）
# OTC : Over-the-Counter , 櫃檯中心 （上櫃）
# BSR : Buy Sell Report , 分公司買賣進出表

class DownloadBotBase(object):
    def __init__(self):
        self.name = "Newbie Bot."
        self.date = strftime('%Y%m%d')
        self.lstCode = []

    def __str__(self):
        return self.name

    def Date(self):
        return str(self.date)

    def setDate(self,date):
        self.date = date

    def Code(self):
        return self.lstCode

    def showCode(self):
        print '[%s]' % ', '.join(map(str, self.lstCode ))

    def setCode(self,Code):
        if type(Code) in [ IntType , StringType ]:
            if len(str(Code).strip()) != 4: # 只接受4位數的商品
                return
            self.lstCode.append(str(Code))

        elif type(Code) == ListType:
            self.lstCode += Code

    def Run(self):
        count = 0
        retry = 0
        print 'Total:%d'%len(self.Code())
        lstRetry = []
        while(len(self.Code())):
            Code = self.Code().pop(0)
            count += 1
            filename = "%s_%s.csv"%(Code,self.Date())
            print 'Process:[%s] Download:%d Left:%d retry:%d'%(Code,count,len(self.Code()),retry)
            ret = self.RunImp(Code, filename)

            if None == ret:
                self.setCode(Code)
                if lstRetry.count(Code) > 3:
                    print '%s 下載三次失敗'%(Code)
                    lstRetry = [x for x in lstRetry if x != Code ]
                    continue
                lstRetry.append(Code)
                retry += 1
                count -= 1
                print '********fail*******'
                sleep( 3 ) #有錯誤停3秒
                continue
            print '\tWrite %s Finish...'%(filename)

class DownloadTSEBot(DownloadBotBase):
    def __init__(self):
        super(DownloadTSEBot, self).__init__()
        self.name = "TSE BSR Download Bot."

    def RunImp(self,Code,filename):

        # step 1. GetMaxPage and POST data
        def GetMaxPageNum(Code):
            try:
                base_url = 'http://bsr.twse.com.tw/bshtm/bsMenu.aspx'
                req = urllib2.Request(base_url)
                response = urllib2.urlopen(req)
                html = response.read()
                soup = BeautifulSoup(html)
                __VIEWSTATE  = soup.find(attrs={"id": "__VIEWSTATE"})['value']
                sp_Date  = soup.find(attrs={"id":"sp_Date"}).contents[0]
                print 'Trade date(%s)'%sp_Date
                __EVENTVALIDATION = soup.find(attrs={"id": "__EVENTVALIDATION"})['value']#.get('Value')

                PostDataDict = {'__EVENTTARGET':''
                                , '__EVENTARGUMENT':''
                                ,'HiddenField_page':'PAGE_BS'
                                ,'txtTASKNO':Code
                                ,'hidTASKNO':Code
                                ,'__VIEWSTATE':__VIEWSTATE
                                ,'__EVENTVALIDATION':__EVENTVALIDATION
                                ,'HiddenField_spDate':sp_Date
                                ,'btnOK':'%E6%9F%A5%E8%A9%A2'}

                postData = urllib.urlencode( PostDataDict)
                req = urllib2.Request( base_url , postData)
                response = urllib2.urlopen( req)
                html = response.read()
                soup = BeautifulSoup(html)
                MaxPageNum  = soup.find(attrs={"id":"sp_ListCount"}).contents[0]
                return MaxPageNum
            except Exception,e:
                #print e
                return None

        # step 2. GetRawData
        def GetBSRawData(Code,MaxPageNum):
            try:
                url = 'http://bsr.twse.com.tw/bshtm/bsContent.aspx?StartNumber=%s&FocusIndex=All_%s'%(Code,MaxPageNum)
                req = urllib2.Request(url)
                response = urllib2.urlopen(req)
                html = response.read()
                return html
            except Exception , e:
                return None

        # step 3. RawToCSV
        def BSRawToCSV(BSRaw):
            soup = BeautifulSoup(BSRaw)
            #取得資料表title
            title_contents =  soup.find(attrs={ 'class': 'column_title_1'})
            title_list = title_contents.find_all('td')
            title = [title.get_text().encode('big5') for title in title_list]

            #取得各分公司買賣內容
            stock_info_content = soup.find_all(attrs={'class':['column_value_price_3','column_value_price_2']})
            CSVData = []
            for i in stock_info_content:
                row_list = i.find_all('td')
                row = [row.get_text().strip().encode('big5') for row in row_list]
                if len(row[0]) > 0:
                    #print '[%s]'%row[0]
                    CSVData.append(row)
            #使用序號排序,因為網頁奇偶數沒穿插
            CSVData.sort(key=lambda element: int(element[0]))
            #將Title加入資料首列
            CSVData.insert(0, title)
            return CSVData

        def CSVToFile(CSVData,filename):
            with open('BSR/'+filename, 'wb') as csvfile:
                writer = csv.writer(csvfile,dialect='excel')
                writer.writerows(CSVData)

        self.RawBSR = "TSE"
        MaxPageNum = GetMaxPageNum(Code)
        if None == MaxPageNum:
            return None
        BSRawData = GetBSRawData(Code, MaxPageNum)
        if None == BSRawData:
            return None
        CSVData = BSRawToCSV(BSRawData)
        CSVToFile(CSVData, filename)
        return True

class DownloadOTCBot(DownloadBotBase):
    def __init__(self):
        super(DownloadOTCBot, self).__init__()
        self.name = "OTC BSR Download Bot."

    def RunImp(self,Code,filename):

        def DownloadOTC(Code,filename,otcdate):
            try:
                #otcdate = str(otcyear)+g_date[4:]
                base_url = 'http://www.gretai.org.tw/ch/stock/aftertrading/broker_trading/download_ALLCSV.php'
                PostDataDict = {'curstk':Code
                                , 'fromw':'0'
                                ,'numbern':'100'
                                ,'stk_date':otcdate
                                }

                postData = urllib.urlencode( PostDataDict)
                req = urllib2.Request( base_url , postData)
                response = urllib2.urlopen( req)
                html = response.read()
            except Exception , e:
                return None
            with open('BSR/'+filename, 'wb') as f:
                f.write(html)

            return True

        self.RawBSR = "OTC"
        ret = DownloadOTC(Code,filename,self.getOtcDate())
        if None == ret:
            return None
        return True

    def getOtcDate(self):
        otcyear = int(self.date)/int(10000) - 1911
        otcdate = str(otcyear) + self.date[4:]
        return otcdate

    def setOtcDate(self,date):
        self.date =  str(int(date[0:3]) + 1911) + date[3:] #1020101

def getCodeDict():
    CodeDict = {'TSE' : [] , 'OTC': [] }
    with open('data/smast.dat','r') as f:
        for row in f:
            try:
                code = row[:6].strip()
                row = row.decode('utf-8').encode('cp950')
                if len(code)== 4 : #忽略權證,公司債
                    print row[:13].decode('cp950').encode('utf-8')
                    if row[12] == '0': #TSE_上市
                        CodeDict['TSE'].append(row[:4])
                    if row[12] == '1': #OTC_上櫃
                        CodeDict['OTC'].append(row[:4])
            except IndexError:
                print 'You have an empty row'
        sleep(5)
    return CodeDict

if __name__ == '__main__':
    if not os.path.exists('BSR'):
        os.makedirs('BSR')
    print 'Start...'
    CodeDict = getCodeDict()
    print 'TSE:%d OTC:%d'%(len(CodeDict['TSE']),len(CodeDict['OTC']))

    tradedate = '20131121'

    tStart = time()

    OTC = DownloadOTCBot()
    OTC.setDate(tradedate)
    OTC.setCode(CodeDict['OTC'])
    OTC.Run()

    tEndOTC = time()

    TSE = DownloadTSEBot()
    TSE.setDate(tradedate)
    TSE.setCode(CodeDict['TSE'])
    TSE.Run()
    tEndTSE = time()

    print 'End...TSE(%f) OTC(%f) Total(%f)'%(tEndTSE-tEndOTC,tEndOTC-tStart,tEndTSE-tStart)

<|endoftext|>


<commit_before>#!/usr/bin/env python
"""Module for the reciprocal blast step using the SARA Life Science Grid Portal."""

from datetime import timedelta
from divergence import concatenate
from divergence.lsgp_client import run_application, submit_application_run, retrieve_run_result, send_request
from divergence.versions import LSGP_MAKEBLASTDB, LSGP_BLASTN, LSGP_BLASTP
import os
import shutil
import tempfile

__author__ = "Tim te Beek"
__contact__ = "brs@nbic.nl"
__copyright__ = "Copyright 2011, Netherlands Bioinformatics Centre"
__license__ = "MIT"

def reciprocal_blast(good_proteins_fasta, fasta_files):
    """Create blast database for good_proteins_fasta, blast all fasta_files against this database & return hits."""
    # Create blast database, retrieve path & name
    database_url = _create_blast_database(good_proteins_fasta)

    # Submit job for each fasta files, and store jobid & hits file name tuples
    jobids_and_hits_filenames = [_submit_blast_run(database_url, fasta_file) for fasta_file in fasta_files]

    # Retrieve all results, which should only take neglishably longer than waiting for the slowest results
    x_vs_all_hits = [_retrieve_blast_hits(jobid, hits_file) for jobid, hits_file in jobids_and_hits_filenames]

    # Concatenate the individual blast result files into one
    allvsall = tempfile.mkstemp(suffix='.tsv', prefix='all_vs_all')[1]
    concatenate(allvsall, x_vs_all_hits)

    # Clean up local and remote
    for x_vs_all in x_vs_all_hits:
        os.remove(x_vs_all)
    #Remote database_url port :444 requires certificate authentication, which we remove to use the basic authentication
    database_url = database_url.replace(':444', '')
    send_request(database_url, method='DELETE')

    return allvsall

def _create_blast_database(fasta_file, nucleotide=False):
    """Create blast database"""
    dbtype = 'nucl' if nucleotide else 'prot'
    db_name = '{0}_blast_db'.format(dbtype)

    #Run MAKEBLASTDB remotely
    params = {'dbtype': dbtype, 'out': db_name, 'portaldb': 1}
    files = {'in-file[]': fasta_file}
    db_dir = run_application(LSGP_MAKEBLASTDB,
                             params=params,
                             files=files,
                             max_duration=timedelta(hours=4).total_seconds())

    #Upload database back to LSG Portal
    with open(os.path.join(db_dir, 'db_url.txt')) as read_handle:
        database_url = read_handle.read().strip()

    #Remove local database directory
    shutil.rmtree(db_dir)

    return database_url

def blast_file_against_database(database_url, fasta_file, nucleotide=False):
    """
    Blast file against database, and immediately retrieve and return the resulting BLAST hits file.
    @param database_url:
    @param fasta_file:
    @param nucleotide:
    """
    jobid, hits_file = _submit_blast_run(database_url, fasta_file, nucleotide)
    outside_path = _retrieve_blast_hits(jobid, hits_file)
    return outside_path

def _submit_blast_run(database_url, fasta_file, nucleotide=False):
    """
    Submit a BLAST run, returning the created jobid and filename for BLAST hits.
    @param database_url:
    @param fasta_file:
    @param nucleotide:
    """
    blast_app = LSGP_BLASTN if nucleotide else LSGP_BLASTP
    # Determine output file name
    hits_file = os.path.splitext(os.path.split(fasta_file)[1])[0] + '-vs-all.tsv'
    # Run BLAST[P/N] remotely
    params = {'db[]': database_url, 'out': hits_file, 'outfmt': 6}
    files = {'query': fasta_file}
    jobid = submit_application_run(blast_app,
                                   params=params,
                                   files=files)
    return jobid, hits_file

def _retrieve_blast_hits(jobid, hits_file):
    """
    Wait for job to complete, retrieve hits and clean up temporary files. Return the path to the created hits_file.
    @param jobid: jobid for which to retrieve the BLAST hits
    @param hits_file: name of the created BLAST hits file to retrieve
    """
    #Wait for job to complete and retrieve results
    results_dir = retrieve_run_result(jobid,
                                      max_duration=timedelta(hours=8).total_seconds())

    # Construct the likely path to the local file
    local_path = os.path.join(results_dir, hits_file)
    assert os.path.isfile(local_path) and 0 < os.path.getsize(local_path), local_path + ' should exist with content'

    # Move result file out of results_dir
    name, ext = os.path.splitext(os.path.split(local_path)[1])
    outside_path = tempfile.mkstemp(suffix=ext, prefix=name)[1]
    shutil.move(local_path, outside_path)

    # Remove results_dir, as we would lose directory reference otherwise
    shutil.rmtree(results_dir)

    return outside_path
<commit_msg>Wait for 1 day for the database creation, wait for 5 hours for the retrieval of blast job results<commit_after>#!/usr/bin/env python
"""Module for the reciprocal blast step using the SARA Life Science Grid Portal."""

from datetime import timedelta
from divergence import concatenate
from divergence.lsgp_client import run_application, submit_application_run, retrieve_run_result, send_request
from divergence.versions import LSGP_MAKEBLASTDB, LSGP_BLASTN, LSGP_BLASTP
import os
import shutil
import tempfile

__author__ = "Tim te Beek"
__contact__ = "brs@nbic.nl"
__copyright__ = "Copyright 2011, Netherlands Bioinformatics Centre"
__license__ = "MIT"

def reciprocal_blast(good_proteins_fasta, fasta_files):
    """Create blast database for good_proteins_fasta, blast all fasta_files against this database & return hits."""
    # Create blast database, retrieve path & name
    database_url = _create_blast_database(good_proteins_fasta)

    # Submit job for each fasta files, and store jobid & hits file name tuples
    jobids_and_hits_filenames = [_submit_blast_run(database_url, fasta_file) for fasta_file in fasta_files]

    # Retrieve all results, which should only take neglishably longer than waiting for the slowest results
    x_vs_all_hits = [_retrieve_blast_hits(jobid, hits_file) for jobid, hits_file in jobids_and_hits_filenames]

    # Concatenate the individual blast result files into one
    allvsall = tempfile.mkstemp(suffix='.tsv', prefix='all_vs_all')[1]
    concatenate(allvsall, x_vs_all_hits)

    # Clean up local and remote
    for x_vs_all in x_vs_all_hits:
        os.remove(x_vs_all)
    #Remote database_url port :444 requires certificate authentication, which we remove to use the basic authentication
    database_url = database_url.replace(':444', '')
    send_request(database_url, method='DELETE')

    return allvsall

def _create_blast_database(fasta_file, nucleotide=False):
    """Create blast database"""
    dbtype = 'nucl' if nucleotide else 'prot'
    db_name = '{0}_blast_db'.format(dbtype)

    #Run MAKEBLASTDB remotely
    params = {'dbtype': dbtype, 'out': db_name, 'portaldb': 1}
    files = {'in-file[]': fasta_file}
    db_dir = run_application(LSGP_MAKEBLASTDB,
                             params=params,
                             files=files,
                             max_duration=timedelta(days=1).total_seconds())

    #Upload database back to LSG Portal
    with open(os.path.join(db_dir, 'db_url.txt')) as read_handle:
        database_url = read_handle.read().strip()

    #Remove local database directory
    shutil.rmtree(db_dir)

    return database_url

def blast_file_against_database(database_url, fasta_file, nucleotide=False):
    """
    Blast file against database, and immediately retrieve and return the resulting BLAST hits file.
    @param database_url:
    @param fasta_file:
    @param nucleotide:
    """
    jobid, hits_file = _submit_blast_run(database_url, fasta_file, nucleotide)
    outside_path = _retrieve_blast_hits(jobid, hits_file)
    return outside_path

def _submit_blast_run(database_url, fasta_file, nucleotide=False):
    """
    Submit a BLAST run, returning the created jobid and filename for BLAST hits.
    @param database_url:
    @param fasta_file:
    @param nucleotide:
    """
    blast_app = LSGP_BLASTN if nucleotide else LSGP_BLASTP
    # Determine output file name
    hits_file = os.path.splitext(os.path.split(fasta_file)[1])[0] + '-vs-all.tsv'
    # Run BLAST[P/N] remotely
    params = {'db[]': database_url, 'out': hits_file, 'outfmt': 6}
    files = {'query': fasta_file}
    jobid = submit_application_run(blast_app,
                                   params=params,
                                   files=files)
    return jobid, hits_file

def _retrieve_blast_hits(jobid, hits_file):
    """
    Wait for job to complete, retrieve hits and clean up temporary files. Return the path to the created hits_file.
    @param jobid: jobid for which to retrieve the BLAST hits
    @param hits_file: name of the created BLAST hits file to retrieve
    """
    #Wait for job to complete and retrieve results
    results_dir = retrieve_run_result(jobid,
                                      max_duration=timedelta(hours=5).total_seconds())

    # Construct the likely path to the local file
    local_path = os.path.join(results_dir, hits_file)
    assert os.path.isfile(local_path) and 0 < os.path.getsize(local_path), local_path + ' should exist with content'

    # Move result file out of results_dir
    name, ext = os.path.splitext(os.path.split(local_path)[1])
    outside_path = tempfile.mkstemp(suffix=ext, prefix=name)[1]
    shutil.move(local_path, outside_path)

    # Remove results_dir, as we would lose directory reference otherwise
    shutil.rmtree(results_dir)

    return outside_path
<|endoftext|>


<commit_before>import yaml

def read_simpleconf(filename, defaults=None):
    """
    Docstring goes here
    """

	outdict = defaults if defaults is not None else {}
    with open(filename) as f: yaml_data = yaml.load(f)
    for tag in outdict:
        if outdict[tag] is None:
            raise ValueError('A value for the required tag \"'
                                + tag + '\" must be specified.')

    for tag in yaml_data: outdict[tag] = yaml_data[tag]

    return outdict<commit_msg>Updated fn name.<commit_after>import yaml

def read_yaml(filename, defaults=None):
    """
    Docstring goes here
    """

	outdict = defaults if defaults is not None else {}
    with open(filename) as f: yaml_data = yaml.load(f)
    for tag in outdict:
        if outdict[tag] is None:
            raise ValueError('A value for the required tag \"'
                                + tag + '\" must be specified.')

    for tag in yaml_data: outdict[tag] = yaml_data[tag]

    return outdict<|endoftext|>


<commit_before>from django.conf import settings
from django.test import TestCase
from django.test.utils import override_settings
from django.utils import six

from .models import Redirect

@override_settings(
    APPEND_SLASH=False,
    MIDDLEWARE_CLASSES=list(settings.MIDDLEWARE_CLASSES) +
        ['regex_redirects.middleware.RedirectFallbackMiddleware'],
    SITE_ID=1,
)
class RedirectTests(TestCase):

    def setUp(self):
        pass

    def test_model(self):
        r1 = Redirect.objects.create(
            old_path='/initial', new_path='/new_target')
        self.assertEqual(six.text_type(r1), "/initial ---> /new_target")

    def test_redirect(self):
        Redirect.objects.create(
            old_path='/initial', new_path='/new_target')
        response = self.client.get('/initial')
        self.assertRedirects(response,
            '/new_target', status_code=301, target_status_code=404)

    @override_settings(APPEND_SLASH=True)
    def test_redirect_with_append_slash(self):
        Redirect.objects.create(
            old_path='/initial/', new_path='/new_target/')
        response = self.client.get('/initial')
        self.assertRedirects(response,
            '/new_target/', status_code=301, target_status_code=404)

    @override_settings(APPEND_SLASH=True)
    def test_redirect_with_append_slash_and_query_string(self):
        Redirect.objects.create(
            old_path='/initial/?foo', new_path='/new_target/')
        response = self.client.get('/initial?foo')
        self.assertRedirects(response,
            '/new_target/', status_code=301, target_status_code=404)

    def test_regular_expression(self):
        Redirect.objects.create(
            old_path='/news/index/(\d+)/(.*)/',
            new_path='/my/news/$2/',
            regular_expression=True)
        response = self.client.get('/news/index/12345/foobar/')
        self.assertRedirects(response,
                             '/my/news/foobar/',
                             status_code=301, target_status_code=404)
        redirect = Redirect.objects.get(regular_expression=True)
        self.assertEqual(redirect.nr_times_visited, 1)

    def test_fallback_redirects(self):
        """
        Ensure redirects with fallback_redirect set are the last evaluated
        """
        Redirect.objects.create(
            old_path='/project/foo',
            new_path='/my/project/foo')

        Redirect.objects.create(
            old_path='/project/foo/(.*)',
            new_path='/my/project/foo/$1',
            regular_expression=True)

        Redirect.objects.create(
            old_path='/project/(.*)',
            new_path='/projects',
            regular_expression=True,
            fallback_redirect=True)

        Redirect.objects.create(
            old_path='/project/bar/(.*)',
            new_path='/my/project/bar/$1',
            regular_expression=True)

        Redirect.objects.create(
            old_path='/project/bar',
            new_path='/my/project/bar')

        response = self.client.get('/project/foo')
        self.assertRedirects(response,
                             '/my/project/foo',
                             status_code=301, target_status_code=404)

        response = self.client.get('/project/bar')
        self.assertRedirects(response,
                             '/my/project/bar',
                             status_code=301, target_status_code=404)

        response = self.client.get('/project/bar/details')
        self.assertRedirects(response,
                             '/my/project/bar/details',
                             status_code=301, target_status_code=404)

        response = self.client.get('/project/foobar')
        self.assertRedirects(response,
                             '/projects',
                             status_code=301, target_status_code=404)

        response = self.client.get('/project/foo/details')
        self.assertRedirects(response,
                             '/my/project/foo/details',
                             status_code=301, target_status_code=404)
<commit_msg>Isolate tests more by using global_settings middleware.<commit_after>from django.conf import global_settings
from django.test import TestCase
from django.test.utils import override_settings
from django.utils import six

from .models import Redirect

@override_settings(
    APPEND_SLASH=False,
    MIDDLEWARE_CLASSES=list(global_settings.MIDDLEWARE_CLASSES) +
        ['regex_redirects.middleware.RedirectFallbackMiddleware'],
    SITE_ID=1,
)
class RedirectTests(TestCase):

    def setUp(self):
        pass

    def test_model(self):
        r1 = Redirect.objects.create(
            old_path='/initial', new_path='/new_target')
        self.assertEqual(six.text_type(r1), "/initial ---> /new_target")

    def test_redirect(self):
        Redirect.objects.create(
            old_path='/initial', new_path='/new_target')
        response = self.client.get('/initial')
        self.assertRedirects(response,
            '/new_target', status_code=301, target_status_code=404)

    @override_settings(APPEND_SLASH=True)
    def test_redirect_with_append_slash(self):
        Redirect.objects.create(
            old_path='/initial/', new_path='/new_target/')
        response = self.client.get('/initial')
        self.assertRedirects(response,
            '/new_target/', status_code=301, target_status_code=404)

    @override_settings(APPEND_SLASH=True)
    def test_redirect_with_append_slash_and_query_string(self):
        Redirect.objects.create(
            old_path='/initial/?foo', new_path='/new_target/')
        response = self.client.get('/initial?foo')
        self.assertRedirects(response,
            '/new_target/', status_code=301, target_status_code=404)

    def test_regular_expression(self):
        Redirect.objects.create(
            old_path='/news/index/(\d+)/(.*)/',
            new_path='/my/news/$2/',
            regular_expression=True)
        response = self.client.get('/news/index/12345/foobar/')
        self.assertRedirects(response,
                             '/my/news/foobar/',
                             status_code=301, target_status_code=404)
        redirect = Redirect.objects.get(regular_expression=True)
        self.assertEqual(redirect.nr_times_visited, 1)

    def test_fallback_redirects(self):
        """
        Ensure redirects with fallback_redirect set are the last evaluated
        """
        Redirect.objects.create(
            old_path='/project/foo',
            new_path='/my/project/foo')

        Redirect.objects.create(
            old_path='/project/foo/(.*)',
            new_path='/my/project/foo/$1',
            regular_expression=True)

        Redirect.objects.create(
            old_path='/project/(.*)',
            new_path='/projects',
            regular_expression=True,
            fallback_redirect=True)

        Redirect.objects.create(
            old_path='/project/bar/(.*)',
            new_path='/my/project/bar/$1',
            regular_expression=True)

        Redirect.objects.create(
            old_path='/project/bar',
            new_path='/my/project/bar')

        response = self.client.get('/project/foo')
        self.assertRedirects(response,
                             '/my/project/foo',
                             status_code=301, target_status_code=404)

        response = self.client.get('/project/bar')
        self.assertRedirects(response,
                             '/my/project/bar',
                             status_code=301, target_status_code=404)

        response = self.client.get('/project/bar/details')
        self.assertRedirects(response,
                             '/my/project/bar/details',
                             status_code=301, target_status_code=404)

        response = self.client.get('/project/foobar')
        self.assertRedirects(response,
                             '/projects',
                             status_code=301, target_status_code=404)

        response = self.client.get('/project/foo/details')
        self.assertRedirects(response,
                             '/my/project/foo/details',
                             status_code=301, target_status_code=404)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

###############################################################################
# Module Imports
###############################################################################

import collections
import csv
import json
import logging
import pyscp
import re
import itertools
import functools

###############################################################################
# Global Constants And Variables
###############################################################################

log = logging.getLogger(__name__)

###############################################################################
# Scalars
#
# These functions receive a list of pages and return a single value.
# They're the smallest blocks on which the rest of the stats module is build.
###############################################################################

# builtin len is a scalar too

def upvotes(pages):
    """Upvotes."""
    return sum([v.value for v in p.votes].count(1) for p in pages)

def rating(pages):
    """Net rating."""
    return sum(p.rating for p in pages)

def average(pages):
    """Average rating."""
    return rating(pages) / len(pages)

def divided(pages):
    """Controversy score."""
    return sum(len(p.votes) / p.rating for p in pages)

def redactions(pages):
    """Redaction score."""
    return sum(
        p.text.count('█') +
        20 * sum(map(p.text.count, ('REDACTED', 'EXPUNGED')))
        for p in pages)

###############################################################################
# Counters
#
# These function receive a list of pages and a scalar function. They then
# split the pages based on some criteria, and apply the scalar to each
# subgroup. The results are returned as a collections.Counter object.
###############################################################################

def make_counter(key, pages, func):
    """Generic counter factory."""
    subgroups = collections.defaultdict(list)
    for p in pages:
        key_value = key(p)
        if key_value:
            subgroups[key_value].append(p)
    return collections.Counter({k: func(v) for k, v in subgroups.items()})

def counter_authors(pages, func):
    """Author counter."""
    return make_counter(lambda p: p.author, pages, func)

###############################################################################
# Filters
#
# These functions take a list of pages and return a list filtered based on
# some criteria. Some of these are basically shortcuts for list comprehensions,
# while others are more complicated.
###############################################################################

def filter_tag(pages, tag):
    """Pages with a given tag."""
    return [p for p in pages if tag in p.tags]

def filter_min_authored(pages, min_val=3):
    """Pages by authors who have at least min_val pages."""
    authors = counter_authors(pages, len)
    return [p for p in pages if authors[p.author] >= min_val]

###############################################################################
# Records
#
# Pretty-printing numbers for the 'SCP WORLD RECORDS' thread.
###############################################################################

def records(pages):
    pages = [p for p in pages if p.author != 'Unknown Author']
    pages = [p for p in pages if '_sys' not in p.tags]
    skips, tales, jokes, essays = [
        filter_tag(pages, tag) for tag in ('scp', 'tale', 'joke', 'essay')]
    messages = (
        'Users with Most Upvotes (General):',
        'Users with Most Upvotes (SCPs):',
        'Users with Most Upvotes (Tales):',
        'Most SCPs Written:',
        'Most Tales Written:',
        'Highest SCP Average (>=3):',
        'Highest Tale Average (>=3):',
        'Most -J Articles Written:',
        'Highest Joke Average (>=3):',
        'Most Essay Articles Written:',
        'Highest Essay Average (>=3):',)
    counters = (
        counter_authors(pages, upvotes),
        counter_authors(skips, upvotes),
        counter_authors(tales, upvotes),
        counter_authors(skips, len),
        counter_authors(tales, len),
        counter_authors(filter_min_authored(skips), average),
        counter_authors(filter_min_authored(tales), average),
        counter_authors(jokes, len),
        counter_authors(filter_min_authored(jokes), average),
        counter_authors(essays, len),
        counter_authors(filter_min_authored(essays), average),)
    for message, counter in zip(messages, counters):
        print(message)
        for k, v in counter.most_common(5):
            print(k.ljust(40), round(v, 2))

###############################################################################

if __name__ == "__main__":
    pyscp.utils.default_logging()
    sn = pyscp.core.SnapshotConnector(
        'www.scp-wiki.net', '/home/anqxyr/heap/_scp/scp-wiki.2015-06-13.db')
    pages = list(map(sn, sn.list_pages()))
    records(pages)
<commit_msg>chainy goodness<commit_after>#!/usr/bin/env python3

###############################################################################
# Module Imports
###############################################################################

import collections
import csv
import json
import logging
import pyscp
import re
import itertools
import functools

###############################################################################
# Global Constants And Variables
###############################################################################

log = logging.getLogger(__name__)

###############################################################################
# Scalars
#
# These functions receive a list of pages and return a single value.
# They're the smallest blocks on which the rest of the stats module is build.
###############################################################################

# builtin len is a scalar too

def upvotes(pages):
    """Upvotes."""
    return sum([v.value for v in p.votes].count(1) for p in pages)

def rating(pages):
    """Net rating."""
    return sum(p.rating for p in pages)

def average(pages):
    """Average rating."""
    return rating(pages) / len(pages)

def divided(pages):
    """Controversy score."""
    return sum(len(p.votes) / p.rating for p in pages)

def redactions(pages):
    """Redaction score."""
    return sum(
        p.text.count('█') +
        20 * sum(map(p.text.count, ('REDACTED', 'EXPUNGED')))
        for p in pages)

###############################################################################
# Counters
#
# These function receive a list of pages and a scalar function. They then
# split the pages based on some criteria, and apply the scalar to each
# subgroup. The results are returned as a collections.Counter object.
###############################################################################

def make_counter(pages, func, key):
    """Generic counter factory."""
    subgroups = collections.defaultdict(list)
    for p in pages:
        key_value = key(p)
        if key_value:
            subgroups[key_value].append(p)
    return collections.Counter({k: func(v) for k, v in subgroups.items()})

def counter_author(pages, func):
    """Group per page author."""
    return make_counter(pages, func, lambda p: p.author)

def counter_month(pages, func):
    """Group per month the page was posted on."""
    return make_counter(pages, func, lambda p: p.created[:7])

def chain_counters(pages, func, *counters):
    """Apply counters one after another."""
    if len(counters) == 1:
        return counters[0](pages, func)
    results = collections.Counter()
    for key, val in counters[0](pages, lambda x: x).items():
        for ikey, ival in chain_counters(val, func, *counters[1:]).items():
            results['%s, %s' % (key, ikey)] = ival
    return results

###############################################################################
# Filters
#
# These functions take a list of pages and return a list filtered based on
# some criteria. Some of these are basically shortcuts for list comprehensions,
# while others are more complicated.
###############################################################################

def filter_tag(pages, tag):
    """Pages with a given tag."""
    return [p for p in pages if tag in p.tags]

def filter_min_authored(pages, min_val=3):
    """Pages by authors who have at least min_val pages."""
    authors = counter_author(pages, len)
    return [p for p in pages if authors[p.author] >= min_val]

###############################################################################
# Records
#
# Pretty-printing numbers for the 'SCP WORLD RECORDS' thread.
###############################################################################

def records(pages):
    pages = [p for p in pages if p.author != 'Unknown Author']
    pages = [p for p in pages if '_sys' not in p.tags]
    skips, tales, jokes, essays = [
        filter_tag(pages, tag) for tag in ('scp', 'tale', 'joke', 'essay')]
    messages = (
        'Users with Most Upvotes (General):',
        'Users with Most Upvotes (SCPs):',
        'Users with Most Upvotes (Tales):',
        'Most SCPs Written:',
        'Most Tales Written:',
        'Highest SCP Average (>=3):',
        'Highest Tale Average (>=3):',
        'Most -J Articles Written:',
        'Highest Joke Average (>=3):',
        'Most Essay Articles Written:',
        'Highest Essay Average (>=3):',
        'Most Successful Articles posted in 1 Month:',
        'Most Successful SCPs posted in 1 Month:')
    counters = (
        counter_author(pages, upvotes),
        counter_author(skips, upvotes),
        counter_author(tales, upvotes),
        counter_author(skips, len),
        counter_author(tales, len),
        counter_author(filter_min_authored(skips), average),
        counter_author(filter_min_authored(tales), average),
        counter_author(jokes, len),
        counter_author(filter_min_authored(jokes), average),
        counter_author(essays, len),
        counter_author(filter_min_authored(essays), average),
        chain_counters(
            [p for p in pages if p.created[:7] != '2008-07'],
            len, counter_author, counter_month),
        chain_counters(
            [p for p in skips if p.created[:7] != '2008-07'],
            len, counter_author, counter_month),)
    for message, counter in zip(messages, counters):
        print(message)
        for k, v in counter.most_common(5):
            print(k.ljust(40), round(v, 2))

###############################################################################

if __name__ == "__main__":
    pyscp.utils.default_logging()
    sn = pyscp.core.SnapshotConnector(
        'www.scp-wiki.net', '/home/anqxyr/heap/_scp/scp-wiki.2015-06-13.db')
    pages = list(map(sn, sn.list_pages()))
    records(pages)
<|endoftext|>


<commit_before># vim: set fileencoding=utf-8
from collections import defaultdict
from itertools import chain
import re

from inflection import pluralize

from regparser.citations import internal_citations, Label
from regparser.grammar import terms as grammar
from regparser.grammar.external_citations import uscode_exp as uscode
from regparser.layer.layer import Layer
from regparser.tree import struct
from regparser.tree.priority_stack import PriorityStack
import settings

class Ref(object):
    def __init__(self, term, label, position):
        self.term = term
        self.label = label
        self.position = position

    def __eq__(self, other):
        """Equality depends on equality of the fields"""
        return (hasattr(other, 'term') and hasattr(other, 'label')
                and hasattr(other, 'position') and self.term == other.term
                and self.label == other.label
                and self.position == other.position)

    def __repr__(self):
        return 'Ref( term=%s, label=%s, position=%s )' % (
            repr(self.term), repr(self.label), repr(self.position))

class ParentStack(PriorityStack):
    """Used to keep track of the parents while processing nodes to find
    terms. This is needed as the definition may need to find its scope in
    parents."""
    def unwind(self):
        """No collapsing needs to happen."""
        self.pop()

class Terms(Layer):
    #   Regexes used in determining scope
    part_re, subpart_re = re.compile(r"\bpart\b"), re.compile(r"\bsubpart\b")
    sect_re, par_re = re.compile(r"\bsection\b"), re.compile(r"\bparagraph\b")
    #   Regex to confirm scope indicator
    scope_re = re.compile(r".*purposes of( this)?\s*$", re.DOTALL)
    scope_used_re = re.compile(
        r".*as used in( this)?\s*$", re.DOTALL | re.IGNORECASE)

    def __init__(self, *args, **kwargs):
        Layer.__init__(self, *args, **kwargs)
        self.layer['referenced'] = {}
        #   scope -> List[(term, definition_ref)]
        self.scoped_terms = defaultdict(list)
        #   subpart -> list[section]
        self.subpart_map = defaultdict(list)

    def add_subparts(self):
        """Document the relationship between sections and subparts"""

        current_subpart = [None]    # Need a reference for the closure

        def per_node(node):
            if node.node_type == struct.Node.SUBPART:
                current_subpart[0] = node.label[2]
            elif node.node_type == struct.Node.EMPTYPART:
                current_subpart[0] = None
            if (node.node_type in (struct.Node.REGTEXT, struct.Node.APPENDIX)
                    and len(node.label) == 2):
                #Subparts
                section = node.label[-1]
                self.subpart_map[current_subpart[0]].append(section)

        struct.walk(self.tree, per_node)

    def scope_of_text(self, text, label_struct, verify_prefix=True):
        """Given specific text, try to determine the definition scope it
        indicates. Implicit return None if none is found."""
        scopes = []
        #   First, make a list of potential scope indicators
        citations = internal_citations(text, label_struct,
                                       require_marker=True)
        indicators = [(c.full_start, c.label.to_list()) for c in citations]
        text = text.lower()
        label_list = label_struct.to_list()
        indicators.extend((m.start(), label_list[:1])
                          for m in Terms.part_re.finditer(text))
        indicators.extend((m.start(), label_list[:2])
                          for m in Terms.sect_re.finditer(text))
        indicators.extend((m.start(), label_list)
                          for m in Terms.par_re.finditer(text))
        #   Subpart's a bit more complicated, as it gets expanded into a
        #   list of sections
        for match in Terms.subpart_re.finditer(text):
            indicators.extend(
                (match.start(), subpart_label)
                for subpart_label in self.subpart_scope(label_list))

        #   Finally, add the scope if we verify its prefix
        for start, label in indicators:
            if not verify_prefix or Terms.scope_re.match(text[:start]):
                scopes.append(label)
            elif Terms.scope_used_re.match(text[:start]):
                scopes.append(label)

        #   Add interpretation to scopes
        scopes = scopes + [s + [struct.Node.INTERP_MARK] for s in scopes]
        if scopes:
            return [tuple(s) for s in scopes]

    def determine_scope(self, stack):
        for node in stack.lineage():
            scopes = self.scope_of_text(node.text, Label.from_node(node))
            if scopes:
                return [tuple(s) for s in scopes]

        #   Couldn't determine scope; default to the entire reg
        return [tuple(node.label[:1])]

    def pre_process(self):
        """Step through every node in the tree, finding definitions. Add
        these definition to self.scoped_terms. Also keep track of which
        subpart we are in. Finally, document all defined terms. """
        self.add_subparts()
        stack = ParentStack()

        def per_node(node):
            if len(node.label) > 1 and node.node_type == struct.Node.REGTEXT:
                #   Add one for the subpart level
                stack.add(len(node.label) + 1, node)
            elif node.node_type in (struct.Node.SUBPART,
                                    struct.Node.EMPTYPART):
                #   Subparts all on the same level
                stack.add(2, node)
            else:
                stack.add(len(node.label), node)

            if node.node_type in (struct.Node.REGTEXT, struct.Node.SUBPART,
                                  struct.Node.EMPTYPART):
                included, excluded = self.node_definitions(node, stack)
                if included:
                    for scope in self.determine_scope(stack):
                        self.scoped_terms[scope].extend(included)
                self.scoped_terms['EXCLUDED'].extend(excluded)

        struct.walk(self.tree, per_node)

        referenced = self.layer['referenced']
        for scope in self.scoped_terms:
            for ref in self.scoped_terms[scope]:
                key = ref.term + ":" + ref.label
                if (key not in referenced     # New term
                        # Or this term is earlier in the paragraph
                        or ref.position[0] < referenced[key]['position'][0]):
                    referenced[key] = {
                        'term': ref.term,
                        'reference': ref.label,
                        'position': ref.position
                    }

    def applicable_terms(self, label):
        """Find all terms that might be applicable to nodes with this label.
        Note that we don't have to deal with subparts as subpart_scope simply
        applies the definition to all sections in a subpart"""
        applicable_terms = {}
        for segment_length in range(1, len(label) + 1):
            scope = tuple(label[:segment_length])
            for ref in self.scoped_terms.get(scope, []):
                applicable_terms[ref.term] = ref    # overwrites
        return applicable_terms

    def is_exclusion(self, term, node):
        """Some definitions are exceptions/exclusions of a previously
        defined term. At the moment, we do not want to include these as they
        would replace previous (correct) definitions."""
        applicable_terms = self.applicable_terms(node.label)
        if term in applicable_terms:
            regex = 'the term .?' + re.escape(term) + '.? does not include'
            return bool(re.search(regex, node.text.lower()))
        return False

    def has_parent_definitions_indicator(self, stack):
        """With smart quotes, we catch some false positives, phrases in quotes
        that are not terms. This extra test lets us know that a parent of the
        node looks like it would contain definitions."""
        for node in stack.lineage():
            if ('Definition' in node.text
                    or 'Definition' in (node.title or '')
                    or re.search('the term .* (means|refers to)',
                                 node.text.lower())
                    or re.search(u'“[^”]+” (means|refers to)',
                                 node.text.lower())):
                return True
        return False

    def node_definitions(self, node, stack=None):
        """Find defined terms in this node's text. 'Act' is a special case,
        as it is also defined as an external citation."""
        included_defs = []
        excluded_defs = []

        def add_match(n, term, pos):
            if ((term == 'act' and list(uscode.scanString(n.text)))
                    or self.is_exclusion(term, n)):
                excluded_defs.append(Ref(term, n.label_id(), pos))
            else:
                included_defs.append(Ref(term, n.label_id(), pos))

        try:
            cfr_part = node.label[0]
        except IndexError:
            cfr_part = None

        if settings.INCLUDE_DEFINITIONS_IN.get(cfr_part):
            for included_term, context in settings.INCLUDE_DEFINITIONS_IN[cfr_part]:
                if context in node.text and included_term in node.text:
                    pos_start = node.text.index(included_term)
                    add_match(node, included_term.lower(),
                            (pos_start, pos_start + len(included_term)))

        if stack and self.has_parent_definitions_indicator(stack):
            for match, _, _ in grammar.smart_quotes.scanString(node.text):
                term = match.term.tokens[0].lower().strip(',.;')
                #   Don't use pos_end because we are stripping some chars
                pos_start = match.term.pos[0]
                add_match(node,
                          term,
                          (pos_start, pos_start + len(term)))

        for match, _, _ in grammar.scope_term_type_parser.scanString(
                node.text):
            # Check that both scope and term look valid
            if (self.scope_of_text(match.scope, Label.from_node(node),
                                   verify_prefix=False)
                    and re.match("^[a-z ]+$", match.term.tokens[0])):
                term = match.term.tokens[0].strip()
                pos_start = node.text.index(term, match.term.pos[0])
                add_match(node, term, (pos_start, pos_start + len(term)))

        if hasattr(node, 'tagged_text'):
            for match, _, _ in grammar.xml_term_parser.scanString(
                    node.tagged_text):
                """Position in match reflects XML tags, so its dropped in
                preference of new values based on node.text."""
                for match in chain([match.head], match.tail):
                    pos_start = self.pos_start_excluding(
                        match.term.tokens[0], node.text,
                        included_defs + excluded_defs)
                    term = node.tagged_text[
                        match.term.pos[0]:match.term.pos[1]].lower()
                    match_len = len(term)
                    add_match(node,
                              term,
                              (pos_start, pos_start + match_len))

        return included_defs, excluded_defs

    def pos_start_excluding(self, needle, haystack, exclusions):
        """Search for the first instance of `needle` in the `haystack`
        excluding any overlaps from `exclusions`. Implicitly returns None if
        it can't be found"""
        start = 0
        while start >= 0:
            start = haystack.find(needle, start)
            if not any(r.position[0] <= start and r.position[1] >= start
                       for r in exclusions):
                return start
            start += 1

    def subpart_scope(self, label_parts):
        """Given a label, determine which sections fall under the same
        subpart"""
        reg = label_parts[0]
        section = label_parts[1]
        for subpart in self.subpart_map:
            if section in self.subpart_map[subpart]:
                return [[reg, sect] for sect in self.subpart_map[subpart]]
        return []

    def process(self, node):
        """Determine which (if any) definitions would apply to this node,
        then find if any of those terms appear in this node"""
        applicable_terms = self.applicable_terms(node.label)

        layer_el = []
        #   Remove any definitions defined in this paragraph
        term_list = [
            (term, ref) for term, ref in applicable_terms.iteritems()
            if ref.label != node.label_id()]

        exclusions = self.excluded_offsets(node.label_id(), node.text)
        exclusions = self.per_regulation_ignores(
            exclusions, node.label, node.text)

        inclusions = self.included_offsets(node.label_id(), node.text)
        inclusions = self.per_regulation_includes(
                inclusions, node.label, node.text)

        matches = self.calculate_offsets(node.text, term_list, exclusions)
        for term, ref, offsets in matches:
            layer_el.append({
                "ref": ref.term + ':' + ref.label,
                "offsets": offsets
                })
        return layer_el

    def per_regulation_ignores(self, exclusions, label, text):
        cfr_part = label[0]
        if settings.IGNORE_DEFINITIONS_IN.get(cfr_part):
            for ignore_term in settings.IGNORE_DEFINITIONS_IN[cfr_part]:
                exclusions.extend(
                    (match.start(), match.end()) for match in
                    re.finditer(r'\b' + re.escape(ignore_term) + r'\b', text))
        return exclusions

    def excluded_offsets(self, label, text):
        """We explicitly exclude certain chunks of text (for example, words
        we are defining shouldn't have links appear within the defined
        term.) More will be added in the future"""
        exclusions = []
        for reflist in self.scoped_terms.values():
            exclusions.extend(
                ref.position for ref in reflist if ref.label == label)
        for ignore_term in settings.IGNORE_DEFINITIONS_IN['ALL']:
            exclusions.extend(
                (match.start(), match.end()) for match in
                re.finditer(r'\b' + re.escape(ignore_term) + r'\b', text))
        return exclusions

    def per_regulation_includes(self, inclusions, label, text):
        cfr_part = label[0]
        if settings.INCLUDE_DEFINITIONS_IN.get(cfr_part):
            for included_term, context in settings.INCLUDE_DEFINITIONS_IN['ALL']:
                inclusions.extend(
                    (match.start(), match.end()) for match in
                    re.finditer(r'\b' + re.escape(include_term) + r'\b', text))
        return inclusions

    def included_offsets(self, label, text):
        """ We explicitly include certain chunks of text (for example,
            words that the parser doesn't necessarily pick up as being
            defined) that should be part of a defined term """
        inclusions = []
        for included_term, context in settings.INCLUDE_DEFINITIONS_IN['ALL']:
            inclusions.extend(
                (match.start(), match.end()) for match in
                re.finditer(r'\b' + re.escape(include_term) + r'\b', text))
        return inclusions

    def calculate_offsets(self, text, applicable_terms, exclusions=[],
            inclusions=[]):
        """Search for defined terms in this text, with a preference for all
        larger (i.e. containing) terms."""

        # don't modify the original
        exclusions = list(exclusions)
        inclusions = list(inclusions)

        # add plurals to applicable terms
        pluralized = [(pluralize(t[0]), t[1]) for t in applicable_terms]
        applicable_terms += pluralized

        #   longer terms first
        applicable_terms.sort(key=lambda x: len(x[0]), reverse=True)

        matches = []
        for term, ref in applicable_terms:
            re_term = ur'\b' + re.escape(term) + ur'\b'
            offsets = [
                (m.start(), m.end())
                for m in re.finditer(re_term, text.lower())]
            safe_offsets = []
            for start, end in offsets:
                #   Start is contained in an existing def
                if any(start >= e[0] and start <= e[1] for e in exclusions):
                    continue
                #   End is contained in an existing def
                if any(end >= e[0] and end <= e[1] for e in exclusions):
                    continue
                safe_offsets.append((start, end))
            if not safe_offsets:
                continue

            exclusions.extend(safe_offsets)
            matches.append((term, ref, safe_offsets))
        return matches
<commit_msg>Override inflection's person->people. Regs use persons.<commit_after># vim: set fileencoding=utf-8
from collections import defaultdict
from itertools import chain
import re

import inflection
try:
    del inflection.PLURALS[inflection.PLURALS.index(('(?i)(p)erson$', '\\1eople'))]
except ValueError:
    pass

from regparser.citations import internal_citations, Label
from regparser.grammar import terms as grammar
from regparser.grammar.external_citations import uscode_exp as uscode
from regparser.layer.layer import Layer
from regparser.tree import struct
from regparser.tree.priority_stack import PriorityStack
import settings

class Ref(object):
    def __init__(self, term, label, position):
        self.term = term
        self.label = label
        self.position = position

    def __eq__(self, other):
        """Equality depends on equality of the fields"""
        return (hasattr(other, 'term') and hasattr(other, 'label')
                and hasattr(other, 'position') and self.term == other.term
                and self.label == other.label
                and self.position == other.position)

    def __repr__(self):
        return 'Ref( term=%s, label=%s, position=%s )' % (
            repr(self.term), repr(self.label), repr(self.position))

class ParentStack(PriorityStack):
    """Used to keep track of the parents while processing nodes to find
    terms. This is needed as the definition may need to find its scope in
    parents."""
    def unwind(self):
        """No collapsing needs to happen."""
        self.pop()

class Terms(Layer):
    #   Regexes used in determining scope
    part_re, subpart_re = re.compile(r"\bpart\b"), re.compile(r"\bsubpart\b")
    sect_re, par_re = re.compile(r"\bsection\b"), re.compile(r"\bparagraph\b")
    #   Regex to confirm scope indicator
    scope_re = re.compile(r".*purposes of( this)?\s*$", re.DOTALL)
    scope_used_re = re.compile(
        r".*as used in( this)?\s*$", re.DOTALL | re.IGNORECASE)

    def __init__(self, *args, **kwargs):
        Layer.__init__(self, *args, **kwargs)
        self.layer['referenced'] = {}
        #   scope -> List[(term, definition_ref)]
        self.scoped_terms = defaultdict(list)
        #   subpart -> list[section]
        self.subpart_map = defaultdict(list)

    def add_subparts(self):
        """Document the relationship between sections and subparts"""

        current_subpart = [None]    # Need a reference for the closure

        def per_node(node):
            if node.node_type == struct.Node.SUBPART:
                current_subpart[0] = node.label[2]
            elif node.node_type == struct.Node.EMPTYPART:
                current_subpart[0] = None
            if (node.node_type in (struct.Node.REGTEXT, struct.Node.APPENDIX)
                    and len(node.label) == 2):
                #Subparts
                section = node.label[-1]
                self.subpart_map[current_subpart[0]].append(section)

        struct.walk(self.tree, per_node)

    def scope_of_text(self, text, label_struct, verify_prefix=True):
        """Given specific text, try to determine the definition scope it
        indicates. Implicit return None if none is found."""
        scopes = []
        #   First, make a list of potential scope indicators
        citations = internal_citations(text, label_struct,
                                       require_marker=True)
        indicators = [(c.full_start, c.label.to_list()) for c in citations]
        text = text.lower()
        label_list = label_struct.to_list()
        indicators.extend((m.start(), label_list[:1])
                          for m in Terms.part_re.finditer(text))
        indicators.extend((m.start(), label_list[:2])
                          for m in Terms.sect_re.finditer(text))
        indicators.extend((m.start(), label_list)
                          for m in Terms.par_re.finditer(text))
        #   Subpart's a bit more complicated, as it gets expanded into a
        #   list of sections
        for match in Terms.subpart_re.finditer(text):
            indicators.extend(
                (match.start(), subpart_label)
                for subpart_label in self.subpart_scope(label_list))

        #   Finally, add the scope if we verify its prefix
        for start, label in indicators:
            if not verify_prefix or Terms.scope_re.match(text[:start]):
                scopes.append(label)
            elif Terms.scope_used_re.match(text[:start]):
                scopes.append(label)

        #   Add interpretation to scopes
        scopes = scopes + [s + [struct.Node.INTERP_MARK] for s in scopes]
        if scopes:
            return [tuple(s) for s in scopes]

    def determine_scope(self, stack):
        for node in stack.lineage():
            scopes = self.scope_of_text(node.text, Label.from_node(node))
            if scopes:
                return [tuple(s) for s in scopes]

        #   Couldn't determine scope; default to the entire reg
        return [tuple(node.label[:1])]

    def pre_process(self):
        """Step through every node in the tree, finding definitions. Add
        these definition to self.scoped_terms. Also keep track of which
        subpart we are in. Finally, document all defined terms. """
        self.add_subparts()
        stack = ParentStack()

        def per_node(node):
            if len(node.label) > 1 and node.node_type == struct.Node.REGTEXT:
                #   Add one for the subpart level
                stack.add(len(node.label) + 1, node)
            elif node.node_type in (struct.Node.SUBPART,
                                    struct.Node.EMPTYPART):
                #   Subparts all on the same level
                stack.add(2, node)
            else:
                stack.add(len(node.label), node)

            if node.node_type in (struct.Node.REGTEXT, struct.Node.SUBPART,
                                  struct.Node.EMPTYPART):
                included, excluded = self.node_definitions(node, stack)
                if included:
                    for scope in self.determine_scope(stack):
                        self.scoped_terms[scope].extend(included)
                self.scoped_terms['EXCLUDED'].extend(excluded)

        struct.walk(self.tree, per_node)

        referenced = self.layer['referenced']
        for scope in self.scoped_terms:
            for ref in self.scoped_terms[scope]:
                key = ref.term + ":" + ref.label
                if (key not in referenced     # New term
                        # Or this term is earlier in the paragraph
                        or ref.position[0] < referenced[key]['position'][0]):
                    referenced[key] = {
                        'term': ref.term,
                        'reference': ref.label,
                        'position': ref.position
                    }

    def applicable_terms(self, label):
        """Find all terms that might be applicable to nodes with this label.
        Note that we don't have to deal with subparts as subpart_scope simply
        applies the definition to all sections in a subpart"""
        applicable_terms = {}
        for segment_length in range(1, len(label) + 1):
            scope = tuple(label[:segment_length])
            for ref in self.scoped_terms.get(scope, []):
                applicable_terms[ref.term] = ref    # overwrites
        return applicable_terms

    def is_exclusion(self, term, node):
        """Some definitions are exceptions/exclusions of a previously
        defined term. At the moment, we do not want to include these as they
        would replace previous (correct) definitions."""
        applicable_terms = self.applicable_terms(node.label)
        if term in applicable_terms:
            regex = 'the term .?' + re.escape(term) + '.? does not include'
            return bool(re.search(regex, node.text.lower()))
        return False

    def has_parent_definitions_indicator(self, stack):
        """With smart quotes, we catch some false positives, phrases in quotes
        that are not terms. This extra test lets us know that a parent of the
        node looks like it would contain definitions."""
        for node in stack.lineage():
            if ('Definition' in node.text
                    or 'Definition' in (node.title or '')
                    or re.search('the term .* (means|refers to)',
                                 node.text.lower())
                    or re.search(u'“[^”]+” (means|refers to)',
                                 node.text.lower())):
                return True
        return False

    def node_definitions(self, node, stack=None):
        """Find defined terms in this node's text. 'Act' is a special case,
        as it is also defined as an external citation."""
        included_defs = []
        excluded_defs = []

        def add_match(n, term, pos):
            if ((term == 'act' and list(uscode.scanString(n.text)))
                    or self.is_exclusion(term, n)):
                excluded_defs.append(Ref(term, n.label_id(), pos))
            else:
                included_defs.append(Ref(term, n.label_id(), pos))

        try:
            cfr_part = node.label[0]
        except IndexError:
            cfr_part = None

        if settings.INCLUDE_DEFINITIONS_IN.get(cfr_part):
            for included_term, context in settings.INCLUDE_DEFINITIONS_IN[cfr_part]:
                if context in node.text and included_term in node.text:
                    pos_start = node.text.index(included_term)
                    add_match(node, included_term.lower(),
                            (pos_start, pos_start + len(included_term)))

        if stack and self.has_parent_definitions_indicator(stack):
            for match, _, _ in grammar.smart_quotes.scanString(node.text):
                term = match.term.tokens[0].lower().strip(',.;')
                #   Don't use pos_end because we are stripping some chars
                pos_start = match.term.pos[0]
                add_match(node,
                          term,
                          (pos_start, pos_start + len(term)))

        for match, _, _ in grammar.scope_term_type_parser.scanString(
                node.text):
            # Check that both scope and term look valid
            if (self.scope_of_text(match.scope, Label.from_node(node),
                                   verify_prefix=False)
                    and re.match("^[a-z ]+$", match.term.tokens[0])):
                term = match.term.tokens[0].strip()
                pos_start = node.text.index(term, match.term.pos[0])
                add_match(node, term, (pos_start, pos_start + len(term)))

        if hasattr(node, 'tagged_text'):
            for match, _, _ in grammar.xml_term_parser.scanString(
                    node.tagged_text):
                """Position in match reflects XML tags, so its dropped in
                preference of new values based on node.text."""
                for match in chain([match.head], match.tail):
                    pos_start = self.pos_start_excluding(
                        match.term.tokens[0], node.text,
                        included_defs + excluded_defs)
                    term = node.tagged_text[
                        match.term.pos[0]:match.term.pos[1]].lower()
                    match_len = len(term)
                    add_match(node,
                              term,
                              (pos_start, pos_start + match_len))

        return included_defs, excluded_defs

    def pos_start_excluding(self, needle, haystack, exclusions):
        """Search for the first instance of `needle` in the `haystack`
        excluding any overlaps from `exclusions`. Implicitly returns None if
        it can't be found"""
        start = 0
        while start >= 0:
            start = haystack.find(needle, start)
            if not any(r.position[0] <= start and r.position[1] >= start
                       for r in exclusions):
                return start
            start += 1

    def subpart_scope(self, label_parts):
        """Given a label, determine which sections fall under the same
        subpart"""
        reg = label_parts[0]
        section = label_parts[1]
        for subpart in self.subpart_map:
            if section in self.subpart_map[subpart]:
                return [[reg, sect] for sect in self.subpart_map[subpart]]
        return []

    def process(self, node):
        """Determine which (if any) definitions would apply to this node,
        then find if any of those terms appear in this node"""
        applicable_terms = self.applicable_terms(node.label)

        layer_el = []
        #   Remove any definitions defined in this paragraph
        term_list = [
            (term, ref) for term, ref in applicable_terms.iteritems()
            if ref.label != node.label_id()]

        exclusions = self.excluded_offsets(node.label_id(), node.text)
        exclusions = self.per_regulation_ignores(
            exclusions, node.label, node.text)

        inclusions = self.included_offsets(node.label_id(), node.text)
        inclusions = self.per_regulation_includes(
                inclusions, node.label, node.text)

        matches = self.calculate_offsets(node.text, term_list, exclusions)
        for term, ref, offsets in matches:
            layer_el.append({
                "ref": ref.term + ':' + ref.label,
                "offsets": offsets
                })
        return layer_el

    def per_regulation_ignores(self, exclusions, label, text):
        cfr_part = label[0]
        if settings.IGNORE_DEFINITIONS_IN.get(cfr_part):
            for ignore_term in settings.IGNORE_DEFINITIONS_IN[cfr_part]:
                exclusions.extend(
                    (match.start(), match.end()) for match in
                    re.finditer(r'\b' + re.escape(ignore_term) + r'\b', text))
        return exclusions

    def excluded_offsets(self, label, text):
        """We explicitly exclude certain chunks of text (for example, words
        we are defining shouldn't have links appear within the defined
        term.) More will be added in the future"""
        exclusions = []
        for reflist in self.scoped_terms.values():
            exclusions.extend(
                ref.position for ref in reflist if ref.label == label)
        for ignore_term in settings.IGNORE_DEFINITIONS_IN['ALL']:
            exclusions.extend(
                (match.start(), match.end()) for match in
                re.finditer(r'\b' + re.escape(ignore_term) + r'\b', text))
        return exclusions

    def per_regulation_includes(self, inclusions, label, text):
        cfr_part = label[0]
        if settings.INCLUDE_DEFINITIONS_IN.get(cfr_part):
            for included_term, context in settings.INCLUDE_DEFINITIONS_IN['ALL']:
                inclusions.extend(
                    (match.start(), match.end()) for match in
                    re.finditer(r'\b' + re.escape(include_term) + r'\b', text))
        return inclusions

    def included_offsets(self, label, text):
        """ We explicitly include certain chunks of text (for example,
            words that the parser doesn't necessarily pick up as being
            defined) that should be part of a defined term """
        inclusions = []
        for included_term, context in settings.INCLUDE_DEFINITIONS_IN['ALL']:
            inclusions.extend(
                (match.start(), match.end()) for match in
                re.finditer(r'\b' + re.escape(include_term) + r'\b', text))
        return inclusions

    def calculate_offsets(self, text, applicable_terms, exclusions=[],
            inclusions=[]):
        """Search for defined terms in this text, with a preference for all
        larger (i.e. containing) terms."""

        # don't modify the original
        exclusions = list(exclusions)
        inclusions = list(inclusions)

        # add plurals to applicable terms
        pluralized = [(inflection.pluralize(t[0]), t[1]) for t in applicable_terms]
        applicable_terms += pluralized

        #   longer terms first
        applicable_terms.sort(key=lambda x: len(x[0]), reverse=True)

        matches = []
        for term, ref in applicable_terms:
            re_term = ur'\b' + re.escape(term) + ur'\b'
            offsets = [
                (m.start(), m.end())
                for m in re.finditer(re_term, text.lower())]
            safe_offsets = []
            for start, end in offsets:
                #   Start is contained in an existing def
                if any(start >= e[0] and start <= e[1] for e in exclusions):
                    continue
                #   End is contained in an existing def
                if any(end >= e[0] and end <= e[1] for e in exclusions):
                    continue
                safe_offsets.append((start, end))
            if not safe_offsets:
                continue

            exclusions.extend(safe_offsets)
            matches.append((term, ref, safe_offsets))
        return matches
<|endoftext|>