<commit_before># First we need to get the pgn file.
import sys
pgnfile = open(sys.argv[1], 'r')
pgn = pgnfile.read()

# Next we need to parse the pgn file for the evaluation scores.  The
# pgn files that this script is programed to parse are generated by
# the program Scid vs PC which encloses evaluation scores in curly
# brackets.  The variation should be stripped to ensure proper
# parsing.
def evalstring2int(es): # Turns evalstring into an integer
    digits = ""
    for char in es:
        if char in "1234567890-":
            digits += char
    return(int(digits))

def getevals(game):
    scores = []
    evalstring = ""
    recording = False
    for char in pgn:
        if recording:
            if char in "1234567890.-+\n":
                evalstring += char
            else:
                recording = False
                scores.append(evalstring2int(evalstring))
                evalstring = ""
        else:
            if char == "{":
                recording = True
    return(scores)

# Now we do the actual math.
def calcacl(evals):
    whiteloss = []
    blackloss = []
    white = True
    lastscore = 0
    for score in evals:
        if white:
            loss = lastscore - score
            if loss > 0:
                whiteloss.append(loss)
            else:
                whiteloss.append(0)
        else:
            loss = (lastscore - score) * -1
            if loss > 0:
                blackloss.append(loss)
            else:
                blackloss.append(0)
        white = not white
        lastscore = score

    avgwl = round(sum(whiteloss) / len(whiteloss))
    avgbl = round(sum(blackloss) / len(blackloss))
    return((avgwl, avgbl))

def gamespliter(pgndata):
    startingpoints = []
    point = 0
    beginstr = '[Event "'
    while point < len(pgndata):
        if pgndata[point:point + len(beginstr)] == beginstr:
            startingpoints.append(point)
        point += 1

    if len(startingpoints) == 1:
        return([pgndata])
    else:
        games = []
        counter = 0
        while counter < len(startingpoints) - 1:
            games.append(pgndata[startingpoints[counter]:startingpoints[counter + 1]])
            counter += 1
        games.append(pgndata[startingpoints[-1]:])
        return(games)

test = gamespliter(pgn)
for game in test:
    print("NEW GAME\n\n\n")
    print(game)
print(len(test))
<commit_msg>wrote pgn tag parser<commit_after># First we need to get the pgn file.
import sys
pgnfile = open(sys.argv[1], 'r')
pgn = pgnfile.read()

# Next we need to parse the pgn file for the evaluation scores.  The
# pgn files that this script is programed to parse are generated by
# the program Scid vs PC which encloses evaluation scores in curly
# brackets.  The variation should be stripped to ensure proper
# parsing.
def evalstring2int(es): # Turns evalstring into an integer
    digits = ""
    for char in es:
        if char in "1234567890-":
            digits += char
    return(int(digits))

def getevals(game):
    scores = []
    evalstring = ""
    recording = False
    for char in pgn:
        if recording:
            if char in "1234567890.-+\n":
                evalstring += char
            else:
                recording = False
                scores.append(evalstring2int(evalstring))
                evalstring = ""
        else:
            if char == "{":
                recording = True
    return(scores)

# Now we do the actual math.
def calcacl(evals):
    whiteloss = []
    blackloss = []
    white = True
    lastscore = 0
    for score in evals:
        if white:
            loss = lastscore - score
            if loss > 0:
                whiteloss.append(loss)
            else:
                whiteloss.append(0)
        else:
            loss = (lastscore - score) * -1
            if loss > 0:
                blackloss.append(loss)
            else:
                blackloss.append(0)
        white = not white
        lastscore = score

    avgwl = round(sum(whiteloss) / len(whiteloss))
    avgbl = round(sum(blackloss) / len(blackloss))
    return((avgwl, avgbl))

def gamespliter(pgndata):
    startingpoints = []
    point = 0
    beginstr = '[Event "'
    while point < len(pgndata):
        if pgndata[point:point + len(beginstr)] == beginstr:
            startingpoints.append(point)
        point += 1

    if len(startingpoints) == 1:
        return([pgndata])
    else:
        games = []
        counter = 0
        while counter < len(startingpoints) - 1:
            games.append(pgndata[startingpoints[counter]:startingpoints[counter + 1]])
            counter += 1
        games.append(pgndata[startingpoints[-1]:])
        return(games)

def parsepgntags(game):
    rawtags = []
    rawtag = ""
    intag = False
    for char in game:
        if intag:
            if char == ']':
                rawtags.append(rawtag)
                rawtag = ""
                intag = False
            else:
                rawtag += char
        else:
            if char == '[':
                intag = True

    tags = []
    for entry in rawtags:
        stage = 0
        tag = ""
        data = ""
        for char in entry:
            if stage == 0:
                if char != " ":
                    tag += char
                else:
                    stage += 1
            elif stage == 1:
                if char == '"':
                    stage += 1
            elif stage == 2:
                if char != '"':
                    data += char
                else:
                    tags.append((tag, data))

    return(tags)

test = gamespliter(pgn)
gametags = []
for game in test:
    gametags.append(parsepgntags(game))

for taglist in gametags:
    for tagpair in taglist:
        print(tagpair[0] + ': ' + tagpair[1])
    print('\n\n')

<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

from tinydb import TinyDB, Query
import aws3
import vision

#user table
#groups table
#payments table
#debt table?

#print('get db start')
#test_db = TinyDB('db/test.json')
#test_db_file = aws3.get_db('test1')
#print(test_db_file)
#test_db = TinyDB(test_db_file)

aws3_db_name = 'checkun'
db_file = aws3.get_db(aws3_db_name)
#test
#db_file = 'db/checkun.json'
#print(db_file)
db = TinyDB(db_file)
#print('get db end')

user_table = db.table('users')
group_table = db.table('groups')
payment_table = db.table('payments')
debt_table = db.table('debt')

def update_db():
    aws3.update_db(aws3_db_name)

###################
# user table
###################
def add_user(uid, name, pict, status, follow):
    ''' ユーザ追加 '''
    if is_user_exist(uid):
        update_user(uid,name,pict,status,follow)
    else:
        user_table.insert({'uid':uid, 'name':name, 'pict':pict, 'status':status, 'follow':follow})

    #save pict to user folder in S3
    if pict is not None:
        aws3.set_user_pict(uid, pict)

    update_db()

#return list
def get_user(uid):
    return user_table.search(Query().uid == uid)

#必要なら分割update_user_xxx
def update_user(uid,name,pict,status,follow):
    user_table.update({'uid':uid, 'name':name, 'pict':pict, 'status':status, 'follow':follow}, Query().uid == uid)
    update_db()

#return list
def get_users():
    return user_table.all()
    #return user_table.get(Query().uid == 'uid')
    #return user_table.get(eid = 'uid')

def delete_user(uid):
    user_table.remove(Query().uid == uid)
    update_db()

def delete_all_users():
    user_table.purge()
    update_db()

def is_user_exist(uid):
    return user_table.contains(Query().uid == uid)

###################
# group table
###################
def add_group(gid,gtype,name=None):
    ''' グループ追加
        name(Noneの場合), users, accountant, settlement_users, round は生成'''

    if is_group_exist(gid):
        update_group(gid, name)
    else:
        if name is None:
            #todo:
            name = 'xxx'

        users = []
        accountant = False
        settlement_users = []
        round_value = 100

        group_table.insert({
                            'gid':gid,
                            'type':gtype,
                            'name':name,
                            'users':users,
                            'accountant':accountant,
                            'settlement_users':settlement_users,
                            'round_value':round_value})

    update_db()

#必要なら分割update_group_xxx
def update_group(gid, name=None, accountant=None, round_value=None):
    group = {}
    if name is not None:
        group['name'] = name

    if accountant is not None:
        group['accountant'] = accountant

    if round_value is not None:
        group['round_value'] = round_value

    group_table.update(group, Query().gid == gid)

    update_db()

def get_groups():
    return group_table.all()
    #return group_table.get(eid = 'gid')

def get_group_info(gid):
    group_info = {}

    group = group_table.search(Query().gid == gid)

    print(group)
    print(len(group))

    #if group is not None:
    if len(group) == 1:
        print(group[0])
        group_info['gid'] = group[0]['gid']
        group_info['type'] = group[0]['type']
        group_info['name'] = group[0]['name']
        group_info['accountant'] = group[0]['accountant']
        group_info['round_value'] = group[0]['round_value']

    return group_info

def delete_group(gid):
    ''' グループ削除 '''
    group_table.remove(Query().gid == gid)

    update_db()

def delete_all_groups():
    group_table.purge()
    update_db()

def add_user_to_group(uid, gid):
    group = group_table.search(Query().gid == gid)

    #if group is not None:
    if len(group) == 1:
        users = group[0]['users']
        for user_id in users:
            if user_id == uid:
                return

        users.append(uid)
        group[0]['users'] = users
        group_table.update(group[0], Query().gid == gid)

        update_db()

def get_group_users(gid):
    users = []

    group = group_table.search(Query().gid == gid)

    #if group is not None:
    if len(group) == 1:
        users = group[0]['users']

    return users

def delete_user_from_group(gid, uid):
    group = group_table.search(Query().gid == gid)

    #if group is not None:
    if len(group) == 1:
        users = group[0]['users']
        for user_id in users:
            if user_id == uid:
                users.remove(user_id)
                break

        group[0]['users'] = users
        group_table.update(group[0], Query().gid == gid)

        update_db()

def get_user_groups(uid):
    ''' table(group)のusersにuidが含まれるグループデータをリストで渡す '''
    groups = []

    all_groups = group_table.all()
    for g in all_groups:
        users = g['users']
        for user_id in users:
            if user_id == uid:
                groups.append(g['gid'])
                break

    return groups

#とりあえずsettlement_usersを使用。usersにflag追加しても対応可能
def add_settlement_user_to_group(gid, uid):
    group = group_table.search(Query().gid == gid)

    #if group is not None:
    if len(group) == 1:
        users = group[0]['settlement_users']
        for user_id in users:
            if user_id == uid:
                return

        users.append(uid)
        group[0]['settlement_users'] = users
        group_table.update(group[0], Query().gid == gid)

        update_db()

def get_group_settlement_users(gid):
    settlement_users = []

    group = group_table.search(Query().gid == gid)

    #if group is not None:
    if len(group) == 1:
        settlement_users = group[0]['settlement_users']

    return settlement_users

def delete_settlement_user_from_group(gid, uid):
    group = group_table.search(Query().gid == gid)

    #if group is not None:
    if len(group) == 1:
        users = group[0]['settlement_users']
        for user_id in users:
            if user_id == uid:
                users.remove(user_id)
                break

        group[0]['settlement_users'] = users
        group_table.update(group[0], Query().gid == gid)

        update_db()

#usersではなくsettlement_usersから?userが所属するすべてのgroupの支払い一覧を返す?
def get_user_groups_payments(uid):
    ''' table(group)のusersにuidが含まれるグループのpaymentsをリストで渡す '''
    peyments = []
    return payments

#paymentsで管理?
def set_group_debt_uid_list(gid, uid_list):
    ''' table(group) の debt_uid を uid_list にする '''

def is_group_exist(gid):
    return group_table.contains(Query().gid == gid)

def is_group_user_exist(gid, uid):
    group = group_table.search(Query().gid == gid)

    #if group is not None:
    if len(group) == 1:
        users = group[0]['settlement_users']
        for user_id in users:
            if user_id == uid:
                return True

    return False

###################
# payment table
###################
def add_payment(gid, payment_uid, amount=None, description=None, receipt=None):
    ''' table(payments) に新しい支払を追加
        id, payment_date, modification_date は生成する
        imageの指定があれば、s3にアップ'''

    p_id = payment_table.count() + 1
    #todo
    payment_date = '2017/02/28'
    modification_date = payment_date
    payment_table.insert({  'gid':gid,
                            'payment_uid':payment_uid,
                            'p_id':p_id,
                            'amount':amount,
                            'description':description,
                            'receipt':receipt,
                            'payment_date':payment_date,
                            'modification_date':modification_date
                        })

    #save receipt to user folder in S3
    if receipt is not None:
        aws3.set_receipt(uid, receipt)

    update_db()

def get_payments():
    return payment_table.all()

def delete_payment(payment_id):
    ''' table(payments) の id=payment_id を削除 or 不可視にする '''
    payment_table.remove(Query().payment_id == payment_id)

    update_db()

def update_payment(payment_id, amount=None, description=None, receipt=None):
    ''' table(payments) の amount, description, image を上書きする
        modification_date更新
        imageの指定があれば、s3にアップ'''
    payment = {}

    if amount is not None:
        payment['amount'] = amount

    if description is not None:
        payment['description'] = description

    if receipt is not None:
        payment['receipt'] = receipt
        #save receipt to user folder in S3
        aws3.set_receipt(uid, receipt)

    #todo
    modification_date = '2017/02/28'
    payment['modification_date'] = modification_date
    payment_table.insert(payment, Query().payment_id == payment_id)

    update_db()

#指定groupの全ユーザーのpaymentリスト
def get_group_payment_payments(gid):
    ''' table(payments) からuserのamountりすとを渡す '''
    return payment_table.search(Query().pid == gid)

#指定group、指定ユーザーのpaymentリスト
def get_group_user_payments(gid, payment_uid):
    return payment_table.search(Query().pid == gid and Query().payment_uid == payment_uid)

#debt
#独立tableでも、groupに追加しても良い
#誰が誰に支払いするかの一覧になる

if __name__ == "__main__":
    pass
<commit_msg>bug fix<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

from tinydb import TinyDB, Query
import aws3
import vision

#user table
#groups table
#payments table
#debt table?

#print('get db start')
#test_db = TinyDB('db/test.json')
#test_db_file = aws3.get_db('test1')
#print(test_db_file)
#test_db = TinyDB(test_db_file)

aws3_db_name = 'checkun'
db_file = aws3.get_db(aws3_db_name)
#test
#db_file = 'db/checkun.json'
#print(db_file)
#db = TinyDB(db_file, indent=2, sort_keys=True, separators=(',', ': '))
#print('get db end')

user_table = db.table('users')
group_table = db.table('groups')
payment_table = db.table('payments')
debt_table = db.table('debt')

def update_db():
    aws3.update_db(aws3_db_name)

###################
# user table
###################
def add_user(uid, name, pict, status, follow):
    ''' ユーザ追加 '''
    if is_user_exist(uid):
        update_user(uid,name,pict,status,follow)
    else:
        user_table.insert({'uid':uid, 'name':name, 'pict':pict, 'status':status, 'follow':follow})

    #save pict to user folder in S3
    if pict is not None:
        aws3.set_user_pict(uid, pict)

    update_db()

#return list
def get_user(uid):
    return user_table.search(Query().uid == uid)

#必要なら分割update_user_xxx
def update_user(uid,name,pict,status,follow):
    user_table.update({'uid':uid, 'name':name, 'pict':pict, 'status':status, 'follow':follow}, Query().uid == uid)
    update_db()

#return list
def get_users():
    return user_table.all()
    #return user_table.get(Query().uid == 'uid')
    #return user_table.get(eid = 'uid')

def delete_user(uid):
    user_table.remove(Query().uid == uid)
    update_db()

def delete_all_users():
    user_table.purge()
    update_db()

def is_user_exist(uid):
    return user_table.contains(Query().uid == uid)

###################
# group table
###################
def add_group(gid,gtype,name=None):
    ''' グループ追加
        name(Noneの場合), users, accountant, settlement_users, round は生成'''

    if is_group_exist(gid):
        update_group(gid, name)
    else:
        if name is None:
            #todo:
            name = gid

        users = []
        accountant = False
        settlement_users = []
        round_value = 100

        group_table.insert({
                            'gid':gid,
                            'type':gtype,
                            'name':name,
                            'users':users,
                            'accountant':accountant,
                            'settlement_users':settlement_users,
                            'round_value':round_value})

    update_db()

#必要なら分割update_group_xxx
def update_group(gid, name=None, accountant=None, round_value=None):
    group = {}
    if name is not None:
        group['name'] = name

    if accountant is not None:
        group['accountant'] = accountant

    if round_value is not None:
        group['round_value'] = round_value

    group_table.update(group, Query().gid == gid)

    update_db()

def get_groups():
    return group_table.all()
    #return group_table.get(eid = 'gid')

def get_group_info(gid):
    group_info = {}

    group = group_table.search(Query().gid == gid)

    if group:
        # print(group[0])
        return group[0]
    return None

def delete_group(gid):
    ''' グループ削除 '''
    group_table.remove(Query().gid == gid)
    payment_table.remove(Query().gid == gid)

    update_db()

def delete_all_groups():
    group_table.purge()
    update_db()

def add_user_to_group(uid, gid):
    group = group_table.search(Query().gid == gid)

    #if group is not None:
    if len(group) == 1:
        users = group[0]['users']
        for user_id in users:
            if user_id == uid:
                return

        users.append(uid)
        group[0]['users'] = users
        group_table.update(group[0], Query().gid == gid)

        update_db()

def get_group_users(gid):
    users = []

    group = group_table.search(Query().gid == gid)

    #if group is not None:
    if len(group) == 1:
        users = group[0]['users']

    return users

def delete_user_from_group(gid, uid):
    group = group_table.search(Query().gid == gid)

    #if group is not None:
    if len(group) == 1:
        users = group[0]['users']
        for user_id in users:
            if user_id == uid:
                users.remove(user_id)
                break

        group[0]['users'] = users
        group_table.update(group[0], Query().gid == gid)

        update_db()

def get_user_groups(uid):
    ''' table(group)のusersにuidが含まれるグループデータをリストで渡す '''
    groups = []

    all_groups = group_table.all()
    for g in all_groups:
        users = g['users']
        for user_id in users:
            if user_id == uid:
                groups.append(g['gid'])
                break

    return groups

#とりあえずsettlement_usersを使用。usersにflag追加しても対応可能
def add_settlement_user_to_group(gid, uid):
    group = group_table.search(Query().gid == gid)

    #if group is not None:
    if len(group) == 1:
        users = group[0]['settlement_users']
        for user_id in users:
            if user_id == uid:
                return

        users.append(uid)
        group[0]['settlement_users'] = users
        group_table.update(group[0], Query().gid == gid)

        update_db()

def get_group_settlement_users(gid):
    settlement_users = []

    group = group_table.search(Query().gid == gid)

    #if group is not None:
    if len(group) == 1:
        settlement_users = group[0]['settlement_users']

    return settlement_users

def delete_settlement_user_from_group(gid, uid):
    group = group_table.search(Query().gid == gid)

    #if group is not None:
    if len(group) == 1:
        users = group[0]['settlement_users']
        for user_id in users:
            if user_id == uid:
                users.remove(user_id)
                break

        group[0]['settlement_users'] = users
        group_table.update(group[0], Query().gid == gid)

        update_db()

def get_groups_payments(gid):
    return payment_table.search(Query().gid == gid)

#usersではなくsettlement_usersから?userが所属するすべてのgroupの支払い一覧を返す?
def get_user_groups_payments(uid):
    ''' table(group)のusersにuidが含まれるグループのpaymentsをリストで渡す '''
    payments = payment_table.search(Query().gid == gid)
    return payments

#paymentsで管理?
def set_group_debt_uid_list(gid, uid_list):
    ''' table(group) の debt_uid を uid_list にする '''

def is_group_exist(gid):
    return group_table.contains(Query().gid == gid)

def is_group_user_exist(gid, uid):
    group = group_table.search(Query().gid == gid)

    #if group is not None:
    if len(group) == 1:
        users = group[0]['settlement_users']
        for user_id in users:
            if user_id == uid:
                return True

    return False

###################
# payment table
###################
def add_payment(gid, payment_uid, amount=None, description=None, receipt=None):
    ''' table(payments) に新しい支払を追加
        id, payment_date, modification_date は生成する
        imageの指定があれば、s3にアップ'''

    p_id = len(payment_table) + 1
    #todo
    payment_date = '2017/02/28'
    modification_date = payment_date
    payment_table.insert({  'gid':gid,
                            'payment_uid':payment_uid,
                            'p_id':p_id,
                            'amount':amount,
                            'description':description,
                            'receipt':receipt,
                            'payment_date':payment_date,
                            'modification_date':modification_date
                        })

    #save receipt to user folder in S3
    if receipt is not None:
        aws3.set_receipt(uid, receipt)
        # pass

    update_db()

def get_payments():
    return payment_table.all()

def delete_payment(payment_id):
    ''' table(payments) の id=payment_id を削除 or 不可視にする '''
    payment_table.remove(Query().payment_id == payment_id)

    update_db()

def update_payment(payment_id, amount=None, description=None, receipt=None):
    ''' table(payments) の amount, description, image を上書きする
        modification_date更新
        imageの指定があれば、s3にアップ'''
    payment = {}

    if amount is not None:
        payment['amount'] = amount

    if description is not None:
        payment['description'] = description

    if receipt is not None:
        payment['receipt'] = receipt
        #save receipt to user folder in S3
        aws3.set_receipt(uid, receipt)

    #todo
    modification_date = '2017/02/28'
    payment['modification_date'] = modification_date
    payment_table.insert(payment, Query().payment_id == payment_id)

    update_db()

#指定groupの全ユーザーのpaymentリスト
def get_group_payment_payments(gid):
    ''' table(payments) からuserのamountりすとを渡す '''
    return payment_table.search(Query().gid == gid)

#指定group、指定ユーザーのpaymentリスト
def get_group_user_payments(gid, payment_uid):
    return payment_table.search(Query().pid == gid and Query().payment_uid == payment_uid)

#debt
#独立tableでも、groupに追加しても良い
#誰が誰に支払いするかの一覧になる

if __name__ == "__main__":
    pass
<|endoftext|>


<commit_before>import time
import logging
import traceback
import pandas as pd
import numpy as np

from collections import OrderedDict
from collections.abc import Sequence
from collections.abc import Mapping
from toolz.dicttoolz import merge

from sanic import Sanic
from sanic.config import Config
from sanic.response import text, json
from aiocache import cached

from survey_stats.log import logger
from survey_stats import error as sserr
from survey_stats import settings
from survey_stats import fetch
from survey_stats import state as st
from survey_stats.processify import processify
from survey_stats.error import SSEmptyFilterError, SSInvalidUsage

Config.REQUEST_TIMEOUT = 50000000

app = Sanic(__name__)

@app.route("/v2/questions")
def fetch_questions(req):
    def get_meta(k, v, dset):
        key = k.lower()
        res = (dict(st.meta[dset].qnmeta_dict[key], **v, id=k) if key in
               st.meta[dset].qnmeta_dict else dict(v, id=k))
        return res
    dset=req.args.get('d')
    national = True
    combined = True
    svy = st.dset[dset].fetch_survey(combined, national)
    res = []
    if svy:
        res = {k: get_meta(k, v, dset) for k, v in svy.vars.items()}
    else:
        qnkey = st.meta[dset].config['qnkey']
        res = st.meta[dset].qnmeta.reset_index(level=0)
        sl_res =[]
        sl_res = res[['facet', 'facet_description',
            'facet_level', 'facet_level_value']].drop_duplicates()
        sl_res = {f[0]: {
            'facet':f[0],
            'facet_description': f[1]['facet_description'].get_values()[0],
            'levels': dict(list(f[1][['facet_level','facet_level_value']].to_records(index=False)))
            } for f in sl_res.groupby('facet')}

        qn_res = res[['questionid', 'topic', 'subtopic', 'question', 'response']].groupby('questionid').agg({
            'topic': lambda x: x.head(1).get_values()[0],
            'subtopic': lambda x: x.get_values()[0],
            'question': lambda x: x.get_values()[0],
            'response': lambda x: list(x.drop_duplicates())
        })
        # qn_res = qn_res['response'].unique()
        # dash = st.meta[dset].dash
        # logger.info(dash.columns)
        # logger.info(res['response'].value_counts().to_dict())

        qn_res = qn_res.to_dict(orient='index')
        res = {'facets':sl_res, 'questions':qn_res}
        #logger.info(res)
    return json(res)

@app.route("/questions")
def fetch_questions(req):
    def get_meta(k, v, dset):
        key = k.lower()
        res = (dict(st.meta[dset].qnmeta_dict[key], **v, id=k) if key in
               st.meta[dset].qnmeta_dict else dict(v, id=k))
        return res
    dset=req.args.get('d')
    national = True
    combined = True
    svy = st.dset[dset].fetch_survey(combined, national)
    res = []
    if svy:
        res = {k: get_meta(k, v, dset) for k, v in svy.vars.items()}
    else:
        qnkey = st.meta[dset].config['qnkey']
        res = st.meta[dset].qnmeta.reset_index(level=0)
        sl_res =[]
        sl_res = res[['facet', 'facet_description',
            'facet_level', 'facet_level_value']].drop_duplicates()
        sl_res = {f[0]: {
            'facet':f[0],
            'facet_description': f[1]['facet_description'].get_values()[0],
            'levels': dict(list(f[1][['facet_level','facet_level_value']].to_records(index=False)))
            } for f in sl_res.groupby('facet')}

        qn_res = res[['questionid', 'topic', 'subtopic', 'question', 'response']].groupby('questionid').agg({
            'topic': lambda x: x.head(1).get_values()[0],
            'subtopic': lambda x: x.get_values()[0],
            'question': lambda x: x.get_values()[0],
            'response': lambda x: list(x.drop_duplicates())
        })
        # qn_res = qn_res['response'].unique()
        # dash = st.meta[dset].dash
        # logger.info(dash.columns)
        # logger.info(res['response'].value_counts().to_dict())

        qn_res = qn_res.to_dict(orient='index')
        res = qn_res
        #logger.info(res)
    return json(res)

def remap_vars(cfg, coll, into=True):
    def map_if(pv, k):
        return pv[k] if k in pv else k
    pv = ({v: k for k, v in cfg['pop_vars'].items()} if
          not into else cfg['pop_vars'])
    res = None
    typ = type(coll)
    if isinstance(coll, str):
        res = coll
    elif isinstance(coll, Sequence):
        res = [map_if(pv, k) for k in coll]
    elif isinstance(coll, Mapping):
        res = {map_if(pv, k): remap_vars(cfg, v, into) for
               k, v in coll.items()}
    else:
        res = coll
    return res

def gen_slices(k, svy, qn, resp, m_vars, m_filt):
    loc = {'svy_id': k, 'dset_id': 'yrbss'}
    slices = [merge(loc, s)
        for s in svy.generate_slices(qn, resp, m_vars, m_filt) ]
    return slices

async def fetch_computed(k, svy, qn, resp, m_vars, m_filt, cfg):
    slices = gen_slices(k, svy, qn, resp, m_vars, m_filt)
    results = await fetch.fetch_all(slices)
    results = [remap_vars(cfg, x, into=False) for x in results]
    return results

def fetch_socrata(qn, resp, vars, filt, meta):
    precomp = meta.fetch_dash(qn, resp, vars, filt)
    precomp = pd.DataFrame(precomp).fillna(-1)
    precomp['method']='socrata'
    return precomp.to_dict(orient='records')

def parse_filter(f):
    return dict(map(lambda fv: (fv.split(':')[0],
                                fv.split(':')[1].split(',')), f.split('|')))

def parse_response(r):
    if r.lower() == 'yes' or r.lower() == 'true' or r.lower() == '1':
        return True
    elif r.lower() == 'no' or r.lower() == 'false' or r.lower() == '0':
        return False
    else:
        raise Exception('Invalid response value specified!')

@cached(ttl=24*60*60)
@app.route('/stats')
async def fetch_survey_stats(req):
    dset = req.args.get('d')
    qn = req.args.get('q')
    vars = [] if not 'v' in req.args else req.args.get('v').split(',')
    resp = None if not 'r' in req.args else parse_response(req.args.get('r'))
    filt = {} if not 'f' in req.args else parse_filter(req.args.get('f'))
    use_socrata = False if not 's' in req.args else not 0 ** int(req.args.get('s'), 2)
    meta = st.meta[dset]
    question = qn #meta.qnmeta[qn]
    results = None #fetch_socrata(qn, resp, vars, filt, national, meta)
    error = None
    try:
        if not use_socrata:
            (k, cfg) = st.dset[dset].fetch_config(national=True, year=None)
            svy = st.dset[dset].surveys[k]
            m_filt = remap_vars(cfg, filt, into=True)
            m_vars = remap_vars(cfg, vars, into=True)
            if not svy.subset(m_filt).sample_size > 1:
                raise SSEmptyFilterError('EmptyFilterError: %s' % (str(m_filt)))
            question = svy.vars[qn]['question']
            var_levels = remap_vars(cfg, {v: svy.vars[v] for v in m_vars}, into=False)
            results = await fetch_computed(k, svy, qn, resp, m_vars, m_filt, cfg)
        else:
            results = fetch_socrata(qn, resp, vars, filt, meta)
    except Exception as e:
        error = str(e)
    return json({
        'error': error,
        'q': qn,
        'filter': filt,
        'question': question,
        'response': resp,
        'vars': vars,
        'var_levels': None, #var_levels,
        'results': results,
        'is_socrata':use_socrata
    })

def serve_app(host, port, workers, debug):
    app.run(host=host, port=port, workers=workers, debug=debug)

if __name__ == '__main__':
    serve_app(host='0.0.0.0', port=7778, workers=1, debug=True)
<commit_msg>clean up commented out code<commit_after>import time
import logging
import traceback
import pandas as pd
import numpy as np

from collections import OrderedDict
from collections.abc import Sequence
from collections.abc import Mapping
from toolz.dicttoolz import merge

from sanic import Sanic
from sanic.config import Config
from sanic.response import text, json
from aiocache import cached

from survey_stats.log import logger
from survey_stats import error as sserr
from survey_stats import settings
from survey_stats import fetch
from survey_stats import state as st
from survey_stats.processify import processify
from survey_stats.error import SSEmptyFilterError, SSInvalidUsage

Config.REQUEST_TIMEOUT = 50000000

app = Sanic(__name__)

@app.route("/v2/questions")
def fetch_questions(req):
    def get_meta(k, v, dset):
        key = k.lower()
        res = (dict(st.meta[dset].qnmeta_dict[key], **v, id=k) if key in
               st.meta[dset].qnmeta_dict else dict(v, id=k))
        return res
    dset=req.args.get('d')
    national = True
    combined = True
    svy = st.dset[dset].fetch_survey(combined, national)
    res = []
    if svy:
        res = {k: get_meta(k, v, dset) for k, v in svy.vars.items()}
    else:
        qnkey = st.meta[dset].config['qnkey']
        res = st.meta[dset].qnmeta.reset_index(level=0)
        sl_res =[]
        sl_res = res[['facet', 'facet_description',
            'facet_level', 'facet_level_value']].drop_duplicates()
        sl_res = {f[0]: {
            'facet':f[0],
            'facet_description': f[1]['facet_description'].get_values()[0],
            'levels': dict(list(f[1][['facet_level','facet_level_value']].to_records(index=False)))
            } for f in sl_res.groupby('facet')}

        qn_res = res[['questionid', 'topic', 'subtopic', 'question', 'response']].groupby('questionid').agg({
            'topic': lambda x: x.head(1).get_values()[0],
            'subtopic': lambda x: x.get_values()[0],
            'question': lambda x: x.get_values()[0],
            'response': lambda x: list(x.drop_duplicates())
        })
        qn_res = qn_res.to_dict(orient='index')
        res = {'facets':sl_res, 'questions':qn_res}
        #logger.info(res)
    return json(res)

@app.route("/questions")
def fetch_questions(req):
    def get_meta(k, v, dset):
        key = k.lower()
        res = (dict(st.meta[dset].qnmeta_dict[key], **v, id=k) if key in
               st.meta[dset].qnmeta_dict else dict(v, id=k))
        return res
    dset=req.args.get('d')
    national = True
    combined = True
    svy = st.dset[dset].fetch_survey(combined, national)
    res = []
    if svy:
        res = {k: get_meta(k, v, dset) for k, v in svy.vars.items()}
    else:
        qnkey = st.meta[dset].config['qnkey']
        res = st.meta[dset].qnmeta.reset_index(level=0)
        sl_res =[]
        sl_res = res[['facet', 'facet_description',
            'facet_level', 'facet_level_value']].drop_duplicates()
        sl_res = {f[0]: {
            'facet':f[0],
            'facet_description': f[1]['facet_description'].get_values()[0],
            'levels': dict(list(f[1][['facet_level','facet_level_value']].to_records(index=False)))
            } for f in sl_res.groupby('facet')}

        qn_res = res[['questionid', 'topic', 'subtopic', 'question', 'response']].groupby('questionid').agg({
            'topic': lambda x: x.head(1).get_values()[0],
            'subtopic': lambda x: x.get_values()[0],
            'question': lambda x: x.get_values()[0],
            'response': lambda x: list(x.drop_duplicates())
        })
        qn_res = qn_res.to_dict(orient='index')
        res = qn_res
        #logger.info(res)
    return json(res)

def remap_vars(cfg, coll, into=True):
    def map_if(pv, k):
        return pv[k] if k in pv else k
    pv = ({v: k for k, v in cfg['pop_vars'].items()} if
          not into else cfg['pop_vars'])
    res = None
    typ = type(coll)
    if isinstance(coll, str):
        res = coll
    elif isinstance(coll, Sequence):
        res = [map_if(pv, k) for k in coll]
    elif isinstance(coll, Mapping):
        res = {map_if(pv, k): remap_vars(cfg, v, into) for
               k, v in coll.items()}
    else:
        res = coll
    return res

def gen_slices(k, svy, qn, resp, m_vars, m_filt):
    loc = {'svy_id': k, 'dset_id': 'yrbss'}
    slices = [merge(loc, s)
        for s in svy.generate_slices(qn, resp, m_vars, m_filt) ]
    return slices

async def fetch_computed(k, svy, qn, resp, m_vars, m_filt, cfg):
    slices = gen_slices(k, svy, qn, resp, m_vars, m_filt)
    results = await fetch.fetch_all(slices)
    results = [remap_vars(cfg, x, into=False) for x in results]
    return results

def fetch_socrata(qn, resp, vars, filt, meta):
    precomp = meta.fetch_dash(qn, resp, vars, filt)
    precomp = pd.DataFrame(precomp).fillna(-1)
    precomp['method']='socrata'
    return precomp.to_dict(orient='records')

def parse_filter(f):
    return dict(map(lambda fv: (fv.split(':')[0],
                                fv.split(':')[1].split(',')), f.split('|')))

def parse_response(r):
    if r.lower() == 'yes' or r.lower() == 'true' or r.lower() == '1':
        return True
    elif r.lower() == 'no' or r.lower() == 'false' or r.lower() == '0':
        return False
    else:
        raise Exception('Invalid response value specified!')

@cached(ttl=24*60*60)
@app.route('/stats')
async def fetch_survey_stats(req):
    dset = req.args.get('d')
    qn = req.args.get('q')
    vars = [] if not 'v' in req.args else req.args.get('v').split(',')
    resp = None if not 'r' in req.args else parse_response(req.args.get('r'))
    filt = {} if not 'f' in req.args else parse_filter(req.args.get('f'))
    use_socrata = False if not 's' in req.args else not 0 ** int(req.args.get('s'), 2)
    meta = st.meta[dset]
    question = qn #meta.qnmeta[qn]
    results = None #fetch_socrata(qn, resp, vars, filt, national, meta)
    error = None
    try:
        if not use_socrata:
            (k, cfg) = st.dset[dset].fetch_config(national=True, year=None)
            svy = st.dset[dset].surveys[k]
            m_filt = remap_vars(cfg, filt, into=True)
            m_vars = remap_vars(cfg, vars, into=True)
            if not svy.subset(m_filt).sample_size > 1:
                raise SSEmptyFilterError('EmptyFilterError: %s' % (str(m_filt)))
            question = svy.vars[qn]['question']
            var_levels = remap_vars(cfg, {v: svy.vars[v] for v in m_vars}, into=False)
            results = await fetch_computed(k, svy, qn, resp, m_vars, m_filt, cfg)
        else:
            results = fetch_socrata(qn, resp, vars, filt, meta)
    except Exception as e:
        error = str(e)
    return json({
        'error': error,
        'q': qn,
        'filter': filt,
        'question': question,
        'response': resp,
        'vars': vars,
        'var_levels': None, #var_levels,
        'results': results,
        'is_socrata':use_socrata
    })

def serve_app(host, port, workers, debug):
    app.run(host=host, port=port, workers=workers, debug=debug)

if __name__ == '__main__':
    serve_app(host='0.0.0.0', port=7778, workers=1, debug=True)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright (c) 2014, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

import unittest
import mock

from openquake.engine.calculators.risk import validation
from openquake.engine.db import models

from openquake.risklib.workflows import RiskModel

class HazardIMTTestCase(unittest.TestCase):
    def test_get_error(self):
        vf1 = mock.Mock()
        vf1.imt = 'PGA'
        vf2 = mock.Mock()
        vf2.imt = 'PGV'
        calc = mock.Mock()
        workflow = mock.Mock()
        workflow.vulnerability_functions = dict(
            structural=vf1, nonstructural=vf2)
        calc.risk_models = {
            'tax1': RiskModel('tax1', workflow),
            'tax2': RiskModel('tax2', workflow)}
        calc.hc.get_imts = mock.Mock(return_value=['PGA', 'PGV'])
        val = validation.HazardIMT(calc)

        self.assertIsNone(val.get_error())
        calc.hc.get_imts = mock.Mock(return_value=['PGA'])
        self.assertEqual(("There is no hazard output for: PGV. "
                          "The available IMTs are: PGA."), val.get_error())

class EmptyExposureTestCase(unittest.TestCase):
    def test_get_error(self):
        calc = mock.Mock()
        calc.taxonomies_asset_count = {}
        val = validation.EmptyExposure(calc)

        self.assertEqual(('Region of interest is not covered by '
                          'the exposure input. '
                          'This configuration is invalid. '
                          'Change the region constraint input or use a proper '
                          'exposure'), val.get_error())

        calc.taxonomies_asset_count = {'RM': 1, 'RC': 2}
        self.assertIsNone(val.get_error())

class OrphanTaxonomiesTestCase(unittest.TestCase):
    def test_get_error(self):
        calc = mock.Mock()
        val = validation.OrphanTaxonomies(calc)

        calc.rc.taxonomies_from_model = True
        calc.risk_models = {'RM': mock.Mock()}
        calc.taxonomies_asset_count = {'RC': 1, 'RM': 2}

        self.assertIsNone(val.get_error())

        calc.rc.taxonomies_from_model = False
        self.assertEqual("The following taxonomies are in the exposure model "
                         "but not in the risk model: set(['RC'])",
                         val.get_error())

        calc.risk_models = {'RM': mock.Mock(), 'RC': mock.Mock()}
        self.assertIsNone(val.get_error())

class ExposureLossTypesTestCase(unittest.TestCase):
    def test_get_error(self):
        calc = mock.Mock()
        val = validation.ExposureLossTypes(calc)

        calc.loss_types = models.LOSS_TYPES
        calc.risk_models = {'RM': mock.Mock()}

        calc.rc.exposure_model.supports_loss_type = mock.Mock(
            return_value=False)

        self.assertEqual(("Invalid exposure "
                          "for computing loss type structural. "),
                         val.get_error())

        calc.rc.exposure_model.supports_loss_type = mock.Mock(
            return_value=True)
        self.assertIsNone(val.get_error())

class NoRiskModelsTestCase(unittest.TestCase):
    def test_get_error(self):
        calc = mock.Mock()
        calc.risk_models = None

        val = validation.NoRiskModels(calc)
        self.assertEqual(
            'At least one risk model of type %s must be defined' % (
                models.LOSS_TYPES), val.get_error())

        calc.risk_models = {'RM': mock.Mock()}
        self.assertIsNone(val.get_error())

class RequireClassicalHazardTestCase(unittest.TestCase):
    def test_get_error(self):
        calc = mock.Mock()

        val = validation.RequireClassicalHazard(calc)

        calc.rc.hazard_calculation.calculation_mode = 'classical'
        self.assertIsNone(val.get_error())

        calc.rc.hazard_calculation.calculation_mode = 'event_based'
        self.assertEqual(("The provided hazard calculation ID "
                          "is not a classical calculation"), val.get_error())

        calc.rc.hazard_calculation = None
        calc.rc.hazard_output.is_hazard_curve = mock.Mock(return_value=True)
        self.assertIsNone(val.get_error())

        calc.rc.hazard_output.is_hazard_curve = mock.Mock(return_value=False)
        self.assertEqual("The provided hazard output is not an hazard curve",
                         val.get_error())

class RequireScenarioHazardTestCase(unittest.TestCase):
    def test_get_error(self):
        calc = mock.Mock()

        val = validation.RequireScenarioHazard(calc)

        calc.rc.hazard_calculation.calculation_mode = 'scenario'
        self.assertIsNone(val.get_error())

        calc.rc.hazard_calculation.calculation_mode = 'event_based'
        self.assertEqual(("The provided hazard calculation ID "
                          "is not a scenario calculation"), val.get_error())

        calc.rc.hazard_calculation = None
        calc.rc.hazard_output.output_type = "gmf_scenario"
        self.assertIsNone(val.get_error())

        calc.rc.hazard_output.output_type = "gmf"
        self.assertEqual(("The provided hazard is not a "
                          "gmf scenario collection"),
                         val.get_error())

class RequireEventBasedHazardTestCase(unittest.TestCase):
    def test_get_error(self):
        calc = mock.Mock()

        output = mock.Mock()
        output.output_type = "gmf"
        calc.rc.hazard_outputs.return_value = [output]

        val = validation.RequireEventBasedHazard(calc)

        calc.rc.hazard_calculation.calculation_mode = 'event_based'
        self.assertIsNone(val.get_error())

        output.output_type = "ses"
        calc.rc.inputs = {}
        self.assertIsNotNone(val.get_error())

        calc.rc.inputs = {'source_model_logic_tree': None}
        self.assertIsNotNone(val.get_error())

        calc.rc.hazard_calculation.calculation_mode = 'classical'
        self.assertEqual(("The provided hazard calculation ID "
                          "is not a event based calculation"), val.get_error())

        calc.rc.hazard_calculation = None
        calc.rc.hazard_output.output_type = "ses"
        self.assertIsNotNone(val.get_error())

        calc.rc.hazard_output.output_type = "gmf_scenario"
        self.assertEqual(("The provided hazard is not a "
                          "gmf or ses collection"),
                         val.get_error())

class ExposureHasInsuranceBoundsTestCase(unittest.TestCase):
    def test_get_error(self):
        calc = mock.Mock()

        val = validation.ExposureHasInsuranceBounds(calc)

        calc.rc.insured_losses = True
        calc.rc.exposure_model.has_insurance_bounds = mock.Mock(
            return_value=True)
        self.assertIsNone(val.get_error())

        calc.rc.insured_losses = True
        calc.rc.exposure_model.has_insurance_bounds = mock.Mock(
            return_value=False)
        self.assertEqual("Deductible or insured limit missing in exposure",
                         val.get_error())

        calc.rc.insured_losses = False
        calc.rc.exposure_model.has_insurance_bounds = mock.Mock(
            return_value=True)
        self.assertIsNone(val.get_error())

        calc.rc.insured_losses = False
        calc.rc.exposure_model.has_insurance_bounds = mock.Mock(
            return_value=False)
        self.assertIsNone(val.get_error())

class ExposureHasRetrofittedCostsTestCase(unittest.TestCase):
    def test_get_error(self):
        calc = mock.Mock()

        val = validation.ExposureHasRetrofittedCosts(calc)

        calc.rc.exposure_model.has_retrofitted_costs = mock.Mock(
            return_value=True)
        self.assertIsNone(val.get_error())

        calc.rc.exposure_model.has_retrofitted_costs = mock.Mock(
            return_value=False)
        self.assertEqual("Some assets do not have retrofitted costs",
                         val.get_error())

class ExposureHasTimeEventTestCase(unittest.TestCase):
    def test_get_error(self):
        calc = mock.Mock()
        val = validation.ExposureHasTimeEvent(calc)

        calc.rc.time_event = "night"

        calc.rc.exposure_model.has_time_event = mock.Mock(
            return_value=True)
        self.assertIsNone(val.get_error())

        calc.rc.exposure_model.has_time_event = mock.Mock(
            return_value=False)
        self.assertEqual("Some assets are missing an "
                         "occupancy with period=night", val.get_error())
<commit_msg>Added some explanations on risk/validation_test.py<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright (c) 2014, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

import unittest
import mock

from openquake.engine.calculators.risk import validation
from openquake.engine.db import models

from openquake.risklib.workflows import RiskModel

class HazardIMTTestCase(unittest.TestCase):
    def test_get_error(self):
        vf1 = mock.Mock()
        vf1.imt = 'PGA'
        vf2 = mock.Mock()
        vf2.imt = 'PGV'
        calc = mock.Mock()
        workflow = mock.Mock()
        workflow.vulnerability_functions = dict(
            structural=vf1, nonstructural=vf2)
        calc.risk_models = {
            'tax1': RiskModel('tax1', workflow),
            'tax2': RiskModel('tax2', workflow)}
        calc.hc.get_imts = mock.Mock(return_value=['PGA', 'PGV'])
        val = validation.HazardIMT(calc)

        self.assertIsNone(val.get_error())
        calc.hc.get_imts = mock.Mock(return_value=['PGA'])
        self.assertEqual(("There is no hazard output for: PGV. "
                          "The available IMTs are: PGA."), val.get_error())

class EmptyExposureTestCase(unittest.TestCase):
    def test_get_error(self):
        calc = mock.Mock()
        calc.taxonomies_asset_count = {}
        val = validation.EmptyExposure(calc)

        self.assertEqual(('Region of interest is not covered by '
                          'the exposure input. '
                          'This configuration is invalid. '
                          'Change the region constraint input or use a proper '
                          'exposure'), val.get_error())

        calc.taxonomies_asset_count = {'RM': 1, 'RC': 2}
        self.assertIsNone(val.get_error())

class OrphanTaxonomiesTestCase(unittest.TestCase):
    def test_get_error(self):
        calc = mock.Mock()
        val = validation.OrphanTaxonomies(calc)

        calc.rc.taxonomies_from_model = True
        calc.risk_models = {'RM': mock.Mock()}
        calc.taxonomies_asset_count = {'RC': 1, 'RM': 2}

        self.assertIsNone(val.get_error())

        calc.rc.taxonomies_from_model = False
        self.assertEqual("The following taxonomies are in the exposure model "
                         "but not in the risk model: set(['RC'])",
                         val.get_error())

        calc.risk_models = {'RM': mock.Mock(), 'RC': mock.Mock()}
        self.assertIsNone(val.get_error())

class ExposureLossTypesTestCase(unittest.TestCase):
    def test_get_error(self):
        calc = mock.Mock()
        val = validation.ExposureLossTypes(calc)

        calc.loss_types = models.LOSS_TYPES
        calc.risk_models = {'RM': mock.Mock()}

        calc.rc.exposure_model.supports_loss_type = mock.Mock(
            return_value=False)

        self.assertEqual(("Invalid exposure "
                          "for computing loss type structural. "),
                         val.get_error())

        calc.rc.exposure_model.supports_loss_type = mock.Mock(
            return_value=True)
        self.assertIsNone(val.get_error())

class NoRiskModelsTestCase(unittest.TestCase):
    def test_get_error(self):
        calc = mock.Mock()
        calc.risk_models = None

        val = validation.NoRiskModels(calc)
        self.assertEqual(
            'At least one risk model of type %s must be defined' % (
                models.LOSS_TYPES), val.get_error())

        calc.risk_models = {'RM': mock.Mock()}
        self.assertIsNone(val.get_error())

class RequireClassicalHazardTestCase(unittest.TestCase):
    def test_get_error(self):
        calc = mock.Mock()

        val = validation.RequireClassicalHazard(calc)

        calc.rc.hazard_calculation.calculation_mode = 'classical'
        self.assertIsNone(val.get_error())

        calc.rc.hazard_calculation.calculation_mode = 'event_based'
        self.assertEqual(("The provided hazard calculation ID "
                          "is not a classical calculation"), val.get_error())

        calc.rc.hazard_calculation = None
        calc.rc.hazard_output.is_hazard_curve = mock.Mock(return_value=True)
        self.assertIsNone(val.get_error())

        calc.rc.hazard_output.is_hazard_curve = mock.Mock(return_value=False)
        self.assertEqual("The provided hazard output is not an hazard curve",
                         val.get_error())

class RequireScenarioHazardTestCase(unittest.TestCase):
    def test_get_error(self):
        calc = mock.Mock()

        val = validation.RequireScenarioHazard(calc)

        calc.rc.hazard_calculation.calculation_mode = 'scenario'
        self.assertIsNone(val.get_error())

        calc.rc.hazard_calculation.calculation_mode = 'event_based'
        self.assertEqual(("The provided hazard calculation ID "
                          "is not a scenario calculation"), val.get_error())

        calc.rc.hazard_calculation = None
        calc.rc.hazard_output.output_type = "gmf_scenario"
        self.assertIsNone(val.get_error())

        calc.rc.hazard_output.output_type = "gmf"
        self.assertEqual(("The provided hazard is not a "
                          "gmf scenario collection"),
                         val.get_error())

class RequireEventBasedHazardTestCase(unittest.TestCase):
    def test_get_error(self):
        calc = mock.Mock()

        output = mock.Mock()
        output.output_type = "gmf"
        calc.rc.hazard_outputs.return_value = [output]

        val = validation.RequireEventBasedHazard(calc)

        calc.rc.hazard_calculation.calculation_mode = 'event_based'
        # here calc.rc.inputs is a Mock object and therefore
        # hc.gsim_logic_tree_file is a not None Mock object;
        # no validation error should be raised
        self.assertIsNone(val.get_error())

        output.output_type = "ses"
        calc.rc.inputs = {}  # hence no gsim_logic_tree_file
        self.assertEqual(
            val.get_error(), 'gsim_logic_tree_file is mandatory when the '
            'hazard output is a ses collection')

        calc.rc.hazard_calculation.calculation_mode = 'classical'
        self.assertEqual(("The provided hazard calculation ID "
                          "is not a event based calculation"), val.get_error())

        calc.rc.hazard_calculation = None
        calc.rc.hazard_output.output_type = "ses"
        # we must get an error because there is no gsim_logic_tree_file
        self.assertEqual(
            val.get_error(), 'gsim_logic_tree_file is mandatory when the '
            'hazard output is a ses collection')

        calc.rc.hazard_output.output_type = "gmf_scenario"
        self.assertEqual(("The provided hazard is not a "
                          "gmf or ses collection"),
                         val.get_error())

class ExposureHasInsuranceBoundsTestCase(unittest.TestCase):
    def test_get_error(self):
        calc = mock.Mock()

        val = validation.ExposureHasInsuranceBounds(calc)

        calc.rc.insured_losses = True
        calc.rc.exposure_model.has_insurance_bounds = mock.Mock(
            return_value=True)
        self.assertIsNone(val.get_error())

        calc.rc.insured_losses = True
        calc.rc.exposure_model.has_insurance_bounds = mock.Mock(
            return_value=False)
        self.assertEqual("Deductible or insured limit missing in exposure",
                         val.get_error())

        calc.rc.insured_losses = False
        calc.rc.exposure_model.has_insurance_bounds = mock.Mock(
            return_value=True)
        self.assertIsNone(val.get_error())

        calc.rc.insured_losses = False
        calc.rc.exposure_model.has_insurance_bounds = mock.Mock(
            return_value=False)
        self.assertIsNone(val.get_error())

class ExposureHasRetrofittedCostsTestCase(unittest.TestCase):
    def test_get_error(self):
        calc = mock.Mock()

        val = validation.ExposureHasRetrofittedCosts(calc)

        calc.rc.exposure_model.has_retrofitted_costs = mock.Mock(
            return_value=True)
        self.assertIsNone(val.get_error())

        calc.rc.exposure_model.has_retrofitted_costs = mock.Mock(
            return_value=False)
        self.assertEqual("Some assets do not have retrofitted costs",
                         val.get_error())

class ExposureHasTimeEventTestCase(unittest.TestCase):
    def test_get_error(self):
        calc = mock.Mock()
        val = validation.ExposureHasTimeEvent(calc)

        calc.rc.time_event = "night"

        calc.rc.exposure_model.has_time_event = mock.Mock(
            return_value=True)
        self.assertIsNone(val.get_error())

        calc.rc.exposure_model.has_time_event = mock.Mock(
            return_value=False)
        self.assertEqual("Some assets are missing an "
                         "occupancy with period=night", val.get_error())
<|endoftext|>


<commit_before>import pandas as pd
import numpy as np
from csv import DictReader
from db import Parcel, addresshasher, JeffMailMash, JeffPropMash, MailStFields, PropStFields, OwnerFields

#
def JEFFCO(datafile):
    """
    2013: data/JeffersonCo/JeffcoData Nov 2013/CLEANED_ATSDTA_ATSP600.csv
    2014:
    """
    return pd.read_csv(datafile, na_values=['nan', ''], index_col=False, low_memory=False)

def GRANDCO(datafile):
    """
    data/GrandCo/All_Account_Details.csv
    """
    pd.read_csv(datafile, na_values=['nan', ''], index_col=False)

def grandco_parcels(parceltable, countytable):
    results = pd.read_csv('data/GrandCo/All_Properties.csv')
    count = 0
    for p in results.Parcel_ID.unique():
        prop = parceltable.create(Parcel_ID=p)
        countytable(Parcel_ID=prop, Co_Name='Grand')
        count += 1
    print('Imported {} Parcels'.format(count))

def grandco_auction(parceltable, lienauction, dataframe):
    """
    Loads auction result data for grand county
    :param parceltable:
    :param lienauction:
    :return:
    """
    dataframe = pd.read_csv('data/GrandCo/All_Results.csv')
    count = 0
    for p in dataframe.iterrows():
        data = p[1]
        try:
            prop = parceltable.get(parceltable.Parcel_ID == data.Parcel_ID)
        except Exception as e:
            if 'Instance matching query does not exist' in str(e):
                prop = parceltable.create(Parcel_ID=data.Parcel_ID)
            else:
                print('The problem is {}'.format(data.Parcel_ID))
                print(e)

        lienauction.create(Parcel_ID=prop,
                           Bidder_ID=data.Bidder_ID,
                           Face_Value=data.Face_Value,
                           Name=data.Name,
                           Tax_Year=data.Tax_Year,
                           Winning_Bid=data.Winning_Bid)
        count += 1
    print('Imported {} auction results'.format(count))

def grandco_account(parceltable, accounttable, dataframe):
    """
    Loads tresury account detail for each property
    :param parceltable:
    :param accounttable:
    :return:
    """
    count = 0
    for row in dataframe.iterrows():
        data = row[1]
        prop = parceltable.get(parceltable.Parcel_ID == data.Parcel_ID)
        accounttable.create(Parcel_ID=prop,
                            Tax_Year=data.Tax_Year,
                            Tax_Type=data.Tax_Type,
                            Effective_Date=data.Effective_Date,
                            Amount=data.Amount,
                            Balance=data.Balance)
        count += 1
    print('Imported {} account details'.format(count))

def jeffco_600_Parcels(dataframe):
    """
    Jeffco ASTP600 file:
        SCH: SCHEDULE NUMBER
    """
    from db import Parcel
    try:
        Parcel.create_table()
    except:
        pass
    # "data/JeffersonCo/JeffcoData Nov 2013/CLEANED_ATSDTA_ATSP600.csv"
    dataframe['SCH'] = pd.to_numeric(dataframe['SCH'], errors='coerce')
    count = 0
    for p in dataframe.SCH.unique():
        prop = Parcel.create(Parcel_ID=p, County='Jefferson')
        count += 1
    print('Imported {} Jefferson Parcels'.format(count))

def jeffco_600_Individuals(dataframe):
    """
    Jeffco ASTP600 file:
        OWNNAM
        OWNNAM2
        OWNNAM3
        OWNICO
        DBA
    """
    from db import Individuals
    try:
        Individuals.create_table()
    except:
        pass
    ownerfields = ['OWNNAM', 'OWNNAM2', 'OWNNAM3', 'OWNICO', 'DBA']
    all_owners = set()
    for o in ownerfields:
        all_owners |= set(dataframe[o][(dataframe[o].notnull())])
    careofnames = set(dataframe['OWNICO'][(dataframe['OWNICO'].notnull())])
    dba = set(dataframe['DBA'][(dataframe['DBA'].notnull())])
    count = 0
    careofnames = set(x.replace('%', '').strip() for x in careofnames)
    dba = set(x.replace('%', '').strip() for x in dba)

    print('ABEL RICHARD A' in all_owners)

    for i in all_owners:
        i.replace('%', '')
        i = i.strip()
        OWNICO = i in careofnames
        DBA = i in dba
        ind = Individuals.create(Name=i, OWNICO=OWNICO, DBA=DBA)
        count += 1
    print('Added {} owners'.format(count))

def Jeffco_600_owners(dataframe):
    """
    Connect owners with property
    :param parceltable:
    :param indvtable:
    :param owntable:
    :param datafile:
    :return:
    """
    from db import Parcel, Individuals, Owners
    try:
        Owners.create_table()
    except:
        pass
    ownerfields = ['OWNNAM', 'OWNNAM2', 'OWNNAM3', 'OWNICO', 'DBA']
    for row in dataframe.iterrows():
        row = row[1]
        print(int(row['SCH']))
        prop = Parcel.get(Parcel.Parcel_ID == int(row['SCH']))
        for i in (row[o] for o in ownerfields):
            if str(i) != 'nan':
                indv = Individuals.get(Individuals.Name == i)
                Owners.create(Owner=indv, Parcel_ID=prop)

def jeffco_600_Addresses(dataframe):
    """
    connect individuals to properties
    :param ownerstable:
    :param datafile:
    :return:
    Jeffco 600
        'MAILSTRNBR': 'MAILING STREET ADDRESS NUMBER'
        'MAILSTRDIR': 'MAILING STREET ADDRESS DIRECTION'
        'MAILSTRNAM': 'MAILING STREET ADDRESS NAME'
        'MAILSTRTYP': 'MAILING STREET ADDRESS TYPE'
        'MAILSTRSFX': 'MAILING STREET ADDRESS SUFFIX'
        'MAILSTRUNT': 'MAILING STREET ADDRESS UNIT'
        'MAILCTYNAM': 'MAILING CITY NAME'
        'MAILSTENAM': 'MAILING STATE NAME'
        'MAILZIP5': 'MAILING ADDRESS ZIP CODE 5'
        'MAILZIP4': 'MAILING ADDRESS ZIP CODE 4'

        'PRPSTRNUM': 'PROPERTY STREET ADDRESS NUMBER'
        'PRPSTRDIR': 'PROPERTY STREET ADDRESS DIRECTION'
        'PRPSTRNAM': 'PROPERTY STREET ADDRESS NAME'
        'PRPSTRTYP': 'PROPERTY STREET ADDRESS TYPE'
        'PRPSTRSFX': 'PROPERTY STREET ADDRESS SUFFIX'
        'PRPSTRUNT': 'PROPERTY STREET ADDRESS UNIT'
        'PRPCTYNAM': 'PROPERTY CITY NAME'
        'PRPSTENAM': 'PROPERTY STATE NAME'
        'PRPZIP5': 'PROPERTY ZIP CODE 5'
        'PRPZIP4': 'PROPERTY ZIP CODE 4'
    """
    from db import Addresses
    try:
        Addresses.create_table()
    except:
        pass
    #dataframe['MAILSTRNBR'] = pd.to_numeric(dataframe['MAILSTRNBR'], errors='coerce')
    #dataframe['PRPSTRNUM'] = pd.to_numeric(dataframe['PRPSTRNUM'], errors='coerce')
    for r in dataframe.iterrows():
        r = r[1]
        street1 = ' '.join(str(r[a]) for a in MailStFields if pd.notnull(r[a]))
        idhashmail = addresshasher(r, JeffMailMash)
        idhashprop = addresshasher(r, JeffPropMash)
        try:
            addr = Addresses.create(Street1=street1.lstrip('0'),
                                       Street2= r['MAILSTRUNT'],
                                       City=r['MAILCTYNAM'],
                                       State=r['MAILSTENAM'],
                                       Zipcode=r['MAILZIP5'],
                                       Zip4=r['MAILZIP4'],
                                       IdHash=idhashmail,
                                       Mailing=True)
            if idhashmail == idhashprop:
                # TODO: need to update exsiting addresses.
                addr.Property = True
                addr.save()
            else:
                street1 = ' '.join(str(r[a]) for a in PropStFields if pd.notnull(r[a]))
                Addresses.create(street1=street1.lstrip('0'),
                                    street2=r['PRPSTRUNT'],
                                    City=r['PRPCTYNAM'],
                                    State=r['PRPSTENAM'],
                                    Zipcode=r['PRPZIP5'],
                                    Zip4=r['PRPZIP4'],
                                    IdHash=idhashprop,
                                    Property=True)
        except Exception as e:
            if 'UNIQUE constraint failed: addresses.IdHash' not in str(e):
                print(e)
                raise
<commit_msg>grand parcel import<commit_after>import pandas as pd
import numpy as np
from csv import DictReader
from db import Parcel, addresshasher, JeffMailMash, JeffPropMash, MailStFields, PropStFields, OwnerFields

#
def JEFFCO(datafile):
    """
    2013: data/JeffersonCo/JeffcoData Nov 2013/CLEANED_ATSDTA_ATSP600.csv
    2014:
    """
    return pd.read_csv(datafile, na_values=['nan', ''], index_col=False, low_memory=False)

def GRANDCO(datafile):
    """
    data/GrandCo/All_Account_Details.csv
    """
    pd.read_csv(datafile, na_values=['nan', ''], index_col=False)

def grandco_parcels():
    from db import Parcel
    try:
        Parcel.create_table()
    except:
        pass
    results = pd.read_csv('data/GrandCo/All_Properties.csv')
    count = 0
    for p in results.Parcel_ID.unique():
        prop = parceltable.create(Parcel_ID=p)
        countytable(Parcel_ID=prop, Co_Name='Grand')
        count += 1
    print('Imported {} Parcels'.format(count))

def grandco_auction(parceltable, lienauction, dataframe):
    """
    Loads auction result data for grand county
    :param parceltable:
    :param lienauction:
    :return:
    """
    dataframe = pd.read_csv('data/GrandCo/All_Results.csv')
    count = 0
    for p in dataframe.iterrows():
        data = p[1]
        try:
            prop = parceltable.get(parceltable.Parcel_ID == data.Parcel_ID)
        except Exception as e:
            if 'Instance matching query does not exist' in str(e):
                prop = parceltable.create(Parcel_ID=data.Parcel_ID)
            else:
                print('The problem is {}'.format(data.Parcel_ID))
                print(e)

        lienauction.create(Parcel_ID=prop,
                           Bidder_ID=data.Bidder_ID,
                           Face_Value=data.Face_Value,
                           Name=data.Name,
                           Tax_Year=data.Tax_Year,
                           Winning_Bid=data.Winning_Bid)
        count += 1
    print('Imported {} auction results'.format(count))

def grandco_account(parceltable, accounttable, dataframe):
    """
    Loads tresury account detail for each property
    :param parceltable:
    :param accounttable:
    :return:
    """
    count = 0
    for row in dataframe.iterrows():
        data = row[1]
        prop = parceltable.get(parceltable.Parcel_ID == data.Parcel_ID)
        accounttable.create(Parcel_ID=prop,
                            Tax_Year=data.Tax_Year,
                            Tax_Type=data.Tax_Type,
                            Effective_Date=data.Effective_Date,
                            Amount=data.Amount,
                            Balance=data.Balance)
        count += 1
    print('Imported {} account details'.format(count))

def jeffco_600_Parcels(dataframe):
    """
    Jeffco ASTP600 file:
        SCH: SCHEDULE NUMBER
    """
    from db import Parcel
    try:
        Parcel.create_table()
    except:
        pass
    # "data/JeffersonCo/JeffcoData Nov 2013/CLEANED_ATSDTA_ATSP600.csv"
    dataframe['SCH'] = pd.to_numeric(dataframe['SCH'], errors='coerce')
    count = 0
    for p in dataframe.SCH.unique():
        prop = Parcel.create(Parcel_ID=p, County='Jefferson')
        count += 1
    print('Imported {} Jefferson Parcels'.format(count))

def jeffco_600_Individuals(dataframe):
    """
    Jeffco ASTP600 file:
        OWNNAM
        OWNNAM2
        OWNNAM3
        OWNICO
        DBA
    """
    from db import Individuals
    try:
        Individuals.create_table()
    except:
        pass
    ownerfields = ['OWNNAM', 'OWNNAM2', 'OWNNAM3', 'OWNICO', 'DBA']
    all_owners = set()
    for o in ownerfields:
        all_owners |= set(dataframe[o][(dataframe[o].notnull())])
    careofnames = set(dataframe['OWNICO'][(dataframe['OWNICO'].notnull())])
    dba = set(dataframe['DBA'][(dataframe['DBA'].notnull())])
    count = 0
    careofnames = set(x.replace('%', '').strip() for x in careofnames)
    dba = set(x.replace('%', '').strip() for x in dba)

    print('ABEL RICHARD A' in all_owners)

    for i in all_owners:
        i.replace('%', '')
        i = i.strip()
        OWNICO = i in careofnames
        DBA = i in dba
        ind = Individuals.create(Name=i, OWNICO=OWNICO, DBA=DBA)
        count += 1
    print('Added {} owners'.format(count))

def Jeffco_600_owners(dataframe):
    """
    Connect owners with property
    :param parceltable:
    :param indvtable:
    :param owntable:
    :param datafile:
    :return:
    """
    from db import Parcel, Individuals, Owners
    try:
        Owners.create_table()
    except:
        pass
    ownerfields = ['OWNNAM', 'OWNNAM2', 'OWNNAM3', 'OWNICO', 'DBA']
    for row in dataframe.iterrows():
        row = row[1]
        print(int(row['SCH']))
        prop = Parcel.get(Parcel.Parcel_ID == int(row['SCH']))
        for i in (row[o] for o in ownerfields):
            if str(i) != 'nan':
                indv = Individuals.get(Individuals.Name == i)
                Owners.create(Owner=indv, Parcel_ID=prop)

def jeffco_600_Addresses(dataframe):
    """
    connect individuals to properties
    :param ownerstable:
    :param datafile:
    :return:
    Jeffco 600
        'MAILSTRNBR': 'MAILING STREET ADDRESS NUMBER'
        'MAILSTRDIR': 'MAILING STREET ADDRESS DIRECTION'
        'MAILSTRNAM': 'MAILING STREET ADDRESS NAME'
        'MAILSTRTYP': 'MAILING STREET ADDRESS TYPE'
        'MAILSTRSFX': 'MAILING STREET ADDRESS SUFFIX'
        'MAILSTRUNT': 'MAILING STREET ADDRESS UNIT'
        'MAILCTYNAM': 'MAILING CITY NAME'
        'MAILSTENAM': 'MAILING STATE NAME'
        'MAILZIP5': 'MAILING ADDRESS ZIP CODE 5'
        'MAILZIP4': 'MAILING ADDRESS ZIP CODE 4'

        'PRPSTRNUM': 'PROPERTY STREET ADDRESS NUMBER'
        'PRPSTRDIR': 'PROPERTY STREET ADDRESS DIRECTION'
        'PRPSTRNAM': 'PROPERTY STREET ADDRESS NAME'
        'PRPSTRTYP': 'PROPERTY STREET ADDRESS TYPE'
        'PRPSTRSFX': 'PROPERTY STREET ADDRESS SUFFIX'
        'PRPSTRUNT': 'PROPERTY STREET ADDRESS UNIT'
        'PRPCTYNAM': 'PROPERTY CITY NAME'
        'PRPSTENAM': 'PROPERTY STATE NAME'
        'PRPZIP5': 'PROPERTY ZIP CODE 5'
        'PRPZIP4': 'PROPERTY ZIP CODE 4'
    """
    from db import Addresses
    try:
        Addresses.create_table()
    except:
        pass
    #dataframe['MAILSTRNBR'] = pd.to_numeric(dataframe['MAILSTRNBR'], errors='coerce')
    #dataframe['PRPSTRNUM'] = pd.to_numeric(dataframe['PRPSTRNUM'], errors='coerce')
    for r in dataframe.iterrows():
        r = r[1]
        street1 = ' '.join(str(r[a]) for a in MailStFields if pd.notnull(r[a]))
        idhashmail = addresshasher(r, JeffMailMash)
        idhashprop = addresshasher(r, JeffPropMash)
        try:
            addr = Addresses.create(Street1=street1.lstrip('0'),
                                       Street2= r['MAILSTRUNT'],
                                       City=r['MAILCTYNAM'],
                                       State=r['MAILSTENAM'],
                                       Zipcode=r['MAILZIP5'],
                                       Zip4=r['MAILZIP4'],
                                       IdHash=idhashmail,
                                       Mailing=True)
            if idhashmail == idhashprop:
                # TODO: need to update exsiting addresses.
                addr.Property = True
                addr.save()
            else:
                street1 = ' '.join(str(r[a]) for a in PropStFields if pd.notnull(r[a]))
                Addresses.create(street1=street1.lstrip('0'),
                                    street2=r['PRPSTRUNT'],
                                    City=r['PRPCTYNAM'],
                                    State=r['PRPSTENAM'],
                                    Zipcode=r['PRPZIP5'],
                                    Zip4=r['PRPZIP4'],
                                    IdHash=idhashprop,
                                    Property=True)
        except Exception as e:
            if 'UNIQUE constraint failed: addresses.IdHash' not in str(e):
                print(e)
                raise
<|endoftext|>


<commit_before>import requests
import re
import os
base = os.path.join(os.path.dirname(os.path.realpath(__file__)), '../../')

def parse_txt_files(txt_files=None):
    import re
    import os
    if not txt_files:
        txt_files = sorted([f for f in os.listdir('/home/ubuntu/redactvideodotorg/opa_closed_case_summaries/txts')])
    opa_files = []
    for filename in txt_files:
    #for filename in files:
        f = open('txts/'+filename, 'r')

        opa_file = str(f.read()).replace('\n', ' ').replace('  ', ' ')
        if not opa_file:
            continue
        print opa_file
        opa_file_dict = {}
        regex =  re.search('Complaint Number(?P<num>.*?)Issued', opa_file)

        opa_file_dict['Complaint number'] = regex.group('num').strip(' :').replace(' ', '') if regex else filename[:filename.find('ccs')]
        regex = re.search('Issued Date(?P<date>.*?)Named', opa_file)
        opa_file_dict['Issued date'] = regex.group('date').strip(' :') if regex else None
        regex = re.search('lssued Date(?P<date>.*?)Named', opa_file)
        if regex:
            opa_file_dict['Issued date'] = regex.group('date').strip(' :')

        regex = re.search('OPA Finding(?P<findings>.*?)Final', opa_file)
        opa_file_dict['OPA finding'] = regex.group('findings').strip(' :') if regex else None

        regex = re.search('FINDINGS(?P<findings>.*?)NOTE', opa_file,  re.MULTILINE|re.DOTALL)
        if regex:
            opa_file_dict['Findings'] = regex.group('findings').strip(' :\n').replace('\n', '')
        regex = re.search('Final Discipline\s*(?P<discipline>.*?)INCIDENT', opa_file)
        opa_file_dict['Final discipline'] = regex.group('discipline').strip(' :') if regex else None
        regex = re.search('INCIDENT SYNOPSIS(?P<synopsis>.*?)(Complaint|COMPLAINT)', opa_file,  re.MULTILINE|re.DOTALL)
        opa_file_dict['Incident synopsis'] = regex.group('synopsis').strip(' :\n').replace('\n', '') if regex else None
        regex = re.search('COMPLAINT(?P<complaint>.*?)(Complaint|INVESTIGATION)', opa_file,  re.MULTILINE|re.DOTALL)
        opa_file_dict['Complaint'] = regex.group('complaint').strip(' :\n').replace('\n', '') if regex else None
        regex = re.search('INVESTIGATION(?P<investigation>.*?)(Complaint|ANALYSIS)', opa_file,  re.MULTILINE|re.DOTALL)
        opa_file_dict['Investigation'] = regex.group('investigation').strip(' :\n').replace('\n', '') if regex else None
        regex = re.search('ANALYSIS AND CONCLUSION(?P<analysis>.*?)(Complaint|FINDINGS)', opa_file,  re.MULTILINE|re.DOTALL)
        opa_file_dict['Analysis and conclusion'] = regex.group('analysis').strip(' :\n').replace('\n', '') if regex else None
        opa_file_dict['txt'] = filename
        opa_file_dict['pdf'] = filename[:-4]+'.pdf'
        try:
            # Get the entire findings section
            f = open('txts/'+filename, 'r')

            opa_file2 = f.read()
            import re
            m = re.search('Issued(.*?)Complaint', opa_file2,  re.MULTILINE|re.DOTALL)
            if m:
                section = m.group()
                lines = section.split('\n')[3:-3]
                structured_version = []
                current = None
                opa_file_dict['is_sustained'] = False
                for i, line in enumerate(lines):
                    if line.startswith('Named Employee'):
                        if current:
                            structured_version.append(current)
                        current = {'heading': line.strip(), 'allegations': []}
                    if line.startswith('Allegation'):
                        allegation_structure = {'heading': line.strip()}
                        allegation_structure['allegation'] = lines[i+2].strip()
                        j = 0
                        while True:
                            j += 1
                            the_line = lines[i + 2 + j]
                            if not the_line:
                                break
                            allegation_structure['allegation'] += ' '+the_line.strip()

                        current['allegations'].append(allegation_structure)
                    if line.startswith('OPA Finding'):
                        current['allegations'][-1]['opa_finding'] = lines[i+2].strip()
                        if current['allegations'][-1]['opa_finding'] == 'Sustained':
                            opa_file_dict['is_sustained'] = True
                    if line.startswith('Final Discipline'):
                        current['final_discipline'] = lines[i+2].strip()
                structured_version.append(current)
            opa_file_dict['Summarized results'] = structured_version
        except Exception, e:
            opa_file_dict['parsing_error'] = e
        opa_files.append(opa_file_dict)
        f.close()
    import sys

    reload(sys)
    sys.setdefaultencoding('utf8')
    print opa_files
    import rethinkdb as r
    conn = r.connect( "localhost", 28015).repl()
    db = r.db('opa')
    try:
        db.table_create('closed_case_summaries', primary_key='Complaint number').run(conn)
    except:
        pass
    try:
        db.table_create('closed_case_summaries_subscribers', primary_key='email').run(conn)
    except:
        pass
    db.table('closed_case_summaries').insert(opa_files, conflict='update').run(conn)
    the_html = '
'
    columns = ['Issued date', 'Complaint number', 'Complaint', 'Incident synopsis', 'Investigation', 'Analysis and conclusion', 'OPA finding', 'Final discipline']
    from dateutil.parser import parse
    import datetime
    mindate = datetime.date(datetime.MINYEAR, 1, 1)
    opa_files = sorted(opa_files, key=lambda x: parse(x['Issued date']).date() if '/' in x['Issued date'] else mindate, reverse=True)

def download():
    os.system('mkdir %s.crawler_data' % (base))
    os.system('mkdir %s.crawler_data/seattle_police_internal_affairs_closed_cases/' % (base))
    os.system('mkdir %s.crawler_data/seattle_police_internal_affairs_closed_cases/pdfs/' % (base))
    os.system('mkdir %s.crawler_data/seattle_police_internal_affairs_closed_cases/txts/' % (base))
    html = requests.get('http://www.seattle.gov/opa/closed-case-summaries').text
    os.system('mkdir pdfs')
    files = re.findall('ClosedCaseSummaries/(?P<filename>.*?)\.pdf', html)
    print files
    for f in files:
        if not f+'.pdf' in os.listdir(base+'seattle_police_internal_affairs_closed_cases/pdfs'):
            os.system('wget -O /home/ubuntu/redactvideodotorg/opa_closed_case_summaries/pdfs/%s.pdf http://www.seattle.gov/Documents/Departments/OPA/ClosedCaseSummaries/%s.pdf' % (f, f))
    import os
    #os.system('rm /home/ubuntu/redactvideodotorg/opa_closed_case_summaries/txts/*')
    os.system('mkdir /home/ubuntu/redactvideodotorg/opa_closed_case_summaries/txts')
    files = sorted([f for f in os.listdir('/home/ubuntu/redactvideodotorg/opa_closed_case_summaries/pdfs')])
    new_files = []
    for filename in files:
        print filename
        #os.system('pdf2txt.py pdfs/%s > txts/%s' % (filename, filename[:-4]+'.txt'))
        if not filename[:-4]+'.txt' in os.listdir('/home/ubuntu/redactvideodotorg/opa_closed_case_summaries/txts'):
            print 'converting'
            os.system('pdf2txt.py -A /home/ubuntu/redactvideodotorg/opa_closed_case_summaries/pdfs/%s > /home/ubuntu/redactvideodotorg/opa_closed_case_summaries/txts/%s' % (filename, filename[:-4]+'.txt'))
            new_files.append(filename[:-4]+'.txt')
    parse_txt_files(new_files)

if __name__ == "__main__":
    download()
<commit_msg>auto commit<commit_after>import requests
import re
import os
base = os.path.join(os.path.dirname(os.path.realpath(__file__)), '../../')

def parse_txt_files(txt_files=None):
    import re
    import os
    if not txt_files:
        txt_files = sorted([f for f in os.listdir(base+'/home/ubuntu/redactvideodotorg/opa_closed_case_summaries/txts')])
    opa_files = []
    for filename in txt_files:
    #for filename in files:
        f = open('txts/'+filename, 'r')

        opa_file = str(f.read()).replace('\n', ' ').replace('  ', ' ')
        if not opa_file:
            continue
        print opa_file
        opa_file_dict = {}
        regex =  re.search('Complaint Number(?P<num>.*?)Issued', opa_file)

        opa_file_dict['Complaint number'] = regex.group('num').strip(' :').replace(' ', '') if regex else filename[:filename.find('ccs')]
        regex = re.search('Issued Date(?P<date>.*?)Named', opa_file)
        opa_file_dict['Issued date'] = regex.group('date').strip(' :') if regex else None
        regex = re.search('lssued Date(?P<date>.*?)Named', opa_file)
        if regex:
            opa_file_dict['Issued date'] = regex.group('date').strip(' :')

        regex = re.search('OPA Finding(?P<findings>.*?)Final', opa_file)
        opa_file_dict['OPA finding'] = regex.group('findings').strip(' :') if regex else None

        regex = re.search('FINDINGS(?P<findings>.*?)NOTE', opa_file,  re.MULTILINE|re.DOTALL)
        if regex:
            opa_file_dict['Findings'] = regex.group('findings').strip(' :\n').replace('\n', '')
        regex = re.search('Final Discipline\s*(?P<discipline>.*?)INCIDENT', opa_file)
        opa_file_dict['Final discipline'] = regex.group('discipline').strip(' :') if regex else None
        regex = re.search('INCIDENT SYNOPSIS(?P<synopsis>.*?)(Complaint|COMPLAINT)', opa_file,  re.MULTILINE|re.DOTALL)
        opa_file_dict['Incident synopsis'] = regex.group('synopsis').strip(' :\n').replace('\n', '') if regex else None
        regex = re.search('COMPLAINT(?P<complaint>.*?)(Complaint|INVESTIGATION)', opa_file,  re.MULTILINE|re.DOTALL)
        opa_file_dict['Complaint'] = regex.group('complaint').strip(' :\n').replace('\n', '') if regex else None
        regex = re.search('INVESTIGATION(?P<investigation>.*?)(Complaint|ANALYSIS)', opa_file,  re.MULTILINE|re.DOTALL)
        opa_file_dict['Investigation'] = regex.group('investigation').strip(' :\n').replace('\n', '') if regex else None
        regex = re.search('ANALYSIS AND CONCLUSION(?P<analysis>.*?)(Complaint|FINDINGS)', opa_file,  re.MULTILINE|re.DOTALL)
        opa_file_dict['Analysis and conclusion'] = regex.group('analysis').strip(' :\n').replace('\n', '') if regex else None
        opa_file_dict['txt'] = filename
        opa_file_dict['pdf'] = filename[:-4]+'.pdf'
        try:
            # Get the entire findings section
            f = open('txts/'+filename, 'r')

            opa_file2 = f.read()
            import re
            m = re.search('Issued(.*?)Complaint', opa_file2,  re.MULTILINE|re.DOTALL)
            if m:
                section = m.group()
                lines = section.split('\n')[3:-3]
                structured_version = []
                current = None
                opa_file_dict['is_sustained'] = False
                for i, line in enumerate(lines):
                    if line.startswith('Named Employee'):
                        if current:
                            structured_version.append(current)
                        current = {'heading': line.strip(), 'allegations': []}
                    if line.startswith('Allegation'):
                        allegation_structure = {'heading': line.strip()}
                        allegation_structure['allegation'] = lines[i+2].strip()
                        j = 0
                        while True:
                            j += 1
                            the_line = lines[i + 2 + j]
                            if not the_line:
                                break
                            allegation_structure['allegation'] += ' '+the_line.strip()

                        current['allegations'].append(allegation_structure)
                    if line.startswith('OPA Finding'):
                        current['allegations'][-1]['opa_finding'] = lines[i+2].strip()
                        if current['allegations'][-1]['opa_finding'] == 'Sustained':
                            opa_file_dict['is_sustained'] = True
                    if line.startswith('Final Discipline'):
                        current['final_discipline'] = lines[i+2].strip()
                structured_version.append(current)
            opa_file_dict['Summarized results'] = structured_version
        except Exception, e:
            opa_file_dict['parsing_error'] = e
        opa_files.append(opa_file_dict)
        f.close()
    import sys

    reload(sys)
    sys.setdefaultencoding('utf8')
    print opa_files
    import rethinkdb as r
    conn = r.connect( "localhost", 28015).repl()
    db = r.db('opa')
    try:
        db.table_create('closed_case_summaries', primary_key='Complaint number').run(conn)
    except:
        pass
    try:
        db.table_create('closed_case_summaries_subscribers', primary_key='email').run(conn)
    except:
        pass
    db.table('closed_case_summaries').insert(opa_files, conflict='update').run(conn)
    the_html = '
'
    columns = ['Issued date', 'Complaint number', 'Complaint', 'Incident synopsis', 'Investigation', 'Analysis and conclusion', 'OPA finding', 'Final discipline']
    from dateutil.parser import parse
    import datetime
    mindate = datetime.date(datetime.MINYEAR, 1, 1)
    opa_files = sorted(opa_files, key=lambda x: parse(x['Issued date']).date() if '/' in x['Issued date'] else mindate, reverse=True)

def download():
    os.system('mkdir %s.crawler_data' % (base))
    os.system('mkdir %s.crawler_data/seattle_police_internal_affairs_closed_cases/' % (base))
    os.system('mkdir %s.crawler_data/seattle_police_internal_affairs_closed_cases/pdfs/' % (base))
    os.system('mkdir %s.crawler_data/seattle_police_internal_affairs_closed_cases/txts/' % (base))
    html = requests.get('http://www.seattle.gov/opa/closed-case-summaries').text
    os.system('mkdir pdfs')
    files = re.findall('ClosedCaseSummaries/(?P<filename>.*?)\.pdf', html)
    print files
    for f in files:
        if not f+'.pdf' in os.listdir(base+'seattle_police_internal_affairs_closed_cases/pdfs'):
            os.system('wget -O /home/ubuntu/redactvideodotorg/opa_closed_case_summaries/pdfs/%s.pdf http://www.seattle.gov/Documents/Departments/OPA/ClosedCaseSummaries/%s.pdf' % (f, f))
    import os
    #os.system('rm /home/ubuntu/redactvideodotorg/opa_closed_case_summaries/txts/*')
    os.system('mkdir /home/ubuntu/redactvideodotorg/opa_closed_case_summaries/txts')
    files = sorted([f for f in os.listdir('/home/ubuntu/redactvideodotorg/opa_closed_case_summaries/pdfs')])
    new_files = []
    for filename in files:
        print filename
        #os.system('pdf2txt.py pdfs/%s > txts/%s' % (filename, filename[:-4]+'.txt'))
        if not filename[:-4]+'.txt' in os.listdir('/home/ubuntu/redactvideodotorg/opa_closed_case_summaries/txts'):
            print 'converting'
            os.system('pdf2txt.py -A /home/ubuntu/redactvideodotorg/opa_closed_case_summaries/pdfs/%s > /home/ubuntu/redactvideodotorg/opa_closed_case_summaries/txts/%s' % (filename, filename[:-4]+'.txt'))
            new_files.append(filename[:-4]+'.txt')
    parse_txt_files(new_files)

if __name__ == "__main__":
    download()
<|endoftext|>