<commit_before>import datetime
import traceback

from jinja2 import Template
from praw.models import Comment

import bot_logger
import config
import crypto
import lang
import re
import user_function
import utils

def register_user(rpc, msg):
    if not user_function.user_exist(msg.author.name):
        address = rpc.getnewaddress("reddit-%s" % msg.author.name)
        if address:
            msg.reply(
                Template(lang.message_register_success + lang.message_footer).render(username=msg.author.name, address=address))
            user_function.add_user(msg.author.name, address)

            user_function.add_to_history(msg.author.name, "", "", "", "register")
        else:
            bot_logger.logger.warning('Error during register !')
    else:
        bot_logger.logger.info('%s are already registered ' % msg.author.name)
        balance = crypto.get_user_confirmed_balance(rpc, msg.author.name)
        address = user_function.get_user_address(msg.author.name)
        msg.reply(Template(lang.message_already_registered + lang.message_account_details+ lang.message_footer).render(username=msg.author.name,
                                                                                address=address, balance=str(
                balance)) )

def balance_user(rpc, msg):
    if user_function.user_exist(msg.author.name):

        balance = crypto.get_user_confirmed_balance(rpc, msg.author.name)
        pending_balance = crypto.get_user_unconfirmed_balance(rpc, msg.author.name)
        spendable_balance = crypto.get_user_spendable_balance(rpc, msg.author.name) + balance
        bot_logger.logger.info('user %s balance = %s' % (msg.author.name, balance))

        balance_value_usd = utils.get_coin_value(balance)
        pending_value_usd = utils.get_coin_value(pending_balance)
        spendable_value_usd = utils.get_coin_value(spendable_balance)
        msg.reply(Template(lang.message_balance + lang.message_footer).render(username=msg.author.name, balance=str(balance),
                                              balance_value_usd=str(balance_value_usd),
                                              pendingbalance=str(pending_balance),
                                              pending_value_usd=str(pending_value_usd),
                                              spendablebalance=str(spendable_balance),
                                              spendable_value_usd=str(spendable_value_usd)))

        user_function.add_to_history(msg.author.name, "", "", balance, "balance")
    else:
        bot_logger.logger.info('user %s not registered ' % msg.author.name)
        msg.reply(Template(lang.message_need_register + lang.message_footer).render(username=msg.author.name))

def info_user(rpc, msg):
    if user_function.user_exist(msg.author.name):
        address = user_function.get_user_address(msg.author.name)
        balance = crypto.get_user_confirmed_balance(rpc, msg.author.name)
        pending_balance = crypto.get_user_unconfirmed_balance(rpc, msg.author.name)
        spendable_balance = crypto.get_user_spendable_balance(rpc, msg.author.name) + balance
        balance_value_usd = utils.get_coin_value(balance)
        pending_value_usd = utils.get_coin_value(pending_balance)
        spendable_value_usd = utils.get_coin_value(spendable_balance)
        msg.reply(lang.message_account_details.render(username=msg.author.name, balance=str(balance),
                                                      balance_value_usd=str(balance_value_usd),
                                                      pendingbalance=str(pending_balance),
                                                      pending_value_usd=str(pending_value_usd),
                                                      spendablebalance=str(spendable_balance),
                                                      spendable_value_usd=str(spendable_value_usd),
                                                      address=address) + lang.message_footer)

    else:
        msg.reply(Template(lang.message_need_register+ lang.message_footer).render(username=msg.author.name))

def help_user(rpc, msg):
    if user_function.user_exist(msg.author.name):
        balance = crypto.get_user_confirmed_balance(rpc, msg.author.name)
        address = user_function.get_user_address(msg.author.name)
        msg.reply(Template(lang.message_help + lang.message_account_details+ lang.message_footer).render(username=msg.author.name, address=address,
                                                                              balance=str(
                                                                                balance)))
    else:
        msg.reply(Template(lang.message_need_register+ lang.message_footer).render(username=msg.author.name))

def withdraw_user(rpc, msg):
    split_message = msg.body.strip().split()

    if user_function.user_exist(msg.author.name):
        sender_address = user_function.get_user_address(msg.author.name)
        amount = split_message[1]
        user_balance = crypto.get_user_confirmed_balance(rpc, msg.author.name)
        user_spendable_balance = crypto.get_user_spendable_balance(rpc, msg.author.name)
        if utils.check_amount_valid(amount):
            if int(amount) >= user_balance + user_spendable_balance:
                bot_logger.logger.info('user %s not have enough to withdraw this amount (%s), balance = %s' % (
                    msg.author.name, amount, user_balance))
                msg.reply(lang.message_balance_low_withdraw.render(
                    username=msg.author.name, user_balance=str(user_balance), amount=str(amount)) + lang.message_footer)
            else:
                receiver_address = split_message[4]
                try:
                    if crypto.send_to(rpc, sender_address, receiver_address, amount):
                        user_function.add_to_history(msg.author.name, sender_address, receiver_address, amount,
                                                     "withdraw")
                        value_usd = utils.get_coin_value(amount)
                        msg.reply(lang.message_withdraw.render(
                            username=msg.author.name, receiver_address=receiver_address, amount=str(amount),
                            value_usd=str(value_usd)) + lang.message_footer)

                except:
                    traceback.print_exc()
        else:
            bot_logger.logger.info(lang.message_invalid_amount)
            msg.reply(lang.message_invalid_amount + lang.message_footer)
    else:
        msg.reply(lang.message_need_register.render(username=msg.author.name) + lang.message_footer)

def tip_user(rpc, reddit, msg):
    bot_logger.logger.info('An user mention detected ')
    split_message = msg.body.lower().strip().split()
    tip_index = split_message.index(str('+/u/' + config.bot_name))

    if split_message[tip_index] == str('+/u/' + config.bot_name) and split_message[tip_index + 2] == 'doge':

        amount = split_message[tip_index + 1]
        value_usd = utils.get_coin_value(amount)

        if utils.check_amount_valid(amount):
            parent_comment = msg.parent()
            if user_function.user_exist(msg.author.name):

                # check we have enough
                user_balance = crypto.get_user_confirmed_balance(rpc, msg.author.name)
                user_pending_balance = crypto.get_user_unconfirmed_balance(rpc, msg.author.name)
                user_spendable_balance = crypto.get_user_spendable_balance(rpc, msg.author.name) + user_balance
                if int(amount) >= user_spendable_balance:
                    # not enough for tip
                    if int(amount) < (user_spendable_balance + user_pending_balance):
                        msg.reply(Template(lang.message_balance_pending_tip).render(username=msg.author.name))
                    else:
                        bot_logger.logger.info('user %s not have enough to tip this amount (%s), balance = %s' % (
                            msg.author.name, str(amount), str(user_balance)))
                        msg.reply(Template(lang.message_balance_low_tip).render(username=msg.author.name))
                else:

                    # check user have address before tip
                    if user_function.user_exist(parent_comment.author.name):
                        if crypto.tip_user(rpc, msg.author.name, parent_comment.author.name, amount):
                            user_function.add_to_history(msg.author.name, msg.author.name, parent_comment.author.name,
                                                         amount,
                                                         "tip send")
                            user_function.add_to_history(parent_comment.author.name, msg.author.name,
                                                         parent_comment.author.name,
                                                         amount,
                                                         "tip receive")

                            bot_logger.logger.info(
                                '%s tip %s to %s' % (msg.author.name, str(amount), parent_comment.author.name))
                            # if user have 'verify' in this command he will have confirmation
                            if split_message.count('verify') or int(amount) >= 1000:
                                msg.reply(Template(lang.message_tip).render(
                                    sender=msg.author.name, receiver=parent_comment.author.name, amount=str(amount),
                                    value_usd=str(value_usd)))
                    else:
                        user_function.save_unregistered_tip(msg.author.name, parent_comment.author.name, amount,
                                                            msg.fullname)
                        user_function.add_to_history(msg.author.name, msg.author.name, parent_comment.author.name,
                                                     amount,
                                                     "tip send", False)
                        user_function.add_to_history(parent_comment.author.name, msg.author.name,
                                                     parent_comment.author.name,
                                                     amount,
                                                     "tip receive", False)
                        bot_logger.logger.info('user %s not registered' % parent_comment.author.name)
                        msg.reply(Template(lang.message_recipient_register).render(username=parent_comment.author.name))

                        reddit.redditor(parent_comment.author.name).message(
                            Template(
                                lang.message_recipient_need_register_title).render(amount=str(amount)),
                            Template(
                                lang.message_recipient_need_register_message).render(
                                username=parent_comment.author.name, sender=msg.author.name, amount=str(amount),
                                value_usd=str(value_usd)))

            else:
                msg.reply(Template(lang.message_need_register).render(username=msg.author.name))
        else:
            bot_logger.logger.info(lang.message_invalid_amount)
            msg.reply(lang.message_invalid_amount)

def history_user(msg):
    if user_function.user_exist(msg.author.name):
        data = user_function.get_user_history(msg.author.name)
        history_table = "\n\nDate|Sender|Receiver|Amount|Action|Finish|\n"
        history_table += "---|---|---|---|:-:|:-:\n"
        for tip in data:
            str_finish = "Pending"

            if tip['finish']:
                str_finish = "Successful"

            history_table += "%s|%s|%s|%s|%s|%s|\n" % (
                datetime.datetime.strptime(tip['time'], '%Y-%m-%dT%H:%M:%S.%f'), tip['sender'], tip['receiver'],
                str(tip['amount']), tip['action'], str_finish)

        msg.reply(Template(lang.message_history + history_table + lang.message_footer).render(username=msg.author.name))
    else:
        bot_logger.logger.info('user %s not registered ' % msg.author.name)
        msg.reply(Template(lang.message_need_register + lang.message_footer).render(username=msg.author.name))

# Resend tips to previously unregistered users that are now registered
def replay_remove_pending_tip(rpc, reddit):
    # check if it's not too old & replay tipping
    limit_date = datetime.datetime.now() - datetime.timedelta(days=3)

    # check if user have pending tips
    list_tips = user_function.get_unregistered_tip()

    if list_tips:
        for tip in list_tips:
            bot_logger.logger.info("replay tipping check for %s" % str(tip['id']))
            if (datetime.datetime.strptime(tip['time'], '%Y-%m-%dT%H:%M:%S.%f') > limit_date):
                if (user_function.user_exist(tip['receiver'])):
                    bot_logger.logger.info(
                        "replay tipping %s - %s send %s to %s  " % (
                            str(tip['id']), tip['sender'], tip['amount'], tip['receiver']))
                    crypto.tip_user(rpc, tip['sender'], tip['receiver'], tip['amount'])
                    user_function.remove_pending_tip(tip['id'])

                    value_usd = utils.get_coin_value(tip['amount'])

                    if 'message_fullname' in tip.keys():
                        msg_id = re.sub(r't\d+_(?P<id>\w+)', r'\g<id>', tip['message_fullname'])
                        msg = Comment(reddit, msg_id)
                        msg.reply(Template(lang.message_tip).render(
                            sender=tip['sender'], receiver=tip['receiver'], amount=str(tip['amount']),
                            value_usd=str(value_usd)))

                else:
                    bot_logger.logger.info(
                        "replay check for %s - user %s not registered " % (str(tip['id']), tip['receiver']))
            else:
                bot_logger.logger.info(
                    "delete old tipping - %s send %s for %s  " % (tip['sender'], tip['amount'], tip['receiver']))
                user_function.remove_pending_tip(tip['id'])
    else:
        bot_logger.logger.info("no pending tipping")
<commit_msg>Update bot_command.py<commit_after>import datetime
import traceback

from jinja2 import Template
from praw.models import Comment

import bot_logger
import config
import crypto
import lang
import re
import user_function
import utils

def register_user(rpc, msg):
    if not user_function.user_exist(msg.author.name):
        address = rpc.getnewaddress("reddit-%s" % msg.author.name)
        if address:
            msg.reply(
                Template(lang.message_register_success + lang.message_footer).render(username=msg.author.name, address=address))
            user_function.add_user(msg.author.name, address)

            user_function.add_to_history(msg.author.name, "", "", "", "register")
        else:
            bot_logger.logger.warning('Error during register !')
    else:
        bot_logger.logger.info('%s are already registered ' % msg.author.name)
        balance = crypto.get_user_confirmed_balance(rpc, msg.author.name)
        address = user_function.get_user_address(msg.author.name)
        msg.reply(Template(lang.message_already_registered + lang.message_account_details+ lang.message_footer).render(username=msg.author.name,
                                                                                address=address, balance=str(
                balance)) )

def balance_user(rpc, msg):
    if user_function.user_exist(msg.author.name):

        balance = crypto.get_user_confirmed_balance(rpc, msg.author.name)
        pending_balance = crypto.get_user_unconfirmed_balance(rpc, msg.author.name)
        spendable_balance = crypto.get_user_spendable_balance(rpc, msg.author.name) + balance
        bot_logger.logger.info('user %s balance = %s' % (msg.author.name, balance))

        balance_value_usd = utils.get_coin_value(balance)
        pending_value_usd = utils.get_coin_value(pending_balance)
        spendable_value_usd = utils.get_coin_value(spendable_balance)
        msg.reply(Template(lang.message_balance + lang.message_footer).render(username=msg.author.name, balance=str(balance),
                                              balance_value_usd=str(balance_value_usd),
                                              pendingbalance=str(pending_balance),
                                              pending_value_usd=str(pending_value_usd),
                                              spendablebalance=str(spendable_balance),
                                              spendable_value_usd=str(spendable_value_usd)))

        user_function.add_to_history(msg.author.name, "", "", balance, "balance")
    else:
        bot_logger.logger.info('user %s not registered ' % msg.author.name)
        msg.reply(Template(lang.message_need_register + lang.message_footer).render(username=msg.author.name))

def info_user(rpc, msg):
    if user_function.user_exist(msg.author.name):
        address = user_function.get_user_address(msg.author.name)
        balance = crypto.get_user_confirmed_balance(rpc, msg.author.name)
        pending_balance = crypto.get_user_unconfirmed_balance(rpc, msg.author.name)
        spendable_balance = crypto.get_user_spendable_balance(rpc, msg.author.name) + balance
        balance_value_usd = utils.get_coin_value(balance)
        pending_value_usd = utils.get_coin_value(pending_balance)
        spendable_value_usd = utils.get_coin_value(spendable_balance)
        msg.reply(Template(lang.message_account_details+ lang.message_footer ).render(username=msg.author.name, balance=str(balance),
                                                      balance_value_usd=str(balance_value_usd),
                                                      pendingbalance=str(pending_balance),
                                                      pending_value_usd=str(pending_value_usd),
                                                      spendablebalance=str(spendable_balance),
                                                      spendable_value_usd=str(spendable_value_usd),
                                                      address=address) )

    else:
        msg.reply(Template(lang.message_need_register+ lang.message_footer).render(username=msg.author.name))

def help_user(rpc, msg):
    if user_function.user_exist(msg.author.name):
        balance = crypto.get_user_confirmed_balance(rpc, msg.author.name)
        address = user_function.get_user_address(msg.author.name)
        msg.reply(Template(lang.message_help + lang.message_account_details+ lang.message_footer).render(username=msg.author.name, address=address,
                                                                              balance=str(
                                                                                balance)))
    else:
        msg.reply(Template(lang.message_need_register+ lang.message_footer).render(username=msg.author.name))

def withdraw_user(rpc, msg):
    split_message = msg.body.strip().split()

    if user_function.user_exist(msg.author.name):
        sender_address = user_function.get_user_address(msg.author.name)
        amount = split_message[1]
        user_balance = crypto.get_user_confirmed_balance(rpc, msg.author.name)
        user_spendable_balance = crypto.get_user_spendable_balance(rpc, msg.author.name)
        if utils.check_amount_valid(amount):
            if int(amount) >= user_balance + user_spendable_balance:
                bot_logger.logger.info('user %s not have enough to withdraw this amount (%s), balance = %s' % (
                    msg.author.name, amount, user_balance))
                msg.reply(lang.message_balance_low_withdraw.render(
                    username=msg.author.name, user_balance=str(user_balance), amount=str(amount)) + lang.message_footer)
            else:
                receiver_address = split_message[4]
                try:
                    if crypto.send_to(rpc, sender_address, receiver_address, amount):
                        user_function.add_to_history(msg.author.name, sender_address, receiver_address, amount,
                                                     "withdraw")
                        value_usd = utils.get_coin_value(amount)
                        msg.reply(lang.message_withdraw.render(
                            username=msg.author.name, receiver_address=receiver_address, amount=str(amount),
                            value_usd=str(value_usd)) + lang.message_footer)

                except:
                    traceback.print_exc()
        else:
            bot_logger.logger.info(lang.message_invalid_amount)
            msg.reply(lang.message_invalid_amount + lang.message_footer)
    else:
        msg.reply(lang.message_need_register.render(username=msg.author.name) + lang.message_footer)

def tip_user(rpc, reddit, msg):
    bot_logger.logger.info('An user mention detected ')
    split_message = msg.body.lower().strip().split()
    tip_index = split_message.index(str('+/u/' + config.bot_name))

    if split_message[tip_index] == str('+/u/' + config.bot_name) and split_message[tip_index + 2] == 'doge':

        amount = split_message[tip_index + 1]
        value_usd = utils.get_coin_value(amount)

        if utils.check_amount_valid(amount):
            parent_comment = msg.parent()
            if user_function.user_exist(msg.author.name):

                # check we have enough
                user_balance = crypto.get_user_confirmed_balance(rpc, msg.author.name)
                user_pending_balance = crypto.get_user_unconfirmed_balance(rpc, msg.author.name)
                user_spendable_balance = crypto.get_user_spendable_balance(rpc, msg.author.name) + user_balance
                if int(amount) >= user_spendable_balance:
                    # not enough for tip
                    if int(amount) < (user_spendable_balance + user_pending_balance):
                        msg.reply(Template(lang.message_balance_pending_tip).render(username=msg.author.name))
                    else:
                        bot_logger.logger.info('user %s not have enough to tip this amount (%s), balance = %s' % (
                            msg.author.name, str(amount), str(user_balance)))
                        msg.reply(Template(lang.message_balance_low_tip).render(username=msg.author.name))
                else:

                    # check user have address before tip
                    if user_function.user_exist(parent_comment.author.name):
                        if crypto.tip_user(rpc, msg.author.name, parent_comment.author.name, amount):
                            user_function.add_to_history(msg.author.name, msg.author.name, parent_comment.author.name,
                                                         amount,
                                                         "tip send")
                            user_function.add_to_history(parent_comment.author.name, msg.author.name,
                                                         parent_comment.author.name,
                                                         amount,
                                                         "tip receive")

                            bot_logger.logger.info(
                                '%s tip %s to %s' % (msg.author.name, str(amount), parent_comment.author.name))
                            # if user have 'verify' in this command he will have confirmation
                            if split_message.count('verify') or int(amount) >= 1000:
                                msg.reply(Template(lang.message_tip).render(
                                    sender=msg.author.name, receiver=parent_comment.author.name, amount=str(amount),
                                    value_usd=str(value_usd)))
                    else:
                        user_function.save_unregistered_tip(msg.author.name, parent_comment.author.name, amount,
                                                            msg.fullname)
                        user_function.add_to_history(msg.author.name, msg.author.name, parent_comment.author.name,
                                                     amount,
                                                     "tip send", False)
                        user_function.add_to_history(parent_comment.author.name, msg.author.name,
                                                     parent_comment.author.name,
                                                     amount,
                                                     "tip receive", False)
                        bot_logger.logger.info('user %s not registered' % parent_comment.author.name)
                        msg.reply(Template(lang.message_recipient_register).render(username=parent_comment.author.name))

                        reddit.redditor(parent_comment.author.name).message(
                            Template(
                                lang.message_recipient_need_register_title).render(amount=str(amount)),
                            Template(
                                lang.message_recipient_need_register_message).render(
                                username=parent_comment.author.name, sender=msg.author.name, amount=str(amount),
                                value_usd=str(value_usd)))

            else:
                msg.reply(Template(lang.message_need_register).render(username=msg.author.name))
        else:
            bot_logger.logger.info(lang.message_invalid_amount)
            msg.reply(lang.message_invalid_amount)

def history_user(msg):
    if user_function.user_exist(msg.author.name):
        data = user_function.get_user_history(msg.author.name)
        history_table = "\n\nDate|Sender|Receiver|Amount|Action|Finish|\n"
        history_table += "---|---|---|---|:-:|:-:\n"
        for tip in data:
            str_finish = "Pending"

            if tip['finish']:
                str_finish = "Successful"

            history_table += "%s|%s|%s|%s|%s|%s|\n" % (
                datetime.datetime.strptime(tip['time'], '%Y-%m-%dT%H:%M:%S.%f'), tip['sender'], tip['receiver'],
                str(tip['amount']), tip['action'], str_finish)

        msg.reply(Template(lang.message_history + history_table + lang.message_footer).render(username=msg.author.name))
    else:
        bot_logger.logger.info('user %s not registered ' % msg.author.name)
        msg.reply(Template(lang.message_need_register + lang.message_footer).render(username=msg.author.name))

# Resend tips to previously unregistered users that are now registered
def replay_remove_pending_tip(rpc, reddit):
    # check if it's not too old & replay tipping
    limit_date = datetime.datetime.now() - datetime.timedelta(days=3)

    # check if user have pending tips
    list_tips = user_function.get_unregistered_tip()

    if list_tips:
        for tip in list_tips:
            bot_logger.logger.info("replay tipping check for %s" % str(tip['id']))
            if (datetime.datetime.strptime(tip['time'], '%Y-%m-%dT%H:%M:%S.%f') > limit_date):
                if (user_function.user_exist(tip['receiver'])):
                    bot_logger.logger.info(
                        "replay tipping %s - %s send %s to %s  " % (
                            str(tip['id']), tip['sender'], tip['amount'], tip['receiver']))
                    crypto.tip_user(rpc, tip['sender'], tip['receiver'], tip['amount'])
                    user_function.remove_pending_tip(tip['id'])

                    value_usd = utils.get_coin_value(tip['amount'])

                    if 'message_fullname' in tip.keys():
                        msg_id = re.sub(r't\d+_(?P<id>\w+)', r'\g<id>', tip['message_fullname'])
                        msg = Comment(reddit, msg_id)
                        msg.reply(Template(lang.message_tip).render(
                            sender=tip['sender'], receiver=tip['receiver'], amount=str(tip['amount']),
                            value_usd=str(value_usd)))

                else:
                    bot_logger.logger.info(
                        "replay check for %s - user %s not registered " % (str(tip['id']), tip['receiver']))
            else:
                bot_logger.logger.info(
                    "delete old tipping - %s send %s for %s  " % (tip['sender'], tip['amount'], tip['receiver']))
                user_function.remove_pending_tip(tip['id'])
    else:
        bot_logger.logger.info("no pending tipping")
<|endoftext|>


<commit_before>import re
from lxml import etree

nameSpace_default = {'oai_dc': 'http://www.openarchives.org/OAI/2.0/oai_dc/',
                     'dc': 'http://purl.org/dc/elements/1.1/',
                     'mods': 'http://www.loc.gov/mods/v3',
                     'dcterms': 'http://purl.org/dc/terms'}

'''
Elements to add 2016-02-11:

    originInfo
        dates

    abstract
    subject
        URI
        authority (& split by authority... i.e. colocate TGM, LCSH, & FAST)
        geographicCode
    realtedItem
    relatedItem (archival host collection)
        purl to finding aid
        collection title
    location - physical location
    recordInfo (??)
'''
class mods:

    def load(input_file, nameSpace_dict=nameSpace_default):
        record_list = []
        tree = etree.parse(input_file)
        root = tree.getroot()
        for record in root.iterfind('.//{%s}mods' % nameSpace_dict['mods']):
            record_list.append(record)
        return record_list

    def abstract(record, nameSpace_dict=nameSpace_default):

        '''
        Currently returning a key error
        '''

        allAbstracts = []
        if record.find('./{%s}abstract' % nameSpace_default['mods']) is not None:
            for abstract in record.iterfind('./{%s}abstract' % nameSpace_dict['mods']):
                if len(abstract.attrib) >= 1:
                    print(abstract.attrib.keys(), abstract.attrib.values())

                    if abstract.attrib['type'] is not None:
                        print('TYPE-TRACE')
                        typed_abstract = {abstract.attrib['type'] : abstract.text}
                        allAbstracts.append(typed_abstract)
#                    elif abstract.attrib['displayLabel'] is not None:
#                        print('LABEL-TRACE')
#                        labeled_abstract = {abstract.attrib['displayLabel'] : abstract.text}
#                        allAbstracts.append(labeled_abstract)
                    else:
                        print('OTHER-TRACE')
                        allAbstracts.append(abstract.text)

                else:
                    allAbstracts.append(abstract.text)
        return allAbstracts

    def physicalDescription_note(record, nameSpace_dict=nameSpace_default):
        allNotes = []
        for physicalDescription in record.iterfind('./{%s}physicalDescription' % nameSpace_dict['mods']):
            for note in physicalDescription.iterfind('./{%s}note' % nameSpace_dict['mods']):
                allNotes.append(note.text)
        return allNotes

    def physicalLocation(record, nameSpace_dict=nameSpace_default):
        allLocations = []
        if record.find('.//{%s}physicalLocation' % nameSpace_dict['mods']) is not None:
            for location in record.iterfind('.//{%s}physicalLocation' % nameSpace_dict['mods']):
                allLocations.append(location.text)
        else:
            allLocations.append(None)
        return allLocations

    def extent(record, nameSpace_dict=nameSpace_default):
        allExtents = []
        if record.find('.//{%s}extent' % nameSpace_dict['mods']) is not None:
            for extent in record.iterfind('.//{%s}extent' % nameSpace_dict['mods']):
                allExtents.append(extent.text)
        else:
            allExtents.append('None')
        return allExtents

    def issuance(record, nameSpace_dict=nameSpace_default):
        allIssuances = []
        if record.find('.//{%s}issuance' % nameSpace_dict['mods']) is not None:
            for issuance in record.iterfind('.//{%s}issuance' % nameSpace_dict['mods']):
                allIssuances.append(issuance.text)
        else:
            allIssuances.append('None')
        return allIssuances

    def publication_place(record, nameSpace_dict=nameSpace_default):
        allPlaces = []
        if record.find('.//{%s}place' % nameSpace_dict['mods']) is not None:
            for place in record.iterfind('.//{%s}place' % nameSpace_dict['mods']):
                places = {}
                for term in place.iterchildren():
                    places[term.attrib['type']] = term.text
                allPlaces.append(places)
        else:
            allPlaces.append('None')
        return allPlaces

    def publisher(record, nameSpace_dict=nameSpace_default):
        allPublishers = []
        if record.find('.//{%s}publisher' % nameSpace_dict['mods']) is not None:
            for publisher in record.iterfind('.//{%s}publisher' % nameSpace_dict['mods']):
                allPublishers.append(publisher.text)
        else:
            allPublishers.append('None')
        return allPublishers

    def note(record, nameSpace_dict=nameSpace_default):
        allNotes = []
        for note in record.iterfind('./{%s}note' % nameSpace_dict['mods']):
            if len(note.attrib) >= 1:
                if note.attrib['type'] is not None:
                    #other option: allNotes.append('{0} : {1}'.format(note.attrib['type'], note.text))
                    typed_note = {note.attrib['type'] : note.text}
                    allNotes.append(typed_note)
            else:
                allNotes.append(note.text)
        return allNotes

    def digitalOrigin(record, nameSpace_dict):
        if record.find('.//{%s}digitalOrigin' % nameSpace_dict['mods']) is not None:
            return record.find('.//{%s}digitalOrigin' % nameSpace_dict['mods']).text
        else:
            return None

    def language(record, nameSpace_dict=nameSpace_default):
        allLanguages = []
        if record.find('.//{%s}language' % nameSpace_dict['mods']) is not None:
            for language in record.iterfind('.//{%s}language' % nameSpace_dict['mods']):
                languages = {}
                for term in language.iterchildren():
                    languages[term.attrib['type']] = term.text
                allLanguages.append(languages)
            return allLanguages
        else:
            return None

    def genre_text(record, nameSpace_dict=nameSpace_default):
        if record.find('.//{%s}genre' % nameSpace_dict['mods']) is not None:
            genre = record.find('.//{%s}genre' % nameSpace_dict['mods'])
            return genre.text
        else:
            return None

    def genre_URIs(record, nameSpace_dict=nameSpace_default):
        genreURIs = []
        for genre_elem in record.iterfind('.//{%s}genre' % nameSpace_dict['mods']):
            if len(genre_elem.attrib) >= 1:
                genreURI = {}
                for key in genre_elem.attrib.keys():
                    genreURI[key] = genre_elem.attrib[key]
                genreURIs.append(genreURI)
        return genreURIs

    def typeOfResource(record, nameSpace_dict=nameSpace_default):
        if record.find('.//{%s}typeOfResource' % nameSpace_dict['mods']) is not None:
            typeOfResource = record.find('.//{%s}typeOfResource' % nameSpace_dict['mods'])
            return typeOfResource.text
        else:
            return None

    def _nameGen_(names, fullName):
        keys = []
        for key in names.keys():
            keys.append(key)
        if all(x in keys for x in ['family', 'given',
                                   'termsOfAddress', 'date']):
            fullName = fullName + names['family'] + ', ' + names['given'] + ', ' + names['termsOfAddress'] + ' ' + names['date']
        elif all(x in keys for x in ['family', 'given', 'date']):
            fullName = fullName + names['family'] + ', ' + names['given'] + ' ' + names['date']
        elif all(x in keys for x in ['family', 'given', 'termsOfAddress']):
            fullName = fullName + names['family'] + ', ' + names['given'] + ', ' + names['termsOfAddress']
        elif all(x in keys for x in ['family', 'termsOfAddress', 'date']):
            fullName = fullName + names['family'] + ', ' + names['termsOfAddress'] + ' ' + names['date']
        elif all(x in keys for x in ['given', 'termsOfAddress', 'date']):
            fullName = fullName + names['given'] + ', ' + names['termsOfAddress'] + ' ' + names['date']
        elif all(x in keys for x in ['family', 'given']):
            fullName = fullName + names['family'] + ', ' + names['given']
        elif all(x in keys for x in ['family', 'date']):
            fullName = fullName + names['family'] + ', ' + names['date']
        elif all(x in keys for x in ['family', 'termsOfAddress']):
            fullName = fullName + names['family'] + ', ' + names['termsOfAddress']
        elif all(x in keys for x in ['given', 'date']):
            fullName = fullName + names['given'] + ', ' + names['date']
        elif all(x in keys for x in ['given', 'termsOfAddress']):
            fullName = fullName + names['given'] + ', ' + names['termsOfAddress']
        elif all(x in keys for x in ['termsOfAddress', 'date']):
            fullName = fullName + ', ' + names['termsOfAddress'] + ' ' + names['date']
        elif 'date' in keys:
            fullName = fullName + ', ' + names['date']
        elif 'termsOfAddress' in keys:
            fullName = fullName + ', ' + names['termsOfAddress']
        return fullName

    def name_generator(record, nameSpace_dict=nameSpace_default):
        allNames = []
        if len(record.findall('./{%s}name' % nameSpace_dict['mods'])) > 0:
            for name in record.iterfind('./{%s}name' % nameSpace_dict['mods']):
                fullName = ""
                if len(name.findall('./{%s}namePart' % nameSpace_dict['mods'])) > 1:
                    # Multipart name
                    names = {}
                    for namePart in name.findall('./{%s}namePart' % nameSpace_dict['mods']):
                        if 'type' not in namePart.attrib.keys():
                            fullName = namePart.text
                        elif 'type' in namePart.attrib.keys():
                            names[namePart.attrib['type']] = namePart.text
                    fullName = mods._nameGen_(names, fullName)
                else:
                    # Single part name
                    fullName = fullName + name.find('./{%s}namePart' % nameSpace_dict['mods']).text
                allNames.append(fullName)
            return allNames
        else:
            allNames.append('None')
            return allNames

    def title_generator(record, nameSpace_dict=nameSpace_default):
        allTitles = []
        for title in record.iterfind('.//{%s}titleInfo' % nameSpace_dict['mods']):
            if title.find('./{%s}nonSort' % nameSpace_dict['mods']) is not None and title.find(
                            './{%s}title' % nameSpace_dict['mods']) is not None and title.find(
                            './{%s}subTitle' % nameSpace_dict['mods']) is not None:
                titleFull = title.find('./{%s}nonSort' % nameSpace_dict['mods']).text + ' ' + title.find(
                    './{%s}title' % nameSpace_dict['mods']).text + ': ' + title.find(
                    './{%s}subTitle' % nameSpace_dict['mods']).text
            elif title.find('./{%s}nonSort' % nameSpace_dict['mods']) is not None and title.find(
                            './{%s}title' % nameSpace_dict['mods']) is not None:
                titleFull = title.find('./{%s}nonSort' % nameSpace_dict['mods']).text + ' ' + title.find(
                    './{%s}title' % nameSpace_dict['mods']).text
            else:
                titleFull = title.find('./{%s}title' % nameSpace_dict['mods']).text
            allTitles.append(titleFull)
        return allTitles

    def date_generator(record, nameSpace_dict=nameSpace_default):
        if record.find('./{%s}originInfo/{%s}copyrightDate' % (
        nameSpace_dict['mods'], nameSpace_dict['mods'])) is not None:
            date = record.find(
                './{%s}originInfo/{%s}copyrightDate' % (nameSpace_dict['mods'], nameSpace_dict['mods'])).text
        elif record.find(
                        './{%s}originInfo/{%s}dateCreated' % (nameSpace_dict['mods'], nameSpace_dict['mods'])) is not None:
            date = record.find(
                './{%s}originInfo/{%s}dateCreated' % (nameSpace_dict['mods'], nameSpace_dict['mods'])).text
        elif record.find(
                        './{%s}originInfo/{%s}dateIssued' % (nameSpace_dict['mods'], nameSpace_dict['mods'])) is not None:
            date = record.find(
                './{%s}originInfo/{%s}dateIssued' % (nameSpace_dict['mods'], nameSpace_dict['mods'])).text
        elif record.find(
                        './{%s}originInfo/{%s}dateOther' % (nameSpace_dict['mods'], nameSpace_dict['mods'])) is not None:
            date = record.find(
                './{%s}originInfo/{%s}dateOther' % (nameSpace_dict['mods'], nameSpace_dict['mods'])).text
        else:
            date = "No date"
        return date

    def subject_generator(record, nameSpace_dict=nameSpace_default):
        allSubjects = []
        for subject in record.iterfind('.//{%s}subject' % nameSpace_dict['mods']):
            fullSubject = []
            for subjectTerm in subject.iterchildren():
                fullSubject.append({subjectTerm.tag[28: ] : subjectTerm.text})
    #        if fullSubject:
    #            allSubjects.append(fullSubject)
            allSubjects.append(fullSubject)
        return allSubjects

class fsudl:

    def purl_search(mods_record, nameSpace_dict=nameSpace_default):
        purl = re.compile('((https?):((//)|(\\\\))+([\w\d:#@%/;$()~_?\+-=\\\.&](#!)?)*)')
        for url in mods_record.iterfind('./{%s}location/{%s}url' % (nameSpace_dict['mods'], nameSpace_dict['mods'])):
            match = purl.search(url.text)
            if match:
                return match.group()

    def pid_search(mods_record, nameSpace_dict=nameSpace_default):
        pid = re.compile('fsu:[0-9]*')
        for identifier in mods_record.iterfind('.//{%s}identifier' % nameSpace_dict['mods']):
            match = pid.search(identifier.text)
            if match:
                return match.group()

class oai_dc:

    '''
    Some implementations of the OAI-PMH standard do not include the
    default namespace for oai_dc. A future release will include a test
    for inclusion of the default namespace,and add it to the file when
    not present.
    '''

    def load(input_file, nameSpace_dict=nameSpace_default):
        record_list = []
        tree = etree.parse(input_file)
        root = tree.getroot()
        for record in root.iterfind('.//{%s}record' % nameSpace_dict['oai_dc']):
            record_list.append(record)
        return record_list

    def pid_search(record, nameSpace_dict=nameSpace_default):
        pid = re.compile('fsu_[0-9]*')
        for identifier in record.iterfind('.//{%s}identifier' % nameSpace_dict['oai_dc']):
            match = pid.search(identifier.text)
            if match:
                return match.group().replace('_',':')
<commit_msg>abstract bugs squashed<commit_after>import re
from lxml import etree

nameSpace_default = {'oai_dc': 'http://www.openarchives.org/OAI/2.0/oai_dc/',
                     'dc': 'http://purl.org/dc/elements/1.1/',
                     'mods': 'http://www.loc.gov/mods/v3',
                     'dcterms': 'http://purl.org/dc/terms'}

'''
Elements to add 2016-02-11:

    originInfo
        dates

    abstract
    subject
        URI
        authority (& split by authority... i.e. colocate TGM, LCSH, & FAST)
        geographicCode
    realtedItem
    relatedItem (archival host collection)
        purl to finding aid
        collection title
    location - physical location
    recordInfo (??)
'''
class mods:

    def load(input_file, nameSpace_dict=nameSpace_default):
        record_list = []
        tree = etree.parse(input_file)
        root = tree.getroot()
        for record in root.iterfind('.//{%s}mods' % nameSpace_dict['mods']):
            record_list.append(record)
        return record_list

    def abstract(record, nameSpace_dict=nameSpace_default):

        allAbstracts = []
        if record.find('./{%s}abstract' % nameSpace_default['mods']) is not None:
            for abstract in record.iterfind('./{%s}abstract' % nameSpace_dict['mods']):
                if len(abstract.attrib) >= 1:
                    if 'type' in abstract.attrib.keys():
                        typed_abstract = {abstract.attrib['type'] : abstract.text}
                        allAbstracts.append(typed_abstract)
                    elif 'displayLabel' in abstract.attrib.keys():
                        labeled_abstract = {abstract.attrib['displayLabel'] : abstract.text}
                        allAbstracts.append(labeled_abstract)
                    else:
                        allAbstracts.append(abstract.text)
                else:
                    allAbstracts.append(abstract.text)
        return allAbstracts

    def physicalDescription_note(record, nameSpace_dict=nameSpace_default):
        allNotes = []
        for physicalDescription in record.iterfind('./{%s}physicalDescription' % nameSpace_dict['mods']):
            for note in physicalDescription.iterfind('./{%s}note' % nameSpace_dict['mods']):
                allNotes.append(note.text)
        return allNotes

    def physicalLocation(record, nameSpace_dict=nameSpace_default):
        allLocations = []
        if record.find('.//{%s}physicalLocation' % nameSpace_dict['mods']) is not None:
            for location in record.iterfind('.//{%s}physicalLocation' % nameSpace_dict['mods']):
                allLocations.append(location.text)
        else:
            allLocations.append(None)
        return allLocations

    def extent(record, nameSpace_dict=nameSpace_default):
        allExtents = []
        if record.find('.//{%s}extent' % nameSpace_dict['mods']) is not None:
            for extent in record.iterfind('.//{%s}extent' % nameSpace_dict['mods']):
                allExtents.append(extent.text)
        else:
            allExtents.append('None')
        return allExtents

    def issuance(record, nameSpace_dict=nameSpace_default):
        allIssuances = []
        if record.find('.//{%s}issuance' % nameSpace_dict['mods']) is not None:
            for issuance in record.iterfind('.//{%s}issuance' % nameSpace_dict['mods']):
                allIssuances.append(issuance.text)
        else:
            allIssuances.append('None')
        return allIssuances

    def publication_place(record, nameSpace_dict=nameSpace_default):
        allPlaces = []
        if record.find('.//{%s}place' % nameSpace_dict['mods']) is not None:
            for place in record.iterfind('.//{%s}place' % nameSpace_dict['mods']):
                places = {}
                for term in place.iterchildren():
                    places[term.attrib['type']] = term.text
                allPlaces.append(places)
        else:
            allPlaces.append('None')
        return allPlaces

    def publisher(record, nameSpace_dict=nameSpace_default):
        allPublishers = []
        if record.find('.//{%s}publisher' % nameSpace_dict['mods']) is not None:
            for publisher in record.iterfind('.//{%s}publisher' % nameSpace_dict['mods']):
                allPublishers.append(publisher.text)
        else:
            allPublishers.append('None')
        return allPublishers

    def note(record, nameSpace_dict=nameSpace_default):
        allNotes = []
        for note in record.iterfind('./{%s}note' % nameSpace_dict['mods']):
            if len(note.attrib) >= 1:
                if note.attrib['type'] is not None:
                    #other option: allNotes.append('{0} : {1}'.format(note.attrib['type'], note.text))
                    typed_note = {note.attrib['type'] : note.text}
                    allNotes.append(typed_note)
            else:
                allNotes.append(note.text)
        return allNotes

    def digitalOrigin(record, nameSpace_dict):
        if record.find('.//{%s}digitalOrigin' % nameSpace_dict['mods']) is not None:
            return record.find('.//{%s}digitalOrigin' % nameSpace_dict['mods']).text
        else:
            return None

    def language(record, nameSpace_dict=nameSpace_default):
        allLanguages = []
        if record.find('.//{%s}language' % nameSpace_dict['mods']) is not None:
            for language in record.iterfind('.//{%s}language' % nameSpace_dict['mods']):
                languages = {}
                for term in language.iterchildren():
                    languages[term.attrib['type']] = term.text
                allLanguages.append(languages)
            return allLanguages
        else:
            return None

    def genre_text(record, nameSpace_dict=nameSpace_default):
        if record.find('.//{%s}genre' % nameSpace_dict['mods']) is not None:
            genre = record.find('.//{%s}genre' % nameSpace_dict['mods'])
            return genre.text
        else:
            return None

    def genre_URIs(record, nameSpace_dict=nameSpace_default):
        genreURIs = []
        for genre_elem in record.iterfind('.//{%s}genre' % nameSpace_dict['mods']):
            if len(genre_elem.attrib) >= 1:
                genreURI = {}
                for key in genre_elem.attrib.keys():
                    genreURI[key] = genre_elem.attrib[key]
                genreURIs.append(genreURI)
        return genreURIs

    def typeOfResource(record, nameSpace_dict=nameSpace_default):
        if record.find('.//{%s}typeOfResource' % nameSpace_dict['mods']) is not None:
            typeOfResource = record.find('.//{%s}typeOfResource' % nameSpace_dict['mods'])
            return typeOfResource.text
        else:
            return None

    def _nameGen_(names, fullName):
        keys = []
        for key in names.keys():
            keys.append(key)
        if all(x in keys for x in ['family', 'given',
                                   'termsOfAddress', 'date']):
            fullName = fullName + names['family'] + ', ' + names['given'] + ', ' + names['termsOfAddress'] + ' ' + names['date']
        elif all(x in keys for x in ['family', 'given', 'date']):
            fullName = fullName + names['family'] + ', ' + names['given'] + ' ' + names['date']
        elif all(x in keys for x in ['family', 'given', 'termsOfAddress']):
            fullName = fullName + names['family'] + ', ' + names['given'] + ', ' + names['termsOfAddress']
        elif all(x in keys for x in ['family', 'termsOfAddress', 'date']):
            fullName = fullName + names['family'] + ', ' + names['termsOfAddress'] + ' ' + names['date']
        elif all(x in keys for x in ['given', 'termsOfAddress', 'date']):
            fullName = fullName + names['given'] + ', ' + names['termsOfAddress'] + ' ' + names['date']
        elif all(x in keys for x in ['family', 'given']):
            fullName = fullName + names['family'] + ', ' + names['given']
        elif all(x in keys for x in ['family', 'date']):
            fullName = fullName + names['family'] + ', ' + names['date']
        elif all(x in keys for x in ['family', 'termsOfAddress']):
            fullName = fullName + names['family'] + ', ' + names['termsOfAddress']
        elif all(x in keys for x in ['given', 'date']):
            fullName = fullName + names['given'] + ', ' + names['date']
        elif all(x in keys for x in ['given', 'termsOfAddress']):
            fullName = fullName + names['given'] + ', ' + names['termsOfAddress']
        elif all(x in keys for x in ['termsOfAddress', 'date']):
            fullName = fullName + ', ' + names['termsOfAddress'] + ' ' + names['date']
        elif 'date' in keys:
            fullName = fullName + ', ' + names['date']
        elif 'termsOfAddress' in keys:
            fullName = fullName + ', ' + names['termsOfAddress']
        return fullName

    def name_generator(record, nameSpace_dict=nameSpace_default):
        allNames = []
        if len(record.findall('./{%s}name' % nameSpace_dict['mods'])) > 0:
            for name in record.iterfind('./{%s}name' % nameSpace_dict['mods']):
                fullName = ""
                if len(name.findall('./{%s}namePart' % nameSpace_dict['mods'])) > 1:
                    # Multipart name
                    names = {}
                    for namePart in name.findall('./{%s}namePart' % nameSpace_dict['mods']):
                        if 'type' not in namePart.attrib.keys():
                            fullName = namePart.text
                        elif 'type' in namePart.attrib.keys():
                            names[namePart.attrib['type']] = namePart.text
                    fullName = mods._nameGen_(names, fullName)
                else:
                    # Single part name
                    fullName = fullName + name.find('./{%s}namePart' % nameSpace_dict['mods']).text
                allNames.append(fullName)
            return allNames
        else:
            allNames.append('None')
            return allNames

    def title_generator(record, nameSpace_dict=nameSpace_default):
        allTitles = []
        for title in record.iterfind('.//{%s}titleInfo' % nameSpace_dict['mods']):
            if title.find('./{%s}nonSort' % nameSpace_dict['mods']) is not None and title.find(
                            './{%s}title' % nameSpace_dict['mods']) is not None and title.find(
                            './{%s}subTitle' % nameSpace_dict['mods']) is not None:
                titleFull = title.find('./{%s}nonSort' % nameSpace_dict['mods']).text + ' ' + title.find(
                    './{%s}title' % nameSpace_dict['mods']).text + ': ' + title.find(
                    './{%s}subTitle' % nameSpace_dict['mods']).text
            elif title.find('./{%s}nonSort' % nameSpace_dict['mods']) is not None and title.find(
                            './{%s}title' % nameSpace_dict['mods']) is not None:
                titleFull = title.find('./{%s}nonSort' % nameSpace_dict['mods']).text + ' ' + title.find(
                    './{%s}title' % nameSpace_dict['mods']).text
            else:
                titleFull = title.find('./{%s}title' % nameSpace_dict['mods']).text
            allTitles.append(titleFull)
        return allTitles

    def date_generator(record, nameSpace_dict=nameSpace_default):
        if record.find('./{%s}originInfo/{%s}copyrightDate' % (
        nameSpace_dict['mods'], nameSpace_dict['mods'])) is not None:
            date = record.find(
                './{%s}originInfo/{%s}copyrightDate' % (nameSpace_dict['mods'], nameSpace_dict['mods'])).text
        elif record.find(
                        './{%s}originInfo/{%s}dateCreated' % (nameSpace_dict['mods'], nameSpace_dict['mods'])) is not None:
            date = record.find(
                './{%s}originInfo/{%s}dateCreated' % (nameSpace_dict['mods'], nameSpace_dict['mods'])).text
        elif record.find(
                        './{%s}originInfo/{%s}dateIssued' % (nameSpace_dict['mods'], nameSpace_dict['mods'])) is not None:
            date = record.find(
                './{%s}originInfo/{%s}dateIssued' % (nameSpace_dict['mods'], nameSpace_dict['mods'])).text
        elif record.find(
                        './{%s}originInfo/{%s}dateOther' % (nameSpace_dict['mods'], nameSpace_dict['mods'])) is not None:
            date = record.find(
                './{%s}originInfo/{%s}dateOther' % (nameSpace_dict['mods'], nameSpace_dict['mods'])).text
        else:
            date = "No date"
        return date

    def subject_generator(record, nameSpace_dict=nameSpace_default):
        allSubjects = []
        for subject in record.iterfind('.//{%s}subject' % nameSpace_dict['mods']):
            fullSubject = []
            for subjectTerm in subject.iterchildren():
                fullSubject.append({subjectTerm.tag[28: ] : subjectTerm.text})
    #        if fullSubject:
    #            allSubjects.append(fullSubject)
            allSubjects.append(fullSubject)
        return allSubjects

class fsudl:

    def purl_search(mods_record, nameSpace_dict=nameSpace_default):
        purl = re.compile('((https?):((//)|(\\\\))+([\w\d:#@%/;$()~_?\+-=\\\.&](#!)?)*)')
        for url in mods_record.iterfind('./{%s}location/{%s}url' % (nameSpace_dict['mods'], nameSpace_dict['mods'])):
            match = purl.search(url.text)
            if match:
                return match.group()

    def pid_search(mods_record, nameSpace_dict=nameSpace_default):
        pid = re.compile('fsu:[0-9]*')
        for identifier in mods_record.iterfind('.//{%s}identifier' % nameSpace_dict['mods']):
            match = pid.search(identifier.text)
            if match:
                return match.group()

class oai_dc:

    '''
    Some implementations of the OAI-PMH standard do not include the
    default namespace for oai_dc. A future release will include a test
    for inclusion of the default namespace,and add it to the file when
    not present.
    '''

    def load(input_file, nameSpace_dict=nameSpace_default):
        record_list = []
        tree = etree.parse(input_file)
        root = tree.getroot()
        for record in root.iterfind('.//{%s}record' % nameSpace_dict['oai_dc']):
            record_list.append(record)
        return record_list

    def pid_search(record, nameSpace_dict=nameSpace_default):
        pid = re.compile('fsu_[0-9]*')
        for identifier in record.iterfind('.//{%s}identifier' % nameSpace_dict['oai_dc']):
            match = pid.search(identifier.text)
            if match:
                return match.group().replace('_',':')
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Pyrana -- a minimalist music player.

This program was written out of frustration with larger music players. Pretty
much all I've ever wanted out of a music player was one that played random
albums by picking a random artist, then picking a random album, then playing
that album, then picking _another random artist_.

For some reason,I've never run into a music player that played random albums
that way, which led to me often ending up having to skip past multiple Frank
Zappa albums when playing random albums. I like Zappa, but not that goddamned
much. Same deal with Slayer and Aphex Twin and Bad Religion, and all these
other artists and bands that I have discographies of.

Another problem I've had is that a lot of 'modern' music players base their
artist and album information off of metadata. I have a couple of issues with
this:

1: My music collection is large. Doing a scan of metadata, even if it only
needs to be done once takes a long time. I hate it.

2: My music collection has unreliable metadata. This is because I'm a lazy
asshole, and haven't kept up with fixing metadata on music that I
download. That's right, I download music -- so do you, shut up. My music
collection is so large that the probability of me fixing the metadata ever is
basically nil. I can, however, rely on the directory structure of my music
collection.

This little program does one thing, and does it acceptably -- play random
albums from a music collection. Maybe it'll expand in the future to include
things like last.fm support. Maybe.
"""

import os
import random
import ConfigParser

import pygst
pygst.require("0.10")
import gst

import pygtk
pygtk.require('2.0')
import gtk

import pynotify
pynotify.init("Basics")

class Pyrana(object):
    """Our player, sending sweet, sweet sounds to our speakers.
    """

    def __init__(self):
        """Initialize our player with a root directory to search through for
        music, and start the gtk main thread.
        """
        self.base_path = os.path.dirname(os.path.realpath(__file__))

        self.status_icon = gtk.StatusIcon()
        self.status_icon.connect('activate', self.on_left_click)
        self.status_icon.connect('popup-menu', self.on_right_click)
        self.status_icon.set_visible(True)
        self.status_icon.set_tooltip("Pyrana!")
        self.status_icon.set_from_file(os.path.join(self.base_path,'stopped.png'))

        self.menu = gtk.Menu()
        skip_song = gtk.MenuItem("Skip Song")
        skip_album = gtk.MenuItem("Skip Album")
        quit_app = gtk.MenuItem("Quit")
        self.menu.append(skip_song)
        self.menu.append(skip_album)
        self.menu.append(quit_app)

        skip_song.connect_object("activate", self.skip_song, "Skip Song")
        skip_album.connect_object("activate", self.skip_album, "Skip Album")
        quit_app.connect_object("activate", self.quit, "Quit")

        skip_song.show()
        skip_album.show()
        quit_app.show()

        self.config = ConfigParser.ConfigParser()
        self.config.read(os.path.join(self.base_path, 'pyrana.cfg'))

        self.pidgin_status = self.config.get('main',
                                             'update_pidgin_status')
        self.use_notify = self.config.get('main', 'use_notify')

        #give us a list of sets of albums by artists, assuming the directory
        #structure
        #Artists
        # Albums
        #  Songs
        #This is a little ugly, but whatever.
        root = os.path.expanduser(
            self.config.get('main', 'music_dir'))
        self.artists = [[os.path.join(artist, album)
                             for album in os.listdir(artist)
                             if os.path.isdir(os.path.join(artist, album))]
                        for artist in
                        [os.path.join(root, artistname)
                         for artistname in os.listdir(root)
                         if os.path.isdir(os.path.join(root, artistname))]]

        #just in case we get some empty directories
        self.artists = [a for a in self.artists if a]

        self.player = gst.element_factory_make("playbin2", "player")

        bus = self.player.get_bus()
        bus.add_signal_watch()
        bus.enable_sync_message_emission()
        bus.connect('message::eos', self.on_eos)

        self.playing = False
        self.cur_song = None
        self.cur_album = None
        self.cur_artist = None

        gtk.main()

    def on_eos(self, bus, msg):
        self.player.set_state(gst.STATE_NULL)
        self.cur_song = None
        self.playing = False
        self.on_left_click(None)

    def on_left_click(self, widget, data=None):
        """Click handler for our status icon. Play or stop playing,
        respectively.
        """
        if not self.playing:
            self.playing = True
            self.status_icon.set_from_file(
                os.path.join(self.base_path, 'playing.png'))
            if not self.cur_song:
                self.cur_song = self.get_next_song()
                self.player.set_property('uri', 'file://%s' % self.cur_song)
            self.player.set_state(gst.STATE_PLAYING)
            self.status_icon.set_tooltip(self.cur_song)
            if self.use_notify:
                self.notify(self.cur_song)
            if self.pidgin_status:
                self.update_pidgin_status(self.cur_song)

        else:
            self.player.set_state(gst.STATE_PAUSED)
            self.playing = False
            self.status_icon.set_from_file(os.path.join(
                    self.base_path, 'stopped.png'))
            self.status_icon.set_tooltip("[PAUSED] %s" % self.cur_song)
            if self.pidgin_status:
                self.update_pidgin_status()

    def on_right_click(self, data, event_button, event_time):
        self.menu.popup(None, None, None, event_button, event_time)

    def get_next_song(self):
        if not self.cur_album:
            artist = self.cur_artist

            while artist == self.cur_artist:
                artist = random.choice(self.artists)

            albumpath = artist.pop(random.randrange(len(artist)))
            self.cur_album = sorted([os.path.join(albumpath, song)
                                     for song in os.listdir(albumpath)])
        song = self.cur_album[0]
        self.cur_album = self.cur_album[1:]
        return song

    def skip_song(self, data=None):
        self.on_eos(None, None)

    def skip_album(self, data=None):
        self.cur_album = None
        self.on_eos(None, None)

    def quit(widget, data=None):
        gtk.main_quit()

    def notify(self, songpath):
        """Use libnotify to show growl-like alerts about what's playing.
        """
        to_display = "Playing: %s" % songpath
        notification = pynotify.Notification("Pyrana", to_display)
        notification.show()

    def update_pidgin_status(self, songpath=None):
        import dbus
        bus = dbus.SessionBus()

        if songpath:
            parts = songpath.split('/')
            artist = parts[-3]
            album = parts[-2]
            song = parts[-1]
            message = "%s (%s): %s" % (artist, album, song)
        else:
            message = "Paused"

        if "im.pidgin.purple.PurpleService" in bus.list_names():
            purple = bus.get_object("im.pidgin.purple.PurpleService",
                                    "/im/pidgin/purple/PurpleObject",
                                    "im.pidgin.purple.PurpleInterface")
            current = purple.PurpleSavedstatusGetType(
                purple.PurpleSavedstatusGetCurrent())
            status = purple.PurpleSavedstatusNew("", current)
            purple.PurpleSavedstatusSetMessage(status, message)

            purple.PurpleSavedstatusActivate(status)

if __name__ == '__main__':
    import sys
    Pyrana()
<commit_msg>only pull audio files from album directory<commit_after>#!/usr/bin/env python
"""
Pyrana -- a minimalist music player.

This program was written out of frustration with larger music players. Pretty
much all I've ever wanted out of a music player was one that played random
albums by picking a random artist, then picking a random album, then playing
that album, then picking _another random artist_.

For some reason,I've never run into a music player that played random albums
that way, which led to me often ending up having to skip past multiple Frank
Zappa albums when playing random albums. I like Zappa, but not that goddamned
much. Same deal with Slayer and Aphex Twin and Bad Religion, and all these
other artists and bands that I have discographies of.

Another problem I've had is that a lot of 'modern' music players base their
artist and album information off of metadata. I have a couple of issues with
this:

1: My music collection is large. Doing a scan of metadata, even if it only
needs to be done once takes a long time. I hate it.

2: My music collection has unreliable metadata. This is because I'm a lazy
asshole, and haven't kept up with fixing metadata on music that I
download. That's right, I download music -- so do you, shut up. My music
collection is so large that the probability of me fixing the metadata ever is
basically nil. I can, however, rely on the directory structure of my music
collection.

This little program does one thing, and does it acceptably -- play random
albums from a music collection. Maybe it'll expand in the future to include
things like last.fm support. Maybe.
"""

import os
import random
import ConfigParser

import pygst
pygst.require("0.10")
import gst

import pygtk
pygtk.require('2.0')
import gtk

import pynotify
pynotify.init("Basics")

class Pyrana(object):
    """Our player, sending sweet, sweet sounds to our speakers.
    """

    audio_types = [
        '.mp3',
        '.m4a',
        '.ogg'
    ]

    def __init__(self):
        """Initialize our player with a root directory to search through for
        music, and start the gtk main thread.
        """
        self.base_path = os.path.dirname(os.path.realpath(__file__))

        self.status_icon = gtk.StatusIcon()
        self.status_icon.connect('activate', self.on_left_click)
        self.status_icon.connect('popup-menu', self.on_right_click)
        self.status_icon.set_visible(True)
        self.status_icon.set_tooltip("Pyrana!")
        self.status_icon.set_from_file(os.path.join(self.base_path,'stopped.png'))

        self.menu = gtk.Menu()
        skip_song = gtk.MenuItem("Skip Song")
        skip_album = gtk.MenuItem("Skip Album")
        quit_app = gtk.MenuItem("Quit")
        self.menu.append(skip_song)
        self.menu.append(skip_album)
        self.menu.append(quit_app)

        skip_song.connect_object("activate", self.skip_song, "Skip Song")
        skip_album.connect_object("activate", self.skip_album, "Skip Album")
        quit_app.connect_object("activate", self.quit, "Quit")

        skip_song.show()
        skip_album.show()
        quit_app.show()

        self.config = ConfigParser.ConfigParser()
        self.config.read(os.path.join(self.base_path, 'pyrana.cfg'))

        self.pidgin_status = self.config.get('main',
                                             'update_pidgin_status')
        self.use_notify = self.config.get('main', 'use_notify')

        #give us a list of sets of albums by artists, assuming the directory
        #structure
        #Artists
        # Albums
        #  Songs
        #This is a little ugly, but whatever.
        root = os.path.expanduser(
            self.config.get('main', 'music_dir'))
        self.artists = [[os.path.join(artist, album)
                             for album in os.listdir(artist)
                             if os.path.isdir(os.path.join(artist, album))]
                        for artist in
                        [os.path.join(root, artistname)
                         for artistname in os.listdir(root)
                         if os.path.isdir(os.path.join(root, artistname))]]

        #just in case we get some empty directories
        self.artists = [a for a in self.artists if a]

        self.player = gst.element_factory_make("playbin2", "player")

        bus = self.player.get_bus()
        bus.add_signal_watch()
        bus.enable_sync_message_emission()
        bus.connect('message::eos', self.on_eos)

        self.playing = False
        self.cur_song = None
        self.cur_album = None
        self.cur_artist = None

        gtk.main()

    def on_eos(self, bus, msg):
        self.player.set_state(gst.STATE_NULL)
        self.cur_song = None
        self.playing = False
        self.on_left_click(None)

    def on_left_click(self, widget, data=None):
        """Click handler for our status icon. Play or stop playing,
        respectively.
        """
        if not self.playing:
            self.playing = True
            self.status_icon.set_from_file(
                os.path.join(self.base_path, 'playing.png'))
            if not self.cur_song:
                self.cur_song = self.get_next_song()
                self.player.set_property('uri', 'file://%s' % self.cur_song)
            self.player.set_state(gst.STATE_PLAYING)
            self.status_icon.set_tooltip(self.cur_song)
            if self.use_notify:
                self.notify(self.cur_song)
            if self.pidgin_status:
                self.update_pidgin_status(self.cur_song)

        else:
            self.player.set_state(gst.STATE_PAUSED)
            self.playing = False
            self.status_icon.set_from_file(os.path.join(
                    self.base_path, 'stopped.png'))
            self.status_icon.set_tooltip("[PAUSED] %s" % self.cur_song)
            if self.pidgin_status:
                self.update_pidgin_status()

    def on_right_click(self, data, event_button, event_time):
        self.menu.popup(None, None, None, event_button, event_time)

    def get_next_song(self):
        import os.path
        if not self.cur_album:
            artist = self.cur_artist

            while artist == self.cur_artist:
                artist = random.choice(self.artists)

            albumpath = artist.pop(random.randrange(len(artist)))
            self.cur_album = sorted(
                [os.path.join(albumpath, song)
                 for song in os.listdir(albumpath)
                 if os.path.splitext(song)[-1] in self.audio_types])

        song = self.cur_album[0]
        self.cur_album = self.cur_album[1:]
        return song

    def skip_song(self, data=None):
        self.on_eos(None, None)

    def skip_album(self, data=None):
        self.cur_album = None
        self.on_eos(None, None)

    def quit(widget, data=None):
        gtk.main_quit()

    def notify(self, songpath):
        """Use libnotify to show growl-like alerts about what's playing.
        """
        to_display = "Playing: %s" % songpath
        notification = pynotify.Notification("Pyrana", to_display)
        notification.show()

    def update_pidgin_status(self, songpath=None):
        import dbus
        bus = dbus.SessionBus()

        if songpath:
            parts = songpath.split('/')
            artist = parts[-3]
            album = parts[-2]
            song = parts[-1]
            message = "%s (%s): %s" % (artist, album, song)
        else:
            message = "Paused"

        if "im.pidgin.purple.PurpleService" in bus.list_names():
            purple = bus.get_object("im.pidgin.purple.PurpleService",
                                    "/im/pidgin/purple/PurpleObject",
                                    "im.pidgin.purple.PurpleInterface")
            current = purple.PurpleSavedstatusGetType(
                purple.PurpleSavedstatusGetCurrent())
            status = purple.PurpleSavedstatusNew("", current)
            purple.PurpleSavedstatusSetMessage(status, message)

            purple.PurpleSavedstatusActivate(status)

if __name__ == '__main__':
    import sys
    Pyrana()
<|endoftext|>


<commit_before>#! /usr/bin/env python3
# -*- coding: utf-8 -*-
""" Python interface to SDMX """

from suds.client import Client
import datetime
import requests
import json
import lxml.etree

#Utter failure with suds and the SOAP interface. Can anybody explain this to me
#client = Client("http://ec.europa.eu/eurostat/SDMX/diss-ws/SdmxServiceService?wsdl")
#GetDataflow = client.factory.create('GetDataflow')
#GetDataflow.DataflowQuery.Header.ID = "QUERY"
#GetDataflow.DataflowQuery.Header.Test = False
#today = datetime.date.today().strftime('%Y-%m-%d')
#GetDataflow.DataflowQuery.Header.Prepared = today
#GetDataflow.DataflowQuery.Header.Sender = "Michael"
#GetDataflow.DataflowQuery.Header.DataProvider = "ESTAT"

class SDMX_ML(object):
    def __init__(self,SDMXML):
        self.SDMXML = SDMXML

class SDMX_REST(object):
    def __init__(self,sdmx_url,agencyID):
        self.sdmx_url = sdmx_url
        self.agencyID = agencyID
        self._dataflow = None

    @property
    def dataflow(self):
        if not self._dataflow:
            resource = 'dataflow'
            resourceID = 'all'
            version = 'latest'
            self._dataflow = requests.get(self.sdmx_url+'/'
                                          +resource+'/'
                                          +self.agencyID+'/'
                                          +resourceID+'/'
                                          +version)
            self._dataflow = self._dataflow.text.encode('utf-8')
            parser = lxml.etree.XMLParser(ns_clean=True, recover=True, encoding='utf-8')
            print(self._dataflow)
            self._dataflow = lxml.etree.fromstring(self._dataflow, parser=parser)
        return self._dataflow

    def data_extraction(self, flowRef, key, startperiod, endperiod):
        resource = 'data'
        request = requests.get(self.sdmx_url+'/'
                               +resource+'/'
                               +flowRef+'/'
                               +key
                               +'?startperiod='+startperiod
                               +'&endPeriod='+endperiod)
        xml_response = request.text.encode('utf-8')
        parser = lxml.etree.XMLParser(ns_clean=True, recover=True, encoding='utf-8')
        SDMXML = lxml.etree.fromstring(xml_response, parser=parser)
        return SDMXML

eurostat = SDMX_REST("http://www.ec.europa.eu/eurostat/SDMX/diss-web/rest", 'ESTAT')
<commit_msg>Fixed encoding headache.<commit_after>#! /usr/bin/env python3
# -*- coding: utf-8 -*-
""" Python interface to SDMX """

from suds.client import Client
import datetime
import requests
import json
import lxml.etree
import logging

#Utter failure with suds and the SOAP interface. Can anybody explain this to me
#client = Client("http://ec.europa.eu/eurostat/SDMX/diss-ws/SdmxServiceService?wsdl")
#GetDataflow = client.factory.create('GetDataflow')
#GetDataflow.DataflowQuery.Header.ID = "QUERY"
#GetDataflow.DataflowQuery.Header.Test = False
#today = datetime.date.today().strftime('%Y-%m-%d')
#GetDataflow.DataflowQuery.Header.Prepared = today
#GetDataflow.DataflowQuery.Header.Sender = "Michael"
#GetDataflow.DataflowQuery.Header.DataProvider = "ESTAT"

def query_rest(url):
	request = requests.get(url)
	parser = lxml.etree.XMLParser(ns_clean=True, recover=True, encoding='utf-8')
	return lxml.etree.fromstring(request.text.encode('utf-8'), parser=parser)

class SDMX_ML(object):
    def __init__(self,SDMXML):
        self.SDMXML = SDMXML
	@property
	def time_series(self):
		raise NotImplementedError("Work in progress")

class SDMX_REST(object):
    def __init__(self,sdmx_url,agencyID):
        self.sdmx_url = sdmx_url
        self.agencyID = agencyID
        self._dataflow = None

    @property
    def dataflow(self):
        if not self._dataflow:
            resource = 'dataflow'
            resourceID = 'all'
            version = 'latest'
            url = (self.sdmx_url+'/'
                               +resource+'/'
						       +self.agencyID+'/'
						       +resourceID+'/'
						       +version)
            self._dataflow = query_rest(url)
        return self._dataflow

    def data_extraction(self, flowRef, key, startperiod, endperiod):
        resource = 'data'
        url = (self.sdmx_url+'/'
					       +resource+'/'
					       +flowRef+'/'
					       +key
					       +'?startperiod='+startperiod
					       +'&endPeriod='+endperiod)
        SDMXML = query_rest(url)
        return SDMXML

eurostat = SDMX_REST("http://www.ec.europa.eu/eurostat/SDMX/diss-web/rest", 'ESTAT')
<|endoftext|>