<commit_before>#!/usr/bin/env python
import collections
import ConfigParser
import getopt
import json
import os
import subprocess
import re
import sys
import tempfile

import circonusapi

config = ConfigParser.SafeConfigParser()
config.read(os.path.expanduser('~/.circusrc'))

def usage():
    print "Usage:"
    print sys.argv[0], "[options] [PATTERN]"
    print
    print "Pattern should be of the form key=pattern, where key is what you"
    print "want to match on (such as target), and pattern is a regular"
    print "expression. If any returned piece of data doesn't have the key"
    print "(e.g. rules don't have targets), then it doesn't match"
    print
    print "  -a -- Specify which account to use"
    print "  -d -- Enable debug mode"
    print "  -e -- Specify endpoints to search (can be used multiple times)"
    print "  -E -- Specify an alternate editor to use (default: $EDITOR)"
    print "  -u -- include underscore entries (e.g. _cid) in json output"

def confirm(text="OK to continue?"):
    response = None
    while response not in ['Y', 'y', 'N', 'n']:
        response = raw_input("%s (y/n) " % text)
    if response in ['Y', 'y']:
        return True
    return False

options = {
    'account': config.get('general', 'default_account', None),
    'debug': False,
    'endpoints': [],
    'editor': os.environ.get('EDITOR', 'vi'),
    'include_underscore': False
}

def parse_options():
    try:
        opts, args = getopt.gnu_getopt(sys.argv[1:], "a:de:E:u?")
    except getopt.GetoptError, err:
        # print help information and exit:
        print str(err) # will print something like "option -a not recognized"
        usage()
        sys.exit(2)

    for o,a in opts:
        if o == '-a':
            options['account'] = a
        if o == '-d':
            options['debug'] = not options['debug']
        if o == '-e':
            options['endpoints'].append(a)
        if o == '-E':
            options['editor'] = a
        if o == '-u':
            options['include_underscore'] = not options['include_underscore']
        if o == '-?':
            usage()
            sys.exit(0)
    return args

def get_api():
    token = config.get('tokens', options['account'], None)
    api = circonusapi.CirconusAPI(token)
    if options['debug']:
        api.debug = True
    return api

def get_circonus_data(api, args):
    if not options['endpoints']:
        # user, account are missing from here TODO - it's probably a good idea
        # to pick only commonly used endpoints by default here, or perhaps
        # make it configurable.
        options['endpoints'] = ['broker', 'check_bundle', 'contact_group',
                'graph', 'rule_set', 'template', 'worksheet']

    # Combined output to be serialized
    data = {}

    for t in options['endpoints']:
        data.update(dict(((i['_cid'], i) for i in api.api_call("GET", t))))

    # Filter based on the pattern
    patterns = []
    for p in args:
        parts = p.split('=', 1)
        if len(parts) != 2:
            print "Invalid pattern: %s" % p
            usage()
            sys.exit(2)
        patterns.append(parts)

    filtered_data = {}
    for i in data:
        for k, p in patterns:
            if k not in data[i]:
                break
            if not re.search(p, data[i][k]):
                break
        else:
            filtered_data[i] = data[i]
    return filtered_data

def edit_file(data):
    tmp = tempfile.mkstemp(suffix='.json')
    fh = os.fdopen(tmp[0], 'w')
    json.dump(data, fh, sort_keys=True, indent=4, separators=(',',': '))
    fh.close()

    ok = False
    while not ok:
        subprocess.call([options['editor'], tmp[1]])

        fh = open(tmp[1])
        try:
            data_new = json.load(fh)
            ok = True
        except ValueError, e:
            print "Error parsing JSON:", e
            if not confirm("Do you want to edit the file again?"):
                sys.exit(1)
        fh.close()

    os.remove(tmp[1])
    return data_new

def calculate_changes(data, data_new):
    changes = []
    for i in data_new:
        if i not in data:
            # Addition
            # We need to make sure the endpoint doesn't contain an ID (just in
            # case the entry contains one)
            changes.append({'action': 'POST', 'data': data_new[i],
                'endpoint': re.sub("(?!^)/.*", "", i)})
        elif data[i] != data_new[i]:
            # Edit
            changes.append({'action': 'PUT', 'data': data_new[i],
                'data_old': data[i], 'endpoint': i})
    for i in data:
        if i not in data_new:
            # Delete
            changes.append({'action': 'DELETE', 'data': data[i], 'endpoint': i})
    return changes

def show_changes(changes):
    pretty_actions = {
        'DELETE': "Delete",
        'POST': "Addition",
        'PUT': "Edit"
    }
    for c in changes:
        print "Action: %s" % pretty_actions[c['action']]
        print "Endpoint: %s" % c['endpoint']
        if c['action'] == 'PUT':
            # We have an edit - show a diff
            print "Diff:"
            old_file = tempfile.mkstemp()
            new_file = tempfile.mkstemp()
            fh = os.fdopen(old_file[0], 'w')
            json.dump(c['data_old'], fh, sort_keys=True,
                    indent=4, separators=(',',': '))
            fh.close()
            fh = os.fdopen(new_file[0], 'w')
            json.dump(c['data'], fh, sort_keys=True,
                    indent=4, separators=(',',': '))
            fh.close()
            subprocess.call(["diff", "-u", old_file[1], new_file[1]])
            os.remove(old_file[1])
            os.remove(new_file[1])
        else:
            print "Data:"
            print json.dumps(c['data'], sort_keys=True,
                    indent=4, separators=(',',': '))

def confirm_changes(changes):
    if not changes:
        print "No Changes. Exiting."
        sys.exit(1)

    counts = collections.Counter((c['action'] for c in changes))

    print "%d additions, %d deletions, %d edits" % (
            counts['POST'], counts['DELETE'], counts['PUT'])
    response = None
    while True:
        response = raw_input("Do you want to proceed? (YyNnSs?) ")
        if response in ['Y', 'y']:
            return True
        if response in ['N', 'n']:
            return False
        if response in ['S', 's']:
            show_changes(changes)
        if response in ['?']:
            print "Y - Proceed"
            print "N - Quit"
            print "S - Show changes"
            print "? - Help"

def make_changes(changes):
    for c in changes:
        print "Making API Call: %s %s ..." % (c['action'], c['endpoint']),
        if c['action'] == 'DELETE':
            # We don't send any data along for deletions
            c['data'] = None
        try:
            api.api_call(c['action'], c['endpoint'], c['data'])
        except circonusapi.CirconusAPIError, e:
            print "Error"
            print "    %s" % e
            continue
        print "Success"

def strip_underscore_keys(data):
    for k in data.keys():
        for l in data[k].keys():
            if l[0] == '_':
                del data[k][l]

if __name__ == '__main__':
    args = parse_options()
    api = get_api()
    data = get_circonus_data(api, args)
    if not options['include_underscore']:
        strip_underscore_keys(data)
    data_new = edit_file(data)
    changes = calculate_changes(data, data_new)
    if confirm_changes(changes):
        make_changes(changes)
    else:
        print "Exiting"
        sys.exit(1)
<commit_msg>Allow duplicate keys when parsing the json<commit_after>#!/usr/bin/env python
import collections
import ConfigParser
import getopt
import json
import os
import subprocess
import re
import sys
import tempfile

import circonusapi

config = ConfigParser.SafeConfigParser()
config.read(os.path.expanduser('~/.circusrc'))

def usage():
    print "Usage:"
    print sys.argv[0], "[options] [PATTERN]"
    print
    print "Pattern should be of the form key=pattern, where key is what you"
    print "want to match on (such as target), and pattern is a regular"
    print "expression. If any returned piece of data doesn't have the key"
    print "(e.g. rules don't have targets), then it doesn't match"
    print
    print "  -a -- Specify which account to use"
    print "  -d -- Enable debug mode"
    print "  -e -- Specify endpoints to search (can be used multiple times)"
    print "  -E -- Specify an alternate editor to use (default: $EDITOR)"
    print "  -u -- include underscore entries (e.g. _cid) in json output"

def confirm(text="OK to continue?"):
    response = None
    while response not in ['Y', 'y', 'N', 'n']:
        response = raw_input("%s (y/n) " % text)
    if response in ['Y', 'y']:
        return True
    return False

options = {
    'account': config.get('general', 'default_account', None),
    'debug': False,
    'endpoints': [],
    'editor': os.environ.get('EDITOR', 'vi'),
    'include_underscore': False
}

def parse_options():
    try:
        opts, args = getopt.gnu_getopt(sys.argv[1:], "a:de:E:u?")
    except getopt.GetoptError, err:
        # print help information and exit:
        print str(err) # will print something like "option -a not recognized"
        usage()
        sys.exit(2)

    for o,a in opts:
        if o == '-a':
            options['account'] = a
        if o == '-d':
            options['debug'] = not options['debug']
        if o == '-e':
            options['endpoints'].append(a)
        if o == '-E':
            options['editor'] = a
        if o == '-u':
            options['include_underscore'] = not options['include_underscore']
        if o == '-?':
            usage()
            sys.exit(0)
    return args

def get_api():
    token = config.get('tokens', options['account'], None)
    api = circonusapi.CirconusAPI(token)
    if options['debug']:
        api.debug = True
    return api

def get_circonus_data(api, args):
    if not options['endpoints']:
        # user, account are missing from here TODO - it's probably a good idea
        # to pick only commonly used endpoints by default here, or perhaps
        # make it configurable.
        options['endpoints'] = ['broker', 'check_bundle', 'contact_group',
                'graph', 'rule_set', 'template', 'worksheet']

    # Combined output to be serialized
    data = {}

    for t in options['endpoints']:
        data.update(dict(((i['_cid'], i) for i in api.api_call("GET", t))))

    # Filter based on the pattern
    patterns = []
    for p in args:
        parts = p.split('=', 1)
        if len(parts) != 2:
            print "Invalid pattern: %s" % p
            usage()
            sys.exit(2)
        patterns.append(parts)

    filtered_data = {}
    for i in data:
        for k, p in patterns:
            if k not in data[i]:
                break
            if not re.search(p, data[i][k]):
                break
        else:
            filtered_data[i] = data[i]
    return filtered_data

def edit_file(data):
    tmp = tempfile.mkstemp(suffix='.json')
    fh = os.fdopen(tmp[0], 'w')
    json.dump(data, fh, sort_keys=True, indent=4, separators=(',',': '))
    fh.close()

    ok = False
    while not ok:
        subprocess.call([options['editor'], tmp[1]])

        fh = open(tmp[1])
        try:
            data_new = json.load(fh,
                    object_pairs_hook=json_pairs_hook_dedup_keys)
            ok = True
        except ValueError, e:
            print "Error parsing JSON:", e
            if not confirm("Do you want to edit the file again?"):
                sys.exit(1)
        fh.close()

    os.remove(tmp[1])
    return data_new

def calculate_changes(data, data_new):
    changes = []
    for i in data_new:
        if i not in data:
            # Addition
            # We need to make sure the endpoint doesn't contain an ID (just in
            # case the entry contains one)
            changes.append({'action': 'POST', 'data': data_new[i],
                'endpoint': re.sub("(?!^)/.*", "", i)})
        elif data[i] != data_new[i]:
            # Edit
            changes.append({'action': 'PUT', 'data': data_new[i],
                'data_old': data[i], 'endpoint': i})
    for i in data:
        if i not in data_new:
            # Delete
            changes.append({'action': 'DELETE', 'data': data[i], 'endpoint': i})
    return changes

def show_changes(changes):
    pretty_actions = {
        'DELETE': "Delete",
        'POST': "Addition",
        'PUT': "Edit"
    }
    for c in changes:
        print "Action: %s" % pretty_actions[c['action']]
        print "Endpoint: %s" % c['endpoint']
        if c['action'] == 'PUT':
            # We have an edit - show a diff
            print "Diff:"
            old_file = tempfile.mkstemp()
            new_file = tempfile.mkstemp()
            fh = os.fdopen(old_file[0], 'w')
            json.dump(c['data_old'], fh, sort_keys=True,
                    indent=4, separators=(',',': '))
            fh.close()
            fh = os.fdopen(new_file[0], 'w')
            json.dump(c['data'], fh, sort_keys=True,
                    indent=4, separators=(',',': '))
            fh.close()
            subprocess.call(["diff", "-u", old_file[1], new_file[1]])
            os.remove(old_file[1])
            os.remove(new_file[1])
        else:
            print "Data:"
            print json.dumps(c['data'], sort_keys=True,
                    indent=4, separators=(',',': '))

def confirm_changes(changes):
    if not changes:
        print "No Changes. Exiting."
        sys.exit(1)

    counts = collections.Counter((c['action'] for c in changes))

    print "%d additions, %d deletions, %d edits" % (
            counts['POST'], counts['DELETE'], counts['PUT'])
    response = None
    while True:
        response = raw_input("Do you want to proceed? (YyNnSs?) ")
        if response in ['Y', 'y']:
            return True
        if response in ['N', 'n']:
            return False
        if response in ['S', 's']:
            show_changes(changes)
        if response in ['?']:
            print "Y - Proceed"
            print "N - Quit"
            print "S - Show changes"
            print "? - Help"

def make_changes(changes):
    for c in changes:
        print "Making API Call: %s %s ..." % (c['action'], c['endpoint']),
        if c['action'] == 'DELETE':
            # We don't send any data along for deletions
            c['data'] = None
        try:
            api.api_call(c['action'], c['endpoint'], c['data'])
        except circonusapi.CirconusAPIError, e:
            print "Error"
            print "    %s" % e
            continue
        print "Success"

def strip_underscore_keys(data):
    for k in data.keys():
        for l in data[k].keys():
            if l[0] == '_':
                del data[k][l]

def json_pairs_hook_dedup_keys(data):
    # json decoder object_pairs_hook that allows duplicate keys, and makes
    # any duplicate keys unique by appending /x1, /x1 and so on to the end.
    # This is used when adding new items via the circonus api, you can just
    # specify /check_bundle multiple times as the endpoint and won't get an
    # error about duplicate keys when decoding the json. Separate code
    # elsewhere automatically strips off the /x1 when selecting the endpoint
    # to use for adding entries.
    d = {}
    ctr = 0
    for k,v in data:
        oldk = k
        while k in d:
            ctr += 1
            k = "%s/x%s" % (oldk, ctr)
        d[k] = v
    return d

if __name__ == '__main__':
    args = parse_options()
    api = get_api()
    data = get_circonus_data(api, args)
    if not options['include_underscore']:
        strip_underscore_keys(data)
    data_new = edit_file(data)
    changes = calculate_changes(data, data_new)
    if confirm_changes(changes):
        make_changes(changes)
    else:
        print "Exiting"
        sys.exit(1)
<|endoftext|>


<commit_before>from threading import Timer, Thread, enumerate
import sqlite3
import time
from bitcoinrpc.authproxy import JSONRPCException

__author__ = 'sammoth'

"""
Every thing in this module pertains to the crediting of orders.
LPCs submit their signed 'open_orders' query through the 'liquidity' server end point.
The server validates each order and determines if they are tier 1 or 2 using the price
at that moment.
The credit function runs each minute and gathers the submitted orders from the database
For each user it calculates what percentage of the total submitted liquidity they have
provided and gives them a 'credit' in the database.
It also uses that data to submit liquidity info back to Nu
"""

def credit(app, rpc, log, start_timer=True):
    """
    This runs every minute and calculates the total liquidity on order (tier 1) and
    each users proportion of it
    :return:
    """
    # Set the timer going again (use the boolean to allow testing)
    if start_timer:
        credit_timer = Timer(60.0, credit,
                             kwargs={'app': app, 'rpc': rpc, 'log': log})
        credit_timer.name = 'credit_timer'
        credit_timer.daemon = True
        credit_timer.start()
    log.info('Starting Credit')
    conn = sqlite3.connect('pool.db')
    db = conn.cursor()
    # Get all the orders from the database.
    # We delete them as soon as we've got them to allow users to begin submitting again
    all_orders = db.execute("SELECT * FROM orders").fetchall()
    db.execute('DELETE FROM orders')
    conn.commit()
    conn.close()
    # get the total amount of liquidity for tier 1 and 2
    total = {'tier_1': get_total_liquidity(app, all_orders, 'tier_1'),
             'tier_2': get_total_liquidity(app, all_orders, 'tier_2')}

    # We've calculated the totals so submit them as liquidity_info
    Thread(target=liquidity_info, kwargs={'rpc': rpc, 'tier_1': total['tier_1'],
                                          'tier_2': total['tier_2'], 'log': log})

    # build the orders into a dictionary of lists based on the user api key
    user_orders = {}
    for order in all_orders:
        if order[1] not in user_orders:
            user_orders[order[1]] = []
        user_orders[order[1]].append(order)

    # set up a list of credited orders to avoid duplicate orders being credited
    credited_orders = []
    for user in user_orders:
        # for each user get all the orders
        orders = user_orders[user]
        # build a dictionary to hold the tier liquidity amounts
        provided_liquidity = {'tier_1': {}, 'tier_2': {}}
        for tier in provided_liquidity:
            for exchange in app.config['exchanges']:
                provided_liquidity[tier][exchange] = {}
                for unit in app.config['{}.units'.format(exchange)]:
                    provided_liquidity[tier][exchange][unit] = {}
                    for side in ['ask', 'bid']:
                        provided_liquidity[tier][exchange][unit][side] = 0.00
        # parse the orders
        for order in orders:
            # Hash the order on order id, amount, side, exchange and unit to avoid
            # duplication
            order_hash = '{}.{}.{}.{}.{}'.format(order[3], order[4], order[5],
                                                 order[6], order[7])
            # check here to see if the order has already been credited
            if order_hash in credited_orders:
                continue
            # add the order to the list
            credited_orders.append(order_hash)

            # add the order amount to the liquidity provided
            provided_liquidity[order[2]][order[6]][order[7]][order[5]] += float(order[4])

        # Calculate the percentage of the tier 1 liquidity that this user provides
        # for each exchange/unit combination
        calculate_rewards(app, 'tier_1', provided_liquidity, total['tier_1'], user)

        # Record for tier 2 orders also to allow for user reports
        # There is no reward for tier 2
        calculate_rewards(app, 'tier_2', provided_liquidity, total['tier_2'], user)
    log.info('Credit finished')
    return

def get_total_liquidity(app, orders, tier):
    """
    Given a list of orders from the database, calculate the total amount of liquidity
    for the given tier
    :param orders:
    :param tier:
    :return:
    """
    # build the liquidity object
    liquidity = {}
    for exchange in app.config['exchanges']:
        if exchange not in liquidity:
            liquidity[exchange] = {}
        for unit in app.config['{}.units'.format(exchange)]:
            if unit not in liquidity[exchange]:
                liquidity[exchange][unit] = {}
            for side in ['ask', 'bid']:
                liquidity[exchange][unit][side] = 0.00
    # parse the orders and update the liquidity object accordingly
    for order in orders:
        # only add orders for the current tier
        if order[2] != tier:
            continue
        # get the order details
        exchange = order[6]
        unit = order[7]
        side = order[5]
        liquidity[exchange][unit][side] += float(order[4])
    return liquidity

def calculate_rewards(app, tier, provided_liquidity, total, user):
    """
    Calculate the rewards for
    :param tier:
    :return:
    """
    credit_time = int(time.time())
    # add one blank line to database to show that credit occurred
    conn = sqlite3.connect('pool.db')
    db = conn.cursor()
    db.execute("UPDATE info set value=? WHERE key=?", (credit_time, 'last_credit_time'))
    conn.commit()
    conn.close()
    for exchange in provided_liquidity[tier]:
        for unit in provided_liquidity[tier][exchange]:
            for side in provided_liquidity[tier][exchange][unit]:
                provided = float(provided_liquidity[tier][exchange][unit][side])
                if provided <= 0.00:
                    continue
                total_l = float(total[exchange][unit][side])
                if total_l <= 0.00:
                    continue
                percentage = (provided / total_l) if total_l > 0.00 else 0.00
                # Use the percentage to calculate the reward for this round
                reward = percentage * app.config['{}.{}.{}.{}'
                                                 '.reward'.format(exchange,
                                                                  unit,
                                                                  side,
                                                                  tier)]
                # save the details to the database
                # set the connection here to keep it open as short as possible
                conn = sqlite3.connect('pool.db')
                db = conn.cursor()
                db.execute("INSERT INTO credits (time,user,exchange,unit,tier,side,"
                           "provided,total,percentage,reward,paid) VALUES  "
                           "(?,?,?,?,?,?,?,?,?,?,?)",
                           (credit_time, user, exchange, unit, tier, side, provided,
                            total_l, (percentage * 100), reward, 0))
                conn.commit()
                conn.close()
    return

def liquidity_info(app, rpc, tier_1, tier_2, log):
    """
    Calculate the current amount of liquidity in tiers 1 and 2 and submit them to Nu
    :return:
    """
    for exchange in tier_1:
        for unit in tier_1[exchange]:
            identifier = "1:{}:{}:{}".format('NBT{}'.format(unit.upper()),
                                             exchange,
                                             app.config['pool.name'])
            try:
                rpc.liquidityinfo('B', tier_1[exchange][unit]['bid'],
                                  tier_1[exchange][unit]['ask'],
                                  app.config['pool.grant_address'], identifier)
            except JSONRPCException as e:
                log.error('Sending tier 1 liquidity info failed: {}'.format(e.message))

    for exchange in tier_2:
        for unit in tier_2[exchange]:
            identifier = "2:{}:{}:{}".format('NBT{}'.format(unit.upper()),
                                             exchange,
                                             app.config['pool.name'])
            try:
                rpc.liquidityinfo('B', tier_2[exchange][unit]['bid'],
                                  tier_2[exchange][unit]['ask'],
                                  app.config['pool.grant_address'], identifier)
            except JSONRPCException as e:
                log.error('Sending tier 2 liquidity info failed: {}'.format(e.message))
<commit_msg>chnage logging text<commit_after>from threading import Timer, Thread, enumerate
import sqlite3
import time
from bitcoinrpc.authproxy import JSONRPCException

__author__ = 'sammoth'

"""
Every thing in this module pertains to the crediting of orders.
LPCs submit their signed 'open_orders' query through the 'liquidity' server end point.
The server validates each order and determines if they are tier 1 or 2 using the price
at that moment.
The credit function runs each minute and gathers the submitted orders from the database
For each user it calculates what percentage of the total submitted liquidity they have
provided and gives them a 'credit' in the database.
It also uses that data to submit liquidity info back to Nu
"""

def credit(app, rpc, log, start_timer=True):
    """
    This runs every minute and calculates the total liquidity on order (tier 1) and
    each users proportion of it
    :return:
    """
    # Set the timer going again (use the boolean to allow testing)
    if start_timer:
        credit_timer = Timer(60.0, credit,
                             kwargs={'app': app, 'rpc': rpc, 'log': log})
        credit_timer.name = 'credit_timer'
        credit_timer.daemon = True
        credit_timer.start()
    log.info('Start Credit')
    conn = sqlite3.connect('pool.db')
    db = conn.cursor()
    # Get all the orders from the database.
    # We delete them as soon as we've got them to allow users to begin submitting again
    all_orders = db.execute("SELECT * FROM orders").fetchall()
    db.execute('DELETE FROM orders')
    conn.commit()
    conn.close()
    # get the total amount of liquidity for tier 1 and 2
    total = {'tier_1': get_total_liquidity(app, all_orders, 'tier_1'),
             'tier_2': get_total_liquidity(app, all_orders, 'tier_2')}

    # We've calculated the totals so submit them as liquidity_info
    Thread(target=liquidity_info, kwargs={'rpc': rpc, 'tier_1': total['tier_1'],
                                          'tier_2': total['tier_2'], 'log': log})

    # build the orders into a dictionary of lists based on the user api key
    user_orders = {}
    for order in all_orders:
        if order[1] not in user_orders:
            user_orders[order[1]] = []
        user_orders[order[1]].append(order)

    # set up a list of credited orders to avoid duplicate orders being credited
    credited_orders = []
    for user in user_orders:
        # for each user get all the orders
        orders = user_orders[user]
        # build a dictionary to hold the tier liquidity amounts
        provided_liquidity = {'tier_1': {}, 'tier_2': {}}
        for tier in provided_liquidity:
            for exchange in app.config['exchanges']:
                provided_liquidity[tier][exchange] = {}
                for unit in app.config['{}.units'.format(exchange)]:
                    provided_liquidity[tier][exchange][unit] = {}
                    for side in ['ask', 'bid']:
                        provided_liquidity[tier][exchange][unit][side] = 0.00
        # parse the orders
        for order in orders:
            # Hash the order on order id, amount, side, exchange and unit to avoid
            # duplication
            order_hash = '{}.{}.{}.{}.{}'.format(order[3], order[4], order[5],
                                                 order[6], order[7])
            # check here to see if the order has already been credited
            if order_hash in credited_orders:
                continue
            # add the order to the list
            credited_orders.append(order_hash)

            # add the order amount to the liquidity provided
            provided_liquidity[order[2]][order[6]][order[7]][order[5]] += float(order[4])

        # Calculate the percentage of the tier 1 liquidity that this user provides
        # for each exchange/unit combination
        calculate_rewards(app, 'tier_1', provided_liquidity, total['tier_1'], user)

        # Record for tier 2 orders also to allow for user reports
        # There is no reward for tier 2
        calculate_rewards(app, 'tier_2', provided_liquidity, total['tier_2'], user)
    log.info('End Credit')
    return

def get_total_liquidity(app, orders, tier):
    """
    Given a list of orders from the database, calculate the total amount of liquidity
    for the given tier
    :param orders:
    :param tier:
    :return:
    """
    # build the liquidity object
    liquidity = {}
    for exchange in app.config['exchanges']:
        if exchange not in liquidity:
            liquidity[exchange] = {}
        for unit in app.config['{}.units'.format(exchange)]:
            if unit not in liquidity[exchange]:
                liquidity[exchange][unit] = {}
            for side in ['ask', 'bid']:
                liquidity[exchange][unit][side] = 0.00
    # parse the orders and update the liquidity object accordingly
    for order in orders:
        # only add orders for the current tier
        if order[2] != tier:
            continue
        # get the order details
        exchange = order[6]
        unit = order[7]
        side = order[5]
        liquidity[exchange][unit][side] += float(order[4])
    return liquidity

def calculate_rewards(app, tier, provided_liquidity, total, user):
    """
    Calculate the rewards for
    :param tier:
    :return:
    """
    credit_time = int(time.time())
    # add one blank line to database to show that credit occurred
    conn = sqlite3.connect('pool.db')
    db = conn.cursor()
    db.execute("UPDATE info set value=? WHERE key=?", (credit_time, 'last_credit_time'))
    conn.commit()
    conn.close()
    for exchange in provided_liquidity[tier]:
        for unit in provided_liquidity[tier][exchange]:
            for side in provided_liquidity[tier][exchange][unit]:
                provided = float(provided_liquidity[tier][exchange][unit][side])
                if provided <= 0.00:
                    continue
                total_l = float(total[exchange][unit][side])
                if total_l <= 0.00:
                    continue
                percentage = (provided / total_l)
                # Use the percentage to calculate the reward for this round
                reward = percentage * app.config['{}.{}.{}.{}'
                                                 '.reward'.format(exchange,
                                                                  unit,
                                                                  side,
                                                                  tier)]
                # save the details to the database
                # set the connection here to keep it open as short as possible
                conn = sqlite3.connect('pool.db')
                db = conn.cursor()
                db.execute("INSERT INTO credits (time,user,exchange,unit,tier,side,"
                           "provided,total,percentage,reward,paid) VALUES  "
                           "(?,?,?,?,?,?,?,?,?,?,?)",
                           (credit_time, user, exchange, unit, tier, side, provided,
                            total_l, (percentage * 100), reward, 0))
                conn.commit()
                conn.close()
    return

def liquidity_info(app, rpc, tier_1, tier_2, log):
    """
    Calculate the current amount of liquidity in tiers 1 and 2 and submit them to Nu
    :return:
    """
    for exchange in tier_1:
        for unit in tier_1[exchange]:
            identifier = "1:{}:{}:{}".format('NBT{}'.format(unit.upper()),
                                             exchange,
                                             app.config['pool.name'])
            try:
                rpc.liquidityinfo('B', tier_1[exchange][unit]['bid'],
                                  tier_1[exchange][unit]['ask'],
                                  app.config['pool.grant_address'], identifier)
            except JSONRPCException as e:
                log.error('Sending tier 1 liquidity info failed: {}'.format(e.message))

    for exchange in tier_2:
        for unit in tier_2[exchange]:
            identifier = "2:{}:{}:{}".format('NBT{}'.format(unit.upper()),
                                             exchange,
                                             app.config['pool.name'])
            try:
                rpc.liquidityinfo('B', tier_2[exchange][unit]['bid'],
                                  tier_2[exchange][unit]['ask'],
                                  app.config['pool.grant_address'], identifier)
            except JSONRPCException as e:
                log.error('Sending tier 2 liquidity info failed: {}'.format(e.message))
<|endoftext|>


<commit_before>from i3pystatus import IntervalModule
from .core.util import round_dict
# requires python-novaclient
from novaclient.v2 import client
import webbrowser

class Openstack_vms(IntervalModule):
    """
    Displays the number of VMs in an openstack cluster in ACTIVE and
    non-ACTIVE states.
    Requires: python-novaclient
    """

    settings = (
        ("auth_url", "OpenStack cluster authentication URL (OS_AUTH_URL)"),
        ("username", "Username for OpenStack authentication (OS_USERNAME)"),
        ("password", "Password for Openstack authentication (OS_PASSWORD)"),
        ("tenant_name", "Tenant/Project name to view (OS_TENANT_NAME)"),
        ("color", "Display color when non-active VMs are =< `threshold` "
            "(default: #00FF00"),
        ("crit_color", "Display color when non-active VMs are => `threshold` "
            "(default: #FF0000"),
        ("threshold", "Set critical indicators when non-active VM pass this "
            "number (default: 0)"),
        ("horizon_url", "When clicked, open this URL in a browser")
    )
    required = ("auth_url", "password", "tenant_name", "username")
    color = "#00FF00"
    crit_color = "#FF0000"
    threshold = 0
    horizon_url = None
    format = "{tenant_name}: {active_servers} active, "\
        "{nonactive_servers} non-active"

    on_leftclick = "openurl"

    def run(self):
        nclient = client.Client(
            self.username,
            self.password,
            self.tenant_name,
            self.auth_url
        )

        active_servers = 0
        nonactive_servers = 0
        server_list = nclient.servers.list()
        for server in server_list:
            if server.status == 'ACTIVE':
                active_servers = active_servers + 1
            else:
                nonactive_servers = nonactive_servers + 1

        if nonactive_servers > self.threshold:
            display_color = self.crit_color
        else:
            display_color = self.color
        cdict = {
            "tenant_name": self.tenant_name,
            "active_servers": active_servers,
            "nonactive_servers": nonactive_servers,
        }

        self.output = {
            "full_text": self.format.format(**cdict),
            "color": display_color
        }

    def openurl(self):
        webbrowser.open_new_tab(self.horizon_url)
<commit_msg>Remove defaults documentation, as they are auto-gen<commit_after>from i3pystatus import IntervalModule
from .core.util import round_dict
# requires python-novaclient
from novaclient.v2 import client
import webbrowser

class Openstack_vms(IntervalModule):
    """
    Displays the number of VMs in an openstack cluster in ACTIVE and
    non-ACTIVE states.
    Requires: python-novaclient
    """

    settings = (
        ("auth_url", "OpenStack cluster authentication URL (OS_AUTH_URL)"),
        ("username", "Username for OpenStack authentication (OS_USERNAME)"),
        ("password", "Password for Openstack authentication (OS_PASSWORD)"),
        ("tenant_name", "Tenant/Project name to view (OS_TENANT_NAME)"),
        ("color", "Display color when non-active VMs are =< `threshold`"),
        ("crit_color", "Display color when non-active VMs are => `threshold`").
        ("threshold", "Set critical indicators when non-active VM pass this "
            "number"),
        ("horizon_url", "When clicked, open this URL in a browser")
    )
    required = ("auth_url", "password", "tenant_name", "username")
    color = "#00FF00"
    crit_color = "#FF0000"
    threshold = 0
    horizon_url = None
    format = "{tenant_name}: {active_servers} active, "\
        "{nonactive_servers} non-active"

    on_leftclick = "openurl"

    def run(self):
        nclient = client.Client(
            self.username,
            self.password,
            self.tenant_name,
            self.auth_url
        )

        active_servers = 0
        nonactive_servers = 0
        server_list = nclient.servers.list()
        for server in server_list:
            if server.status == 'ACTIVE':
                active_servers = active_servers + 1
            else:
                nonactive_servers = nonactive_servers + 1

        if nonactive_servers > self.threshold:
            display_color = self.crit_color
        else:
            display_color = self.color
        cdict = {
            "tenant_name": self.tenant_name,
            "active_servers": active_servers,
            "nonactive_servers": nonactive_servers,
        }

        self.output = {
            "full_text": self.format.format(**cdict),
            "color": display_color
        }

    def openurl(self):
        webbrowser.open_new_tab(self.horizon_url)
<|endoftext|>


<commit_before>''' Autostop facility '''
import copy
import logging
import re

from Tank.Plugins.Aggregator import AggregatorPlugin, AggregateResultListener
from Tank.Plugins.ConsoleOnline import AbstractInfoWidget, ConsoleOnlinePlugin
from tankcore import AbstractPlugin
import tankcore

class AutostopPlugin(AbstractPlugin, AggregateResultListener):
    ''' Plugin that accepts criteria classes and triggers autostop '''
    SECTION = 'autostop'

    def __init__(self, core):
        AbstractPlugin.__init__(self, core)
        self.cause_criteria = None
        self.criterias = []
        self.custom_criterias = []
        self.counting = []
        self.criteria_str = ''

    @staticmethod
    def get_key():
        return __file__

    def get_counting(self):
        ''' get criterias that are activated '''
        return self.counting

    def add_counting(self, obj):
        ''' add criteria that activated '''
        self.counting += [obj]

    def add_criteria_class(self, criteria_class):
        ''' add new criteria class '''
        self.custom_criterias += [criteria_class]

    def get_available_options(self):
        return ["autostop"]

    def configure(self):
        aggregator = self.core.get_plugin_of_type(AggregatorPlugin)
        aggregator.add_result_listener(self)
        self.criteria_str = " ".join(self.get_option("autostop", '').split("\n"))
        self.add_criteria_class(AvgTimeCriteria)
        self.add_criteria_class(NetCodesCriteria)
        self.add_criteria_class(HTTPCodesCriteria)
        self.add_criteria_class(QuantileCriteria)

    def prepare_test(self):
        for criteria_str in self.criteria_str.strip().split(")"):
            if not criteria_str:
                continue
            self.log.debug("Criteria string: %s", criteria_str)
            self.criterias.append(self.__create_criteria(criteria_str))

        self.log.debug("Criteria object: %s", self.criterias)

        try:
            console = self.core.get_plugin_of_type(ConsoleOnlinePlugin)
        except Exception, ex:
            self.log.debug("Console not found: %s", ex)
            console = None

        if console:
            console.add_info_widget(AutostopWidget(self))

    def is_test_finished(self):
        if self.cause_criteria:
            self.log.info("Autostop criteria requested test stop: %s", self.cause_criteria.explain())
            return self.cause_criteria.get_rc()
        else:
            return -1

    def __create_criteria(self, criteria_str):
        ''' instantiate criteria from config string '''
        parsed = criteria_str.split("(")
        type_str = parsed[0].strip().lower()
        parsed[1] = parsed[1].split(")")[0].strip()

        for criteria_class in self.custom_criterias:
            if criteria_class.get_type_string() == type_str:
                return criteria_class(self, parsed[1])
        raise ValueError("Unsupported autostop criteria type: %s" % criteria_str)

    def aggregate_second(self, second_aggregate_data):
        self.counting = []
        if not self.cause_criteria:
            for criteria in self.criterias:
                if criteria.notify(second_aggregate_data):
                    self.log.debug("Autostop criteria requested test stop: %s", criteria)
                    self.cause_criteria = criteria

class AutostopWidget(AbstractInfoWidget):
    ''' widget that displays counting criterias '''

    def __init__(self, sender):
        AbstractInfoWidget.__init__(self)
        self.owner = sender

    def get_index(self):
        return 25

    def render(self, screen):
        res = []
        candidates = self.owner.get_counting()
        for candidate in candidates:
            text, perc = candidate.widget_explain()
            if perc >= 0.95:
                res += [screen.markup.RED_DARK + text + screen.markup.RESET]
            elif perc >= 0.8:
                res += [screen.markup.RED + text + screen.markup.RESET]
            elif perc >= 0.5:
                res += [screen.markup.YELLOW + text + screen.markup.RESET]
            else:
                res += [text]

        if res:
            return "Autostop:\n  " + ("\n  ".join(res))
        else:
            return ''

class AbstractCriteria:
    ''' parent class for all criterias '''
    RC_TIME = 21
    RC_HTTP = 22
    RC_NET = 23

    def __init__(self):
        self.log = logging.getLogger(__name__)
        self.cause_second = None

    def count_matched_codes(self, codes_regex, codes_dict):
        ''' helper to aggregate codes by mask '''
        total = 0
        for code, count in codes_dict.items():
            if codes_regex.match(str(code)):
                total += count
        return total

    def notify(self, aggregate_second):
        ''' notification about aggregate data goes here '''
        raise NotImplementedError("Abstract methods requires overriding")

    def get_rc(self):
        ''' get return code for test '''
        raise NotImplementedError("Abstract methods requires overriding")

    def explain(self):
        ''' long explanation to show after test stop '''
        raise NotImplementedError("Abstract methods requires overriding")

    def widget_explain(self):
        ''' short explanation to display in right panel '''
        return (self.explain(), 0)

    @staticmethod
    def get_type_string():
        ''' returns string that used as config name for criteria '''
        raise NotImplementedError("Abstract methods requires overriding")

class AvgTimeCriteria(AbstractCriteria):
    ''' average response time criteria '''

    @staticmethod
    def get_type_string():
        return 'time'

    def __init__(self, autostop, param_str):
        AbstractCriteria.__init__(self)
        self.seconds_count = 0
        self.rt_limit = tankcore.expand_to_milliseconds(param_str.split(',')[0])
        self.seconds_limit = tankcore.expand_to_seconds(param_str.split(',')[1])
        self.autostop = autostop

    def notify(self, aggregate_second):
        if aggregate_second.overall.avg_response_time > self.rt_limit:
            if not self.seconds_count:
                self.cause_second = aggregate_second

            self.log.debug(self.explain())

            self.seconds_count += 1
            self.autostop.add_counting(self)
            if self.seconds_count >= self.seconds_limit:
                return True
        else:
            self.seconds_count = 0

        return False

    def get_rc(self):
        return self.RC_TIME

    def explain(self):
        items = (self.rt_limit, self.seconds_count, self.cause_second.time)
        return "Average response time higher than %sms for %ss, since %s" % items

    def widget_explain(self):
        items = (self.rt_limit, self.seconds_count, self.seconds_limit)
        return ("Avg Time >%sms for %s/%ss" % items, float(self.seconds_count) / self.seconds_limit)

class HTTPCodesCriteria(AbstractCriteria):
    ''' HTTP codes criteria '''

    @staticmethod
    def get_type_string():
        return 'http'

    def __init__(self, autostop, param_str):
        AbstractCriteria.__init__(self)
        self.seconds_count = 0
        self.codes_mask = param_str.split(',')[0].lower()
        self.codes_regex = re.compile(self.codes_mask.replace("x", '.'))
        self.autostop = autostop

        level_str = param_str.split(',')[1].strip()
        if level_str[-1:] == '%':
            self.level = float(level_str[:-1]) / 100
            self.is_relative = True
        else:
            self.level = int(level_str)
            self.is_relative = False
        self.seconds_limit = tankcore.expand_to_seconds(param_str.split(',')[2])

    def notify(self, aggregate_second):
        matched_responses = self.count_matched_codes(self.codes_regex, aggregate_second.overall.http_codes)
        if self.is_relative:
            if aggregate_second.overall.RPS:
                matched_responses = float(matched_responses) / aggregate_second.overall.RPS
            else:
                matched_responses = 0
        self.log.debug("HTTP codes matching mask %s: %s/%s", self.codes_mask, matched_responses, self.level)

        if matched_responses >= self.level:
            if not self.seconds_count:
                self.cause_second = aggregate_second

            self.log.debug(self.explain())

            self.seconds_count += 1
            self.autostop.add_counting(self)
            if self.seconds_count >= self.seconds_limit:
                return True
        else:
            self.seconds_count = 0

        return False

    def get_rc(self):
        return self.RC_HTTP

    def get_level_str(self):
        ''' format level str '''
        if self.is_relative:
            level_str = str(100 * self.level) + "%"
        else:
            level_str = self.level
        return level_str

    def explain(self):
        items = (self.codes_mask, self.get_level_str(), self.seconds_count, self.cause_second.time)
        return "%s codes count higher than %s for %ss, since %s" % items

    def widget_explain(self):
        items = (self.codes_mask, self.get_level_str(), self.seconds_count, self.seconds_limit)
        return ("HTTP %s>%s for %s/%ss" % items, float(self.seconds_count) / self.seconds_limit)

class NetCodesCriteria(AbstractCriteria):
    ''' Net codes criteria '''

    @staticmethod
    def get_type_string():
        return 'net'

    def __init__(self, autostop, param_str):
        AbstractCriteria.__init__(self)
        self.seconds_count = 0
        self.codes_mask = param_str.split(',')[0].lower()
        self.codes_regex = re.compile(self.codes_mask.replace("x", '.'))
        self.autostop = autostop

        level_str = param_str.split(',')[1].strip()
        if level_str[-1:] == '%':
            self.level = float(level_str[:-1]) / 100
            self.is_relative = True
        else:
            self.level = int(level_str)
            self.is_relative = False
        self.seconds_limit = tankcore.expand_to_seconds(param_str.split(',')[2])

    def notify(self, aggregate_second):
        codes = copy.deepcopy(aggregate_second.overall.net_codes)
        if '0' in codes.keys():
            codes.pop('0')
        matched_responses = self.count_matched_codes(self.codes_regex, codes)
        if self.is_relative:
            if aggregate_second.overall.RPS:
                matched_responses = float(matched_responses) / aggregate_second.overall.RPS
            else:
                matched_responses = 0
        self.log.debug("Net codes matching mask %s: %s/%s", self.codes_mask, matched_responses, self.level)

        if matched_responses >= self.level:
            if not self.seconds_count:
                self.cause_second = aggregate_second

            self.log.debug(self.explain())

            self.seconds_count += 1
            self.autostop.add_counting(self)
            if self.seconds_count >= self.seconds_limit:
                return True
        else:
            self.seconds_count = 0

        return False

    def get_rc(self):
        return self.RC_NET

    def get_level_str(self):
        ''' format level str '''
        if self.is_relative:
            level_str = str(100 * self.level) + "%"
        else:
            level_str = self.level
        return level_str

    def explain(self):
        items = (self.codes_mask, self.get_level_str(), self.seconds_count, self.cause_second.time)
        return "%s net codes count higher than %s for %ss, since %s" % items

    def widget_explain(self):
        items = (self.codes_mask, self.get_level_str(), self.seconds_count, self.seconds_limit)
        return ("Net %s>%s for %s/%ss" % items, float(self.seconds_count) / self.seconds_limit)

class QuantileCriteria(AbstractCriteria):
    ''' quantile criteria '''

    @staticmethod
    def get_type_string():
        return 'quantile'

    def __init__(self, autostop, param_str):
        AbstractCriteria.__init__(self)
        self.seconds_count = 0
        self.quantile = float(param_str.split(',')[0])
        self.rt_limit = tankcore.expand_to_milliseconds(param_str.split(',')[1])
        self.seconds_limit = tankcore.expand_to_seconds(param_str.split(',')[2])
        self.autostop = autostop

    def notify(self, aggregate_second):
        if not (self.quantile in aggregate_second.overall.quantiles.keys()):
            self.log.warning("No quantile %s in %s", self.quantile, aggregate_second.overall.quantiles)
        if self.quantile in aggregate_second.overall.quantiles.keys() \
                and aggregate_second.overall.quantiles[self.quantile] > self.rt_limit:
            if not self.seconds_count:
                self.cause_second = aggregate_second

            self.log.debug(self.explain())

            self.seconds_count += 1
            self.autostop.add_counting(self)
            if self.seconds_count >= self.seconds_limit:
                return True
        else:
            self.seconds_count = 0

        return False

    def get_rc(self):
        return self.RC_TIME

    def explain(self):
        items = (self.quantile, self.rt_limit, self.seconds_count, self.cause_second.time)
        return "Percentile %s higher than %sms for %ss, since %s" % items

    def widget_explain(self):
        items = (self.quantile, self.rt_limit, self.seconds_count, self.seconds_limit)
        return ("%s%% >%sms for %s/%ss" % items, float(self.seconds_count) / self.seconds_limit)

<commit_msg>Cosmetics<commit_after>""" Autostop facility """
import copy
import logging
import re

from Tank.Plugins.Aggregator import AggregatorPlugin, AggregateResultListener
from Tank.Plugins.ConsoleOnline import AbstractInfoWidget, ConsoleOnlinePlugin
from tankcore import AbstractPlugin
import tankcore

class AutostopPlugin(AbstractPlugin, AggregateResultListener):
    """ Plugin that accepts criteria classes and triggers autostop """
    SECTION = 'autostop'

    def __init__(self, core):
        AbstractPlugin.__init__(self, core)
        self.cause_criteria = None
        self.criterias = []
        self.custom_criterias = []
        self.counting = []
        self.criteria_str = ''

    @staticmethod
    def get_key():
        return __file__

    def get_counting(self):
        """ get criterias that are activated """
        return self.counting

    def add_counting(self, obj):
        """ add criteria that activated """
        self.counting += [obj]

    def add_criteria_class(self, criteria_class):
        """ add new criteria class """
        self.custom_criterias += [criteria_class]

    def get_available_options(self):
        return ["autostop"]

    def configure(self):
        aggregator = self.core.get_plugin_of_type(AggregatorPlugin)
        aggregator.add_result_listener(self)
        self.criteria_str = " ".join(self.get_option("autostop", '').split("\n"))
        self.add_criteria_class(AvgTimeCriteria)
        self.add_criteria_class(NetCodesCriteria)
        self.add_criteria_class(HTTPCodesCriteria)
        self.add_criteria_class(QuantileCriteria)

    def prepare_test(self):
        for criteria_str in self.criteria_str.strip().split(")"):
            if not criteria_str:
                continue
            self.log.debug("Criteria string: %s", criteria_str)
            self.criterias.append(self.__create_criteria(criteria_str))

        self.log.debug("Criteria object: %s", self.criterias)

        try:
            console = self.core.get_plugin_of_type(ConsoleOnlinePlugin)
        except Exception, ex:
            self.log.debug("Console not found: %s", ex)
            console = None

        if console:
            console.add_info_widget(AutostopWidget(self))

    def is_test_finished(self):
        if self.cause_criteria:
            self.log.info("Autostop criteria requested test stop: %s", self.cause_criteria.explain())
            return self.cause_criteria.get_rc()
        else:
            return -1

    def __create_criteria(self, criteria_str):
        """ instantiate criteria from config string """
        parsed = criteria_str.split("(")
        type_str = parsed[0].strip().lower()
        parsed[1] = parsed[1].split(")")[0].strip()

        for criteria_class in self.custom_criterias:
            if criteria_class.get_type_string() == type_str:
                return criteria_class(self, parsed[1])
        raise ValueError("Unsupported autostop criteria type: %s" % criteria_str)

    def aggregate_second(self, second_aggregate_data):
        self.counting = []
        if not self.cause_criteria:
            for criteria in self.criterias:
                if criteria.notify(second_aggregate_data):
                    self.log.debug("Autostop criteria requested test stop: %s", criteria)
                    self.cause_criteria = criteria

class AutostopWidget(AbstractInfoWidget):
    """ widget that displays counting criterias """

    def __init__(self, sender):
        AbstractInfoWidget.__init__(self)
        self.owner = sender

    def get_index(self):
        return 25

    def render(self, screen):
        res = []
        candidates = self.owner.get_counting()
        for candidate in candidates:
            text, perc = candidate.widget_explain()
            if perc >= 0.95:
                res += [screen.markup.RED_DARK + text + screen.markup.RESET]
            elif perc >= 0.8:
                res += [screen.markup.RED + text + screen.markup.RESET]
            elif perc >= 0.5:
                res += [screen.markup.YELLOW + text + screen.markup.RESET]
            else:
                res += [text]

        if res:
            return "Autostop:\n  " + ("\n  ".join(res))
        else:
            return ''

class AbstractCriteria:
    """ parent class for all criterias """
    RC_TIME = 21
    RC_HTTP = 22
    RC_NET = 23

    def __init__(self):
        self.log = logging.getLogger(__name__)
        self.cause_second = None

    @staticmethod
    def count_matched_codes(codes_regex, codes_dict):
        """ helper to aggregate codes by mask """
        total = 0
        for code, count in codes_dict.items():
            if codes_regex.match(str(code)):
                total += count
        return total

    def notify(self, aggregate_second):
        """ notification about aggregate data goes here """
        raise NotImplementedError("Abstract methods requires overriding")

    def get_rc(self):
        """ get return code for test """
        raise NotImplementedError("Abstract methods requires overriding")

    def explain(self):
        """ long explanation to show after test stop """
        raise NotImplementedError("Abstract methods requires overriding")

    def widget_explain(self):
        """ short explanation to display in right panel """
        return self.explain(), 0

    @staticmethod
    def get_type_string():
        """ returns string that used as config name for criteria """
        raise NotImplementedError("Abstract methods requires overriding")

class AvgTimeCriteria(AbstractCriteria):
    """ average response time criteria """

    @staticmethod
    def get_type_string():
        return 'time'

    def __init__(self, autostop, param_str):
        AbstractCriteria.__init__(self)
        self.seconds_count = 0
        self.rt_limit = tankcore.expand_to_milliseconds(param_str.split(',')[0])
        self.seconds_limit = tankcore.expand_to_seconds(param_str.split(',')[1])
        self.autostop = autostop

    def notify(self, aggregate_second):
        if aggregate_second.overall.avg_response_time > self.rt_limit:
            if not self.seconds_count:
                self.cause_second = aggregate_second

            self.log.debug(self.explain())

            self.seconds_count += 1
            self.autostop.add_counting(self)
            if self.seconds_count >= self.seconds_limit:
                return True
        else:
            self.seconds_count = 0

        return False

    def get_rc(self):
        return self.RC_TIME

    def explain(self):
        items = (self.rt_limit, self.seconds_count, self.cause_second.time)
        return "Average response time higher than %sms for %ss, since %s" % items

    def widget_explain(self):
        items = (self.rt_limit, self.seconds_count, self.seconds_limit)
        return "Avg Time >%sms for %s/%ss" % items, float(self.seconds_count) / self.seconds_limit

class HTTPCodesCriteria(AbstractCriteria):
    """ HTTP codes criteria """

    @staticmethod
    def get_type_string():
        return 'http'

    def __init__(self, autostop, param_str):
        AbstractCriteria.__init__(self)
        self.seconds_count = 0
        self.codes_mask = param_str.split(',')[0].lower()
        self.codes_regex = re.compile(self.codes_mask.replace("x", '.'))
        self.autostop = autostop

        level_str = param_str.split(',')[1].strip()
        if level_str[-1:] == '%':
            self.level = float(level_str[:-1]) / 100
            self.is_relative = True
        else:
            self.level = int(level_str)
            self.is_relative = False
        self.seconds_limit = tankcore.expand_to_seconds(param_str.split(',')[2])

    def notify(self, aggregate_second):
        matched_responses = self.count_matched_codes(self.codes_regex, aggregate_second.overall.http_codes)
        if self.is_relative:
            if aggregate_second.overall.RPS:
                matched_responses = float(matched_responses) / aggregate_second.overall.RPS
            else:
                matched_responses = 0
        self.log.debug("HTTP codes matching mask %s: %s/%s", self.codes_mask, matched_responses, self.level)

        if matched_responses >= self.level:
            if not self.seconds_count:
                self.cause_second = aggregate_second

            self.log.debug(self.explain())

            self.seconds_count += 1
            self.autostop.add_counting(self)
            if self.seconds_count >= self.seconds_limit:
                return True
        else:
            self.seconds_count = 0

        return False

    def get_rc(self):
        return self.RC_HTTP

    def get_level_str(self):
        """ format level str """
        if self.is_relative:
            level_str = str(100 * self.level) + "%"
        else:
            level_str = self.level
        return level_str

    def explain(self):
        items = (self.codes_mask, self.get_level_str(), self.seconds_count, self.cause_second.time)
        return "%s codes count higher than %s for %ss, since %s" % items

    def widget_explain(self):
        items = (self.codes_mask, self.get_level_str(), self.seconds_count, self.seconds_limit)
        return "HTTP %s>%s for %s/%ss" % items, float(self.seconds_count) / self.seconds_limit

class NetCodesCriteria(AbstractCriteria):
    """ Net codes criteria """

    @staticmethod
    def get_type_string():
        return 'net'

    def __init__(self, autostop, param_str):
        AbstractCriteria.__init__(self)
        self.seconds_count = 0
        self.codes_mask = param_str.split(',')[0].lower()
        self.codes_regex = re.compile(self.codes_mask.replace("x", '.'))
        self.autostop = autostop

        level_str = param_str.split(',')[1].strip()
        if level_str[-1:] == '%':
            self.level = float(level_str[:-1]) / 100
            self.is_relative = True
        else:
            self.level = int(level_str)
            self.is_relative = False
        self.seconds_limit = tankcore.expand_to_seconds(param_str.split(',')[2])

    def notify(self, aggregate_second):
        codes = copy.deepcopy(aggregate_second.overall.net_codes)
        if '0' in codes.keys():
            codes.pop('0')
        matched_responses = self.count_matched_codes(self.codes_regex, codes)
        if self.is_relative:
            if aggregate_second.overall.RPS:
                matched_responses = float(matched_responses) / aggregate_second.overall.RPS
            else:
                matched_responses = 0
        self.log.debug("Net codes matching mask %s: %s/%s", self.codes_mask, matched_responses, self.level)

        if matched_responses >= self.level:
            if not self.seconds_count:
                self.cause_second = aggregate_second

            self.log.debug(self.explain())

            self.seconds_count += 1
            self.autostop.add_counting(self)
            if self.seconds_count >= self.seconds_limit:
                return True
        else:
            self.seconds_count = 0

        return False

    def get_rc(self):
        return self.RC_NET

    def get_level_str(self):
        """ format level str """
        if self.is_relative:
            level_str = str(100 * self.level) + "%"
        else:
            level_str = self.level
        return level_str

    def explain(self):
        items = (self.codes_mask, self.get_level_str(), self.seconds_count, self.cause_second.time)
        return "%s net codes count higher than %s for %ss, since %s" % items

    def widget_explain(self):
        items = (self.codes_mask, self.get_level_str(), self.seconds_count, self.seconds_limit)
        return "Net %s>%s for %s/%ss" % items, float(self.seconds_count) / self.seconds_limit

class QuantileCriteria(AbstractCriteria):
    """ quantile criteria """

    @staticmethod
    def get_type_string():
        return 'quantile'

    def __init__(self, autostop, param_str):
        AbstractCriteria.__init__(self)
        self.seconds_count = 0
        self.quantile = float(param_str.split(',')[0])
        self.rt_limit = tankcore.expand_to_milliseconds(param_str.split(',')[1])
        self.seconds_limit = tankcore.expand_to_seconds(param_str.split(',')[2])
        self.autostop = autostop

    def notify(self, aggregate_second):
        if not (self.quantile in aggregate_second.overall.quantiles.keys()):
            self.log.warning("No quantile %s in %s", self.quantile, aggregate_second.overall.quantiles)
        if self.quantile in aggregate_second.overall.quantiles.keys() \
                and aggregate_second.overall.quantiles[self.quantile] > self.rt_limit:
            if not self.seconds_count:
                self.cause_second = aggregate_second

            self.log.debug(self.explain())

            self.seconds_count += 1
            self.autostop.add_counting(self)
            if self.seconds_count >= self.seconds_limit:
                return True
        else:
            self.seconds_count = 0

        return False

    def get_rc(self):
        return self.RC_TIME

    def explain(self):
        items = (self.quantile, self.rt_limit, self.seconds_count, self.cause_second.time)
        return "Percentile %s higher than %sms for %ss, since %s" % items

    def widget_explain(self):
        items = (self.quantile, self.rt_limit, self.seconds_count, self.seconds_limit)
        return "%s%% >%sms for %s/%ss" % items, float(self.seconds_count) / self.seconds_limit
<|endoftext|>


<commit_before># Copyright (c) 2015, NVIDIA CORPORATION.  All rights reserved.

import os
import shutil
import platform
import ConfigParser
from collections import OrderedDict

import digits

class ConfigFile(object):
    """
    Handles IO on a config file
    """
    config_section = 'DIGITS'

    def __init__(self, filename):
        """
        Doesn't make a fuss if the file doesn't exist
        Use exists() to check
        """
        self._filename = filename
        self._options = OrderedDict()
        self.load()
        self._dirty = False

    def __str__(self):
        s = ''
        for item in self._options.iteritems():
            s += '%15s = %s\n' % item
        return s

    def filename(self):
        return self._filename

    def exists(self):
        """
        Returns True if the file exists
        """
        return os.path.isfile(self._filename)

    def can_read(self):
        """
        Returns True if the file can be read
        """
        return self.exists() and os.access(self._filename, os.R_OK)

    def can_write(self):
        """
        Returns True if the file can be written
        """
        if os.path.isfile(self._filename):
            return os.access(self._filename, os.W_OK)
        else:
            return os.access(
                    os.path.dirname(self._filename),
                    os.W_OK)

    def load(self):
        """
        Load options from the file
        Overwrites any values in self._options
        Returns True if the file loaded successfully
        """
        if not self.exists():
            return False
        cfg = ConfigParser.SafeConfigParser()
        cfg.read(self._filename)
        if not cfg.has_section(self.config_section):
            raise ValueError('expected section "%s" in config file at "%s"' % (
                self.config_section, self._filename))

        for key, val in cfg.items(self.config_section):
            self._options[key] = val
        return True

    def get(self, name):
        """
        Get a config option by name
        """
        if name in self._options:
            return self._options[name]
        else:
            return None

    def set(self, name, value):
        """
        Set a config option by name
        """
        if value is None:
            if name in self._options:
                del self._options[name]
                self._dirty = True
        else:
            if not (name in self._options and self._options[name] == value):
                self._dirty = True
            self._options[name] = value

    def dirty(self):
        """
        Returns True if there are changes to be written to disk
        """
        return self._dirty

    def save(self):
        """
        Save config file to disk
        """
        cfg = ConfigParser.SafeConfigParser()
        cfg.add_section(self.config_section)
        for name, value in self._options.iteritems():
            cfg.set(self.config_section, name, value)
        with open(self._filename, 'w') as outfile:
            cfg.write(outfile)

class SystemConfigFile(ConfigFile):
    def __init__(self):
        if platform.system() in ['Linux','Darwin']:
            filename = '/etc/digits.cfg'
        else:
            filename = None
        super(SystemConfigFile, self).__init__(filename)

class UserConfigFile(ConfigFile):
    def __init__(self):
        if 'HOME' in os.environ:
            filename = os.path.join(os.environ['HOME'], '.digits.cfg')
            old_filename = os.path.join(os.environ['HOME'], '.digits', 'digits.cfg')
            if not os.path.exists(filename) and os.path.exists(old_filename):
                try:
                    shutil.copyfile(old_filename, filename)
                    print 'Copied file at "%s" to "%s".' % (old_filename, filename)
                except (IOError, OSError):
                    pass
        else:
            filename = None
        super(UserConfigFile, self).__init__(filename)

class InstanceConfigFile(ConfigFile):
    def __init__(self):
        filename = os.path.join(os.path.dirname(digits.__file__), 'digits.cfg')
        super(InstanceConfigFile, self).__init__(filename)

<commit_msg>Don't carry over configuration from older versions<commit_after># Copyright (c) 2015, NVIDIA CORPORATION.  All rights reserved.

import os
import shutil
import platform
import ConfigParser
from collections import OrderedDict

import digits

class ConfigFile(object):
    """
    Handles IO on a config file
    """
    config_section = 'DIGITS'

    def __init__(self, filename):
        """
        Doesn't make a fuss if the file doesn't exist
        Use exists() to check
        """
        self._filename = filename
        self._options = OrderedDict()
        self.load()
        self._dirty = False

    def __str__(self):
        s = ''
        for item in self._options.iteritems():
            s += '%15s = %s\n' % item
        return s

    def filename(self):
        return self._filename

    def exists(self):
        """
        Returns True if the file exists
        """
        return os.path.isfile(self._filename)

    def can_read(self):
        """
        Returns True if the file can be read
        """
        return self.exists() and os.access(self._filename, os.R_OK)

    def can_write(self):
        """
        Returns True if the file can be written
        """
        if os.path.isfile(self._filename):
            return os.access(self._filename, os.W_OK)
        else:
            return os.access(
                    os.path.dirname(self._filename),
                    os.W_OK)

    def load(self):
        """
        Load options from the file
        Overwrites any values in self._options
        Returns True if the file loaded successfully
        """
        if not self.exists():
            return False
        cfg = ConfigParser.SafeConfigParser()
        cfg.read(self._filename)
        if not cfg.has_section(self.config_section):
            raise ValueError('expected section "%s" in config file at "%s"' % (
                self.config_section, self._filename))

        for key, val in cfg.items(self.config_section):
            self._options[key] = val
        return True

    def get(self, name):
        """
        Get a config option by name
        """
        if name in self._options:
            return self._options[name]
        else:
            return None

    def set(self, name, value):
        """
        Set a config option by name
        """
        if value is None:
            if name in self._options:
                del self._options[name]
                self._dirty = True
        else:
            if not (name in self._options and self._options[name] == value):
                self._dirty = True
            self._options[name] = value

    def dirty(self):
        """
        Returns True if there are changes to be written to disk
        """
        return self._dirty

    def save(self):
        """
        Save config file to disk
        """
        cfg = ConfigParser.SafeConfigParser()
        cfg.add_section(self.config_section)
        for name, value in self._options.iteritems():
            cfg.set(self.config_section, name, value)
        with open(self._filename, 'w') as outfile:
            cfg.write(outfile)

class SystemConfigFile(ConfigFile):
    def __init__(self):
        if platform.system() in ['Linux','Darwin']:
            filename = '/etc/digits.cfg'
        else:
            filename = None
        super(SystemConfigFile, self).__init__(filename)

class UserConfigFile(ConfigFile):
    def __init__(self):
        if 'HOME' in os.environ:
            filename = os.path.join(os.environ['HOME'], '.digits.cfg')
        else:
            filename = None
        super(UserConfigFile, self).__init__(filename)

class InstanceConfigFile(ConfigFile):
    def __init__(self):
        filename = os.path.join(os.path.dirname(digits.__file__), 'digits.cfg')
        super(InstanceConfigFile, self).__init__(filename)

<|endoftext|>


<commit_before>"""test for wqflask/show_trait/test_show_trait.py"""
import unittest
import pytest
from unittest import mock
from wqflask.show_trait.show_trait import check_if_attr_exists
from wqflask.show_trait.show_trait import get_ncbi_summary
from wqflask.show_trait.show_trait import has_num_cases
from wqflask.show_trait.show_trait import get_table_widths
from wqflask.show_trait.show_trait import get_categorical_variables
from wqflask.show_trait.show_trait import get_trait_units
from wqflask.show_trait.show_trait import get_nearest_marker
from wqflask.show_trait.show_trait import get_genotype_scales
from wqflask.show_trait.show_trait import get_scales_from_genofile

class TraitObject:
    def __init__(self, obj):
        self.distinct_values = []
        self.id = ""
        for key, value in obj.items():
            setattr(self, key, value)
            self.id += str(value)

@pytest.mark.parametrize(
    ('trait', 'id_type', 'expected'),
    (
        (TraitObject({"id_type": "id"}), "id_type", True),
        (TraitObject({"sample_name": ['samp1']}), "id_type", False),
        (TraitObject({"sample": ""}), "sample", False),
        (TraitObject({"group": None}), "group", False),
        (TraitObject({}), "any", False)
    ),
)
def test_check_if_attr_exists(trait, id_type, expected):
    """"test check_if_attr_exists"""
    assert check_if_attr_exists(trait, id_type) == expected

def test_get_ncbi_summary_request(mocker):
    trait = TraitObject({"geneid": "id"})
    mocker.patch("wqflask.show_trait.show_trait.check_if_attr_exists",
                 return_value=True)
    mock_get = mocker.patch(
        "wqflask.show_trait.show_trait.requests.get",
        return_value=TraitObject({"content": """{
          "result":{
            "id":{
              "summary":"this is a summary of the geneid"
            }
          }
        }
        """}))
    assert get_ncbi_summary(trait) == "this is a summary of the geneid"
    mock_get.assert_called_once_with(
        "http://eutils.ncbi.nlm.nih.gov/entrez/"
        "eutils/esummary.fcgi?db=gene&id="
        f"{trait.geneid}&retmode=json"
    )
    mock_get.side_effect = Exception("an error occurred")
    assert get_ncbi_summary(trait) == None

class TestTraits(unittest.TestCase):
    def test_hash_num_cases_is_probeset(self):
        """test for hash num_cases with dataset.type set to Probeset"""
        create_dataset = TraitObject({"type": "ProbeSet"})
        create_trait = TraitObject({"dataset": create_dataset})
        self.assertFalse(has_num_cases(create_trait))

    def test_hash_num_cases_no_probeset(self):
        """test for hash num cases with dataset.type not Probeset"""
        create_dataset = TraitObject({"type": "Temp"})
        construct_data = {
            "nm1": TraitObject({"num_cases": False}),
            "nm2": TraitObject({"num_cases": True}),
            "nm3": TraitObject({"num_cases": False})
        }
        construct_data2 = {
            "nm1": TraitObject({"num_cases": False}),
            "nm2": TraitObject({"num_cases": False}),
            "nm3": TraitObject({"num_cases": False})
        }
        create_trait = TraitObject(
            {"dataset": create_dataset, "data": construct_data})
        create_trait2 = TraitObject(
            {"dataset": create_dataset, "data": construct_data2})

        results = has_num_cases(create_trait)
        self.assertTrue(has_num_cases(create_trait))
        self.assertFalse(has_num_cases(create_trait2))

    def test_get_table_widths(self):
        """test for getting table widths"""
        sample_groups = [TraitObject({'se_exists': True, "attributes": ["attr1", "attr2", "attr3"]}
                                     ), TraitObject(
            {"se_exists": False, "attributes": ["at1", "at2"]
             })]

        results_with_numcase = get_table_widths(sample_groups, True)
        result_no_numcase = get_table_widths(sample_groups, False)

        results_one_sample = get_table_widths(
            [TraitObject({"se_exists": True, "attributes": []})], True)
        expected_with_numcase = (450, "750px")
        expected_no_numcase = (450, "670px")
        expected_one_sample = (250, "540px")
        self.assertEqual(results_with_numcase, expected_with_numcase)
        self.assertEqual(result_no_numcase, expected_no_numcase)
        self.assertEqual(results_one_sample,
                         expected_one_sample)

    def test_get_categorical_variables_no_sample_attributes(self):
        """test for getting categorical variable names with no samples"""
        trait = TraitObject({})
        sample_list = TraitObject({"se_exists": True, "attributes": []})
        self.assertEqual(get_categorical_variables(trait, sample_list), [])

    def test_get_categorical_variables_with_sample_attributes(self):
        """test for getting categorical variable names with no samples"""
        this_trait = TraitObject({"data": {
            "Gene1": TraitObject({"extra_attributes": {"ex1": "ex1value"}}),
            "Gene2": TraitObject({"extra_attributes": {"ex2": "ex2value"}}),
            "Gene3": TraitObject({"extra_attributes": {"ex3": "ex3value"}})
        }})
        sample_list = TraitObject({"attributes": {
            "sample_attribute_1": TraitObject({"name": "ex1"}),
            "sample_attribute_2": TraitObject({"name": "ex2"}),
            "sample_attribute_3": TraitObject({"name": "ex3"}),
            "sample_attribute_4": TraitObject({"name": "not_in_extra_attributes"})
        }})
        results = get_categorical_variables(this_trait, sample_list)
        self.assertEqual(
            ["ex1", "ex2", "ex3", "not_in_extra_attributes"], results)

    def test_get_trait_units(self):
        """test for getting trait units"""
        trait = TraitObject(
            {"description_fmt": "[this is a description] another test [N/A]"})
        trait_no_unit_type = TraitObject({"description_fmt": ""})
        results = get_trait_units(trait)
        results_no_unit = get_trait_units(trait_no_unit_type)
        self.assertEqual(results, "this is a descriptionN/A")
        self.assertEqual(results_no_unit, "Value")

    @mock.patch("wqflask.show_trait.show_trait.g")
    def test_get_nearest_marker(self, mock_db):
        """test for getting nearest marker with non-empty db"""

        mock_db.db.execute.return_value.fetchall.return_value = [
            ["Geno1", "Geno2"], ["Geno3"]]

        trait = TraitObject({"locus_chr": "test_chr", "locus_mb": "test_mb"})
        group_name = TraitObject({"name": "group_name"})
        this_db = TraitObject({"group": group_name})
        results_with_item_db = get_nearest_marker(trait, this_db)
        called_with_value = """SELECT Geno.Name
               FROM Geno, GenoXRef, GenoFreeze
               WHERE Geno.Chr = 'test_chr' AND
                     GenoXRef.GenoId = Geno.Id AND
                     GenoFreeze.Id = GenoXRef.GenoFreezeId AND
                     GenoFreeze.Name = 'group_nameGeno'
               ORDER BY ABS( Geno.Mb - test_mb) LIMIT 1"""

        mock_db.db.execute.assert_called_with(called_with_value)

        self.assertEqual(results_with_item_db, "Geno1")

    @mock.patch("wqflask.show_trait.show_trait.g")
    def test_get_nearest_marker_empty_db(self, mock_db):
        """test for getting nearest marker with empty db"""
        mock_db.db.execute.return_value.fetchall.return_value = []
        trait = TraitObject({"locus_chr": "test_chr", "locus_mb": "test_mb"})
        group_name = TraitObject({"name": "group_name"})
        this_db = TraitObject({"group": group_name})
        results_empty_db = get_nearest_marker(trait, this_db)
        mock_db.db.execute.assert_called_once()
        self.assertEqual(results_empty_db, "")

    @mock.patch("wqflask.show_trait.show_trait.get_scales_from_genofile")
    def test_get_genotype_scales_with_genofile_is_list(self, mock_get_scales):
        """test for getting genotype scales with genofile as list """
        # where genofile is instance of list
        genofiles_list = [{"filename": "file1", "location": "~/data/files/f1"},
                          {"filename": "file2", "location": "~/data/files/f2"},
                          {"filename": "file3", "location": "~/data/files/f3"}]

        mock_get_scales.side_effect = [[["morgan", "cM"]],
                                       [["morgan", "cM"]],
                                       [["physic", "Mb"]]]

        results = get_genotype_scales(genofiles_list)
        expected_results = {
            "~/data/files/f1": [["morgan", "cM"]],
            "~/data/files/f2": [["morgan", "cM"]],
            "~/data/files/f3": [["physic", "Mb"]]
        }

        multiple_calls = [mock.call('~/data/files/f1'), mock.call('~/data/files/f2'),
                          mock.call('~/data/files/f3')]
        mock_get_scales.assert_has_calls(multiple_calls)
        self.assertEqual(results, expected_results)

    @mock.patch("wqflask.show_trait.show_trait.get_scales_from_genofile")
    def test_genotype_scales_with_genofile_other(self, mock_get_scales):
        """test for getting genotype scales with genofile as a string"""
        file_location = "~/another_file_location"
        mock_get_scales.return_value = [["physic", "Mb"]]
        expected_results = {f"{file_location}": [["physic", "Mb"]]}
        self.assertEqual(get_genotype_scales(file_location), expected_results)
        mock_get_scales.assert_called_once_with(file_location)

    @mock.patch("wqflask.show_trait.show_trait.locate_ignore_error")
    def test_get_scales_from_genofile_found(self, mock_ignore_location):
        """"add test for get scales from genofile where file is found"""
        mock_ignore_location.return_value = True
        geno_file = """
                #sample line    with no  @scales:other\n
                #sample line     @scales and :separated   by semicolon\n
                This attempts    to check whether\n
                """

        geno_file_string = "@line start with  @ and has @scale:morgan"

        file_location = "~/data/file.geno"

        mock_open_geno_file = mock.mock_open(read_data=geno_file)
        with mock.patch("builtins.open", mock_open_geno_file):
            results = get_scales_from_genofile(file_location)
            self.assertEqual(results, [["morgan", "cM"]])

        mock_open_string = mock.mock_open(read_data=geno_file_string)

        with mock.patch("builtins.open", mock_open_string):
            result2 = get_scales_from_genofile(file_location)
            self.assertEqual([['morgan', 'cM']], result2)

    @mock.patch("wqflask.show_trait.show_trait.locate_ignore_error")
    def test_get_scales_from_genofile_not_found(self, mock_location_ignore):
        mock_location_ignore.return_value = False

        expected_results = [["physic", "Mb"]]
        results = get_scales_from_genofile("~/data/file")
        mock_location_ignore.assert_called_once_with("~/data/file", "genotype")
        self.assertEqual(results, expected_results)
<commit_msg>Adjust expected table width<commit_after>"""test for wqflask/show_trait/test_show_trait.py"""
import unittest
import pytest
from unittest import mock
from wqflask.show_trait.show_trait import check_if_attr_exists
from wqflask.show_trait.show_trait import get_ncbi_summary
from wqflask.show_trait.show_trait import has_num_cases
from wqflask.show_trait.show_trait import get_table_widths
from wqflask.show_trait.show_trait import get_categorical_variables
from wqflask.show_trait.show_trait import get_trait_units
from wqflask.show_trait.show_trait import get_nearest_marker
from wqflask.show_trait.show_trait import get_genotype_scales
from wqflask.show_trait.show_trait import get_scales_from_genofile

class TraitObject:
    def __init__(self, obj):
        self.distinct_values = []
        self.id = ""
        for key, value in obj.items():
            setattr(self, key, value)
            self.id += str(value)

@pytest.mark.parametrize(
    ('trait', 'id_type', 'expected'),
    (
        (TraitObject({"id_type": "id"}), "id_type", True),
        (TraitObject({"sample_name": ['samp1']}), "id_type", False),
        (TraitObject({"sample": ""}), "sample", False),
        (TraitObject({"group": None}), "group", False),
        (TraitObject({}), "any", False)
    ),
)
def test_check_if_attr_exists(trait, id_type, expected):
    """"test check_if_attr_exists"""
    assert check_if_attr_exists(trait, id_type) == expected

def test_get_ncbi_summary_request(mocker):
    trait = TraitObject({"geneid": "id"})
    mocker.patch("wqflask.show_trait.show_trait.check_if_attr_exists",
                 return_value=True)
    mock_get = mocker.patch(
        "wqflask.show_trait.show_trait.requests.get",
        return_value=TraitObject({"content": """{
          "result":{
            "id":{
              "summary":"this is a summary of the geneid"
            }
          }
        }
        """}))
    assert get_ncbi_summary(trait) == "this is a summary of the geneid"
    mock_get.assert_called_once_with(
        "http://eutils.ncbi.nlm.nih.gov/entrez/"
        "eutils/esummary.fcgi?db=gene&id="
        f"{trait.geneid}&retmode=json"
    )
    mock_get.side_effect = Exception("an error occurred")
    assert get_ncbi_summary(trait) == None

class TestTraits(unittest.TestCase):
    def test_hash_num_cases_is_probeset(self):
        """test for hash num_cases with dataset.type set to Probeset"""
        create_dataset = TraitObject({"type": "ProbeSet"})
        create_trait = TraitObject({"dataset": create_dataset})
        self.assertFalse(has_num_cases(create_trait))

    def test_hash_num_cases_no_probeset(self):
        """test for hash num cases with dataset.type not Probeset"""
        create_dataset = TraitObject({"type": "Temp"})
        construct_data = {
            "nm1": TraitObject({"num_cases": False}),
            "nm2": TraitObject({"num_cases": True}),
            "nm3": TraitObject({"num_cases": False})
        }
        construct_data2 = {
            "nm1": TraitObject({"num_cases": False}),
            "nm2": TraitObject({"num_cases": False}),
            "nm3": TraitObject({"num_cases": False})
        }
        create_trait = TraitObject(
            {"dataset": create_dataset, "data": construct_data})
        create_trait2 = TraitObject(
            {"dataset": create_dataset, "data": construct_data2})

        results = has_num_cases(create_trait)
        self.assertTrue(has_num_cases(create_trait))
        self.assertFalse(has_num_cases(create_trait2))

    def test_get_table_widths(self):
        """test for getting table widths"""
        sample_groups = [TraitObject({'se_exists': True, "attributes": ["attr1", "attr2", "attr3"]}
                                     ), TraitObject(
            {"se_exists": False, "attributes": ["at1", "at2"]
             })]

        results_with_numcase = get_table_widths(sample_groups, True)
        result_no_numcase = get_table_widths(sample_groups, False)

        results_one_sample = get_table_widths(
            [TraitObject({"se_exists": True, "attributes": []})], True)
        expected_with_numcase = (450, 645)
        expected_no_numcase = (450, 644)
        expected_one_sample = (250, 381)
        self.assertEqual(results_with_numcase, expected_with_numcase)
        self.assertEqual(result_no_numcase, expected_no_numcase)
        self.assertEqual(results_one_sample,
                         expected_one_sample)

    def test_get_categorical_variables_no_sample_attributes(self):
        """test for getting categorical variable names with no samples"""
        trait = TraitObject({})
        sample_list = TraitObject({"se_exists": True, "attributes": []})
        self.assertEqual(get_categorical_variables(trait, sample_list), [])

    def test_get_categorical_variables_with_sample_attributes(self):
        """test for getting categorical variable names with no samples"""
        this_trait = TraitObject({"data": {
            "Gene1": TraitObject({"extra_attributes": {"ex1": "ex1value"}}),
            "Gene2": TraitObject({"extra_attributes": {"ex2": "ex2value"}}),
            "Gene3": TraitObject({"extra_attributes": {"ex3": "ex3value"}})
        }})
        sample_list = TraitObject({"attributes": {
            "sample_attribute_1": TraitObject({"name": "ex1"}),
            "sample_attribute_2": TraitObject({"name": "ex2"}),
            "sample_attribute_3": TraitObject({"name": "ex3"}),
            "sample_attribute_4": TraitObject({"name": "not_in_extra_attributes"})
        }})
        results = get_categorical_variables(this_trait, sample_list)
        self.assertEqual(
            ["ex1", "ex2", "ex3", "not_in_extra_attributes"], results)

    def test_get_trait_units(self):
        """test for getting trait units"""
        trait = TraitObject(
            {"description_fmt": "[this is a description] another test [N/A]"})
        trait_no_unit_type = TraitObject({"description_fmt": ""})
        results = get_trait_units(trait)
        results_no_unit = get_trait_units(trait_no_unit_type)
        self.assertEqual(results, "this is a descriptionN/A")
        self.assertEqual(results_no_unit, "Value")

    @mock.patch("wqflask.show_trait.show_trait.g")
    def test_get_nearest_marker(self, mock_db):
        """test for getting nearest marker with non-empty db"""

        mock_db.db.execute.return_value.fetchall.return_value = [
            ["Geno1", "Geno2"], ["Geno3"]]

        trait = TraitObject({"locus_chr": "test_chr", "locus_mb": "test_mb"})
        group_name = TraitObject({"name": "group_name"})
        this_db = TraitObject({"group": group_name})
        results_with_item_db = get_nearest_marker(trait, this_db)
        called_with_value = """SELECT Geno.Name
               FROM Geno, GenoXRef, GenoFreeze
               WHERE Geno.Chr = 'test_chr' AND
                     GenoXRef.GenoId = Geno.Id AND
                     GenoFreeze.Id = GenoXRef.GenoFreezeId AND
                     GenoFreeze.Name = 'group_nameGeno'
               ORDER BY ABS( Geno.Mb - test_mb) LIMIT 1"""

        mock_db.db.execute.assert_called_with(called_with_value)

        self.assertEqual(results_with_item_db, "Geno1")

    @mock.patch("wqflask.show_trait.show_trait.g")
    def test_get_nearest_marker_empty_db(self, mock_db):
        """test for getting nearest marker with empty db"""
        mock_db.db.execute.return_value.fetchall.return_value = []
        trait = TraitObject({"locus_chr": "test_chr", "locus_mb": "test_mb"})
        group_name = TraitObject({"name": "group_name"})
        this_db = TraitObject({"group": group_name})
        results_empty_db = get_nearest_marker(trait, this_db)
        mock_db.db.execute.assert_called_once()
        self.assertEqual(results_empty_db, "")

    @mock.patch("wqflask.show_trait.show_trait.get_scales_from_genofile")
    def test_get_genotype_scales_with_genofile_is_list(self, mock_get_scales):
        """test for getting genotype scales with genofile as list """
        # where genofile is instance of list
        genofiles_list = [{"filename": "file1", "location": "~/data/files/f1"},
                          {"filename": "file2", "location": "~/data/files/f2"},
                          {"filename": "file3", "location": "~/data/files/f3"}]

        mock_get_scales.side_effect = [[["morgan", "cM"]],
                                       [["morgan", "cM"]],
                                       [["physic", "Mb"]]]

        results = get_genotype_scales(genofiles_list)
        expected_results = {
            "~/data/files/f1": [["morgan", "cM"]],
            "~/data/files/f2": [["morgan", "cM"]],
            "~/data/files/f3": [["physic", "Mb"]]
        }

        multiple_calls = [mock.call('~/data/files/f1'), mock.call('~/data/files/f2'),
                          mock.call('~/data/files/f3')]
        mock_get_scales.assert_has_calls(multiple_calls)
        self.assertEqual(results, expected_results)

    @mock.patch("wqflask.show_trait.show_trait.get_scales_from_genofile")
    def test_genotype_scales_with_genofile_other(self, mock_get_scales):
        """test for getting genotype scales with genofile as a string"""
        file_location = "~/another_file_location"
        mock_get_scales.return_value = [["physic", "Mb"]]
        expected_results = {f"{file_location}": [["physic", "Mb"]]}
        self.assertEqual(get_genotype_scales(file_location), expected_results)
        mock_get_scales.assert_called_once_with(file_location)

    @mock.patch("wqflask.show_trait.show_trait.locate_ignore_error")
    def test_get_scales_from_genofile_found(self, mock_ignore_location):
        """"add test for get scales from genofile where file is found"""
        mock_ignore_location.return_value = True
        geno_file = """
                #sample line    with no  @scales:other\n
                #sample line     @scales and :separated   by semicolon\n
                This attempts    to check whether\n
                """

        geno_file_string = "@line start with  @ and has @scale:morgan"

        file_location = "~/data/file.geno"

        mock_open_geno_file = mock.mock_open(read_data=geno_file)
        with mock.patch("builtins.open", mock_open_geno_file):
            results = get_scales_from_genofile(file_location)
            self.assertEqual(results, [["morgan", "cM"]])

        mock_open_string = mock.mock_open(read_data=geno_file_string)

        with mock.patch("builtins.open", mock_open_string):
            result2 = get_scales_from_genofile(file_location)
            self.assertEqual([['morgan', 'cM']], result2)

    @mock.patch("wqflask.show_trait.show_trait.locate_ignore_error")
    def test_get_scales_from_genofile_not_found(self, mock_location_ignore):
        mock_location_ignore.return_value = False

        expected_results = [["physic", "Mb"]]
        results = get_scales_from_genofile("~/data/file")
        mock_location_ignore.assert_called_once_with("~/data/file", "genotype")
        self.assertEqual(results, expected_results)
<|endoftext|>