<commit_before>import getpass
import binascii
import logging
logger = logging.getLogger(__name__)
import sys
import json
import time
from decimal import Decimal as D

import bitcoin as bitcoinlib
import bitcoin.rpc as bitcoinlib_rpc
from bitcoin.core import CBlock

from counterpartylib.lib import util
from counterpartylib.lib import script
from counterpartylib.lib import config
from counterpartylib.lib import exceptions

from counterpartylib.lib.backend import addrindex, btcd

MEMPOOL_CACHE_INITIALIZED = False

def sortkeypicker(keynames):
    """http://stackoverflow.com/a/1143719"""
    negate = set()
    for i, k in enumerate(keynames):
        if k[:1] == '-':
            keynames[i] = k[1:]
            negate.add(k[1:])
    def getit(adict):
       composite = [adict[k] for k in keynames]
       for i, (k, v) in enumerate(zip(keynames, composite)):
           if k in negate:
               composite[i] = -v
       return composite
    return getit

def BACKEND():
    return sys.modules['counterpartylib.lib.backend.{}'.format(config.BACKEND_NAME)]

def getblockcount():
    return BACKEND().getblockcount()

def getblockhash(blockcount):
    return BACKEND().getblockhash(blockcount)

def getblock(block_hash):
    block_hex = BACKEND().getblock(block_hash)
    return CBlock.deserialize(util.unhexlify(block_hex))

def searchrawtransactions(address, unconfirmed=False):
    return BACKEND().searchrawtransactions(address, unconfirmed=unconfirmed)

def getrawtransaction(tx_hash, verbose=False, skip_missing=False):
    return BACKEND().getrawtransaction(tx_hash, verbose=verbose, skip_missing=skip_missing)

def getrawtransaction_batch(txhash_list, verbose=False, skip_missing=False):
    return BACKEND().getrawtransaction_batch(txhash_list, verbose=verbose, skip_missing=skip_missing)

def sendrawtransaction(tx_hex):
    return BACKEND().sendrawtransaction(tx_hex)

def getrawmempool():
    return BACKEND().getrawmempool()

def extract_addresses(txhash_list):
    return BACKEND().extract_addresses(txhash_list)

def fee_per_kb(nblocks):
    """
    :param nblocks:
    :return: fee_per_kb in satoshis, or None when unable to determine
    """

    return BACKEND().fee_per_kb(nblocks)

def refresh_unconfirmed_transactions_cache(mempool_txhash_list):
    return BACKEND().refresh_unconfirmed_transactions_cache(mempool_txhash_list)

def deserialize(tx_hex):
    return bitcoinlib.core.CTransaction.deserialize(binascii.unhexlify(tx_hex))

def serialize(ctx):
    return bitcoinlib.core.CTransaction.serialize(ctx)

def is_valid(address):
    try:
        script.validate(address)
        return True
    except script.AddressError:
        return False

def get_txhash_list(block):
    return [bitcoinlib.core.b2lx(ctx.GetHash()) for ctx in block.vtx]

def get_tx_list(block):
    raw_transactions = {}
    tx_hash_list = []

    for ctx in block.vtx:
        tx_hash = bitcoinlib.core.b2lx(ctx.GetHash())
        raw = ctx.serialize()

        tx_hash_list.append(tx_hash)
        raw_transactions[tx_hash] = bitcoinlib.core.b2x(raw)

    return (tx_hash_list, raw_transactions)

def input_value_weight(amount):
    # Prefer outputs less than dust size, then bigger is better.
    if amount * config.UNIT <= config.DEFAULT_REGULAR_DUST_SIZE:
        return 0
    else:
        return 1 / amount

def sort_unspent_txouts(unspent, unconfirmed=False):
    # Get deterministic results (for multiAPIConsensus type requirements), sort by timestamp and vout index.
    # (Oldest to newest so the nodes don’t have to be exactly caught up to each other for consensus to be achieved.)
    # searchrawtransactions doesn’t support unconfirmed transactions
    try:
        unspent = sorted(unspent, key=sortkeypicker(['ts', 'vout']))
    except KeyError: # If timestamp isn’t given.
        pass

    # Sort by amount.
    unspent = sorted(unspent, key=lambda x: input_value_weight(x['amount']))

    return unspent

def get_btc_supply(normalize=False):
    """returns the total supply of {} (based on what Bitcoin Core says the current block height is)""".format(config.BTC)
    block_count = getblockcount()
    blocks_remaining = block_count
    total_supply = 0
    reward = 50.0
    while blocks_remaining > 0:
        if blocks_remaining >= 210000:
            blocks_remaining -= 210000
            total_supply += 210000 * reward
            reward /= 2
        else:
            total_supply += (blocks_remaining * reward)
            blocks_remaining = 0
    return total_supply if normalize else int(total_supply * config.UNIT)

def is_scriptpubkey_spendable(scriptpubkey_hex, source, multisig_inputs=False):
    c_scriptpubkey = bitcoinlib.core.CScript(bitcoinlib.core.x(scriptpubkey_hex))

    try:
        vout_address = script.scriptpubkey_to_address(c_scriptpubkey)
    except exceptions.DecodeError:
        return False

    if not vout_address:
        return False

    source = script.make_canonical(source)

    if vout_address == source:
        return True

    return False

class MempoolError(Exception):
    pass

def get_unspent_txouts(source, unconfirmed=False, multisig_inputs=False, unspent_tx_hash=None):
    """returns a list of unspent outputs for a specific address
    @return: A list of dicts, with each entry in the dict having the following keys:
    """
    if not MEMPOOL_CACHE_INITIALIZED:
        raise MempoolError('Mempool is not yet ready; please try again in a few minutes.')

    # Get all outputs.
    logger.debug('Getting outputs for {}'.format(source))
    if unspent_tx_hash:
        raw_transactions = [getrawtransaction(unspent_tx_hash, verbose=True)]
    else:
        if script.is_multisig(source):
            pubkeyhashes = script.pubkeyhash_array(source)
            raw_transactions = searchrawtransactions(pubkeyhashes[1], unconfirmed=True) # unconfirmed=True to prune unconfirmed spent outputs
        else:
            pubkeyhashes = [source]
            raw_transactions = searchrawtransactions(source, unconfirmed=True)

    # Change format.
    # TODO: Slow.
    logger.debug('Formatting outputs for {}'.format(source))
    outputs = {}
    for tx in raw_transactions:
        for vout in tx['vout']:
            txid = tx['txid']
            confirmations = tx['confirmations'] if 'confirmations' in tx else 0
            outkey = '{}{}'.format(txid, vout['n']) # edge case: avoid duplicate output
            if outkey not in outputs or outputs[outkey]['confirmations'] < confirmations:
                coin = {
                        'amount': float(vout['value']),
                        'confirmations': confirmations,
                        'scriptPubKey': vout['scriptPubKey']['hex'],
                        'txid': txid,
                        'vout': vout['n']
                       }
                outputs[outkey] = coin
    outputs = sorted(outputs.values(), key=lambda x: x['confirmations'])

    # Prune unspendable.
    logger.debug('Pruning unspendable outputs for {}'.format(source))
    # TODO: Slow.
    outputs = [output for output in outputs if is_scriptpubkey_spendable(output['scriptPubKey'], source)]

    # Prune spent outputs.
    logger.debug('Pruning spent outputs for {}'.format(source))
    vins = {(vin['txid'], vin['vout']) for tx in raw_transactions for vin in tx['vin'] if 'coinbase' not in vin}
    unspent = []
    for output in outputs:
        if (output['txid'], output['vout']) not in vins:
            unspent.append(output)
    unspent = sorted(unspent, key=lambda x: x['txid'])

    # Remove unconfirmed txouts, if desired.
    if not unconfirmed:
        unspent = [output for output in unspent if output['confirmations'] > 0]
    else:
        # Hackish: Allow only inputs which are either already confirmed or were seen only recently. (Skip outputs from slow‐to‐confirm transanctions.)
        try:
            unspent = [output for output in unspent if output['confirmations'] > 0 or (time.time() - output['ts']) < 6 * 3600] # Cutoff: six hours
        except (KeyError, TypeError):
            pass

    return unspent

class UnknownPubKeyError(Exception):
    pass

def pubkeyhash_to_pubkey(pubkeyhash, provided_pubkeys=None):
    # Search provided pubkeys.
    if provided_pubkeys:
        if type(provided_pubkeys) != list:
            provided_pubkeys = [provided_pubkeys]
        for pubkey in provided_pubkeys:
            if pubkeyhash == script.pubkey_to_pubkeyhash(util.unhexlify(pubkey)):
                return pubkey

    # Search blockchain.
    raw_transactions = searchrawtransactions(pubkeyhash, unconfirmed=True)
    for tx in raw_transactions:
        for vin in tx['vin']:
            if 'coinbase' not in vin:
                scriptsig = vin['scriptSig']
                asm = scriptsig['asm'].split(' ')
                if len(asm) >= 2:
                    # catch unhexlify errs for when asm[1] isn't a pubkey (eg; for P2SH)
                    try:
                        pubkey = asm[1]
                        if pubkeyhash == script.pubkey_to_pubkeyhash(util.unhexlify(pubkey)):
                            return pubkey
                    except binascii.Error:
                        pass

    raise UnknownPubKeyError('Public key was neither provided nor published in blockchain.')

def multisig_pubkeyhashes_to_pubkeys(address, provided_pubkeys=None):
    signatures_required, pubkeyhashes, signatures_possible = script.extract_array(address)
    pubkeys = [pubkeyhash_to_pubkey(pubkeyhash, provided_pubkeys) for pubkeyhash in pubkeyhashes]
    return script.construct_array(signatures_required, pubkeys, signatures_possible)

def init_mempool_cache():
    """prime the mempool cache, so that functioning is faster...
    """
    global MEMPOOL_CACHE_INITIALIZED
    logger.debug('Initializing mempool cache...')
    start = time.time()

    mempool_txhash_list = getrawmempool()

    #with this function, don't try to load in more than BACKEND_RAW_TRANSACTIONS_CACHE_SIZE entries
    num_tx = min(len(mempool_txhash_list), config.BACKEND_RAW_TRANSACTIONS_CACHE_SIZE)
    mempool_tx = BACKEND().getrawtransaction_batch(mempool_txhash_list[:num_tx], skip_missing=True, verbose=True)

    vin_txhash_list = []
    max_remaining_num_tx = config.BACKEND_RAW_TRANSACTIONS_CACHE_SIZE - num_tx
    if max_remaining_num_tx:
        for txid in mempool_tx:
            tx = mempool_tx[txid]
            vin_txhash_list += [vin['txid'] for vin in tx['vin']]
        BACKEND().getrawtransaction_batch(vin_txhash_list[:max_remaining_num_tx], skip_missing=True, verbose=True)

    MEMPOOL_CACHE_INITIALIZED = True
    logger.info('Mempool cache initialized: {:.2f}s for {:,} transactions'.format(time.time() - start, num_tx + min(max_remaining_num_tx, len(vin_txhash_list))))

# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
<commit_msg>fixes #969<commit_after>import getpass
import binascii
import logging
logger = logging.getLogger(__name__)
import sys
import json
import time
from decimal import Decimal as D

import bitcoin as bitcoinlib
import bitcoin.rpc as bitcoinlib_rpc
from bitcoin.core import CBlock

from counterpartylib.lib import util
from counterpartylib.lib import script
from counterpartylib.lib import config
from counterpartylib.lib import exceptions

from counterpartylib.lib.backend import addrindex, btcd

MEMPOOL_CACHE_INITIALIZED = False

def sortkeypicker(keynames):
    """http://stackoverflow.com/a/1143719"""
    negate = set()
    for i, k in enumerate(keynames):
        if k[:1] == '-':
            keynames[i] = k[1:]
            negate.add(k[1:])
    def getit(adict):
       composite = [adict[k] for k in keynames]
       for i, (k, v) in enumerate(zip(keynames, composite)):
           if k in negate:
               composite[i] = -v
       return composite
    return getit

def BACKEND():
    return sys.modules['counterpartylib.lib.backend.{}'.format(config.BACKEND_NAME)]

def getblockcount():
    return BACKEND().getblockcount()

def getblockhash(blockcount):
    return BACKEND().getblockhash(blockcount)

def getblock(block_hash):
    block_hex = BACKEND().getblock(block_hash)
    return CBlock.deserialize(util.unhexlify(block_hex))

def searchrawtransactions(address, unconfirmed=False):
    return BACKEND().searchrawtransactions(address, unconfirmed=unconfirmed)

def getrawtransaction(tx_hash, verbose=False, skip_missing=False):
    return BACKEND().getrawtransaction(tx_hash, verbose=verbose, skip_missing=skip_missing)

def getrawtransaction_batch(txhash_list, verbose=False, skip_missing=False):
    return BACKEND().getrawtransaction_batch(txhash_list, verbose=verbose, skip_missing=skip_missing)

def sendrawtransaction(tx_hex):
    return BACKEND().sendrawtransaction(tx_hex)

def getrawmempool():
    return BACKEND().getrawmempool()

def extract_addresses(txhash_list):
    return BACKEND().extract_addresses(txhash_list)

def fee_per_kb(nblocks):
    """
    :param nblocks:
    :return: fee_per_kb in satoshis, or None when unable to determine
    """

    return BACKEND().fee_per_kb(nblocks)

def refresh_unconfirmed_transactions_cache(mempool_txhash_list):
    return BACKEND().refresh_unconfirmed_transactions_cache(mempool_txhash_list)

def deserialize(tx_hex):
    return bitcoinlib.core.CTransaction.deserialize(binascii.unhexlify(tx_hex))

def serialize(ctx):
    return bitcoinlib.core.CTransaction.serialize(ctx)

def is_valid(address):
    try:
        script.validate(address)
        return True
    except script.AddressError:
        return False

def get_txhash_list(block):
    return [bitcoinlib.core.b2lx(ctx.GetHash()) for ctx in block.vtx]

def get_tx_list(block):
    raw_transactions = {}
    tx_hash_list = []

    for ctx in block.vtx:
        tx_hash = bitcoinlib.core.b2lx(ctx.GetHash())
        raw = ctx.serialize()

        tx_hash_list.append(tx_hash)
        raw_transactions[tx_hash] = bitcoinlib.core.b2x(raw)

    return (tx_hash_list, raw_transactions)

def sort_unspent_txouts(unspent, unconfirmed=False):
    # Sort by amount, using the largest UTXOs available, which avoids dust outputs if at all possible
    unspent = sorted(unspent, key=lambda x: x['amount'], reverse=True)

    return unspent

def get_btc_supply(normalize=False):
    """returns the total supply of {} (based on what Bitcoin Core says the current block height is)""".format(config.BTC)
    block_count = getblockcount()
    blocks_remaining = block_count
    total_supply = 0
    reward = 50.0
    while blocks_remaining > 0:
        if blocks_remaining >= 210000:
            blocks_remaining -= 210000
            total_supply += 210000 * reward
            reward /= 2
        else:
            total_supply += (blocks_remaining * reward)
            blocks_remaining = 0
    return total_supply if normalize else int(total_supply * config.UNIT)

def is_scriptpubkey_spendable(scriptpubkey_hex, source, multisig_inputs=False):
    c_scriptpubkey = bitcoinlib.core.CScript(bitcoinlib.core.x(scriptpubkey_hex))

    try:
        vout_address = script.scriptpubkey_to_address(c_scriptpubkey)
    except exceptions.DecodeError:
        return False

    if not vout_address:
        return False

    source = script.make_canonical(source)

    if vout_address == source:
        return True

    return False

class MempoolError(Exception):
    pass

def get_unspent_txouts(source, unconfirmed=False, multisig_inputs=False, unspent_tx_hash=None):
    """returns a list of unspent outputs for a specific address
    @return: A list of dicts, with each entry in the dict having the following keys:
    """
    if not MEMPOOL_CACHE_INITIALIZED:
        raise MempoolError('Mempool is not yet ready; please try again in a few minutes.')

    # Get all outputs.
    logger.debug('Getting outputs for {}'.format(source))
    if unspent_tx_hash:
        raw_transactions = [getrawtransaction(unspent_tx_hash, verbose=True)]
    else:
        if script.is_multisig(source):
            pubkeyhashes = script.pubkeyhash_array(source)
            raw_transactions = searchrawtransactions(pubkeyhashes[1], unconfirmed=True) # unconfirmed=True to prune unconfirmed spent outputs
        else:
            pubkeyhashes = [source]
            raw_transactions = searchrawtransactions(source, unconfirmed=True)

    # Change format.
    # TODO: Slow.
    logger.debug('Formatting outputs for {}'.format(source))
    outputs = {}
    for tx in raw_transactions:
        for vout in tx['vout']:
            txid = tx['txid']
            confirmations = tx['confirmations'] if 'confirmations' in tx else 0
            outkey = '{}{}'.format(txid, vout['n']) # edge case: avoid duplicate output
            if outkey not in outputs or outputs[outkey]['confirmations'] < confirmations:
                coin = {
                        'amount': float(vout['value']),
                        'confirmations': confirmations,
                        'scriptPubKey': vout['scriptPubKey']['hex'],
                        'txid': txid,
                        'vout': vout['n']
                       }
                outputs[outkey] = coin
    outputs = sorted(outputs.values(), key=lambda x: x['confirmations'])

    # Prune unspendable.
    logger.debug('Pruning unspendable outputs for {}'.format(source))
    # TODO: Slow.
    outputs = [output for output in outputs if is_scriptpubkey_spendable(output['scriptPubKey'], source)]

    # Prune spent outputs.
    logger.debug('Pruning spent outputs for {}'.format(source))
    vins = {(vin['txid'], vin['vout']) for tx in raw_transactions for vin in tx['vin'] if 'coinbase' not in vin}
    unspent = []
    for output in outputs:
        if (output['txid'], output['vout']) not in vins:
            unspent.append(output)
    unspent = sorted(unspent, key=lambda x: x['txid'])

    # Remove unconfirmed txouts, if desired.
    if not unconfirmed:
        unspent = [output for output in unspent if output['confirmations'] > 0]
    else:
        # Hackish: Allow only inputs which are either already confirmed or were seen only recently. (Skip outputs from slow‐to‐confirm transanctions.)
        try:
            unspent = [output for output in unspent if output['confirmations'] > 0 or (time.time() - output['ts']) < 6 * 3600] # Cutoff: six hours
        except (KeyError, TypeError):
            pass

    return unspent

class UnknownPubKeyError(Exception):
    pass

def pubkeyhash_to_pubkey(pubkeyhash, provided_pubkeys=None):
    # Search provided pubkeys.
    if provided_pubkeys:
        if type(provided_pubkeys) != list:
            provided_pubkeys = [provided_pubkeys]
        for pubkey in provided_pubkeys:
            if pubkeyhash == script.pubkey_to_pubkeyhash(util.unhexlify(pubkey)):
                return pubkey

    # Search blockchain.
    raw_transactions = searchrawtransactions(pubkeyhash, unconfirmed=True)
    for tx in raw_transactions:
        for vin in tx['vin']:
            if 'coinbase' not in vin:
                scriptsig = vin['scriptSig']
                asm = scriptsig['asm'].split(' ')
                if len(asm) >= 2:
                    # catch unhexlify errs for when asm[1] isn't a pubkey (eg; for P2SH)
                    try:
                        pubkey = asm[1]
                        if pubkeyhash == script.pubkey_to_pubkeyhash(util.unhexlify(pubkey)):
                            return pubkey
                    except binascii.Error:
                        pass

    raise UnknownPubKeyError('Public key was neither provided nor published in blockchain.')

def multisig_pubkeyhashes_to_pubkeys(address, provided_pubkeys=None):
    signatures_required, pubkeyhashes, signatures_possible = script.extract_array(address)
    pubkeys = [pubkeyhash_to_pubkey(pubkeyhash, provided_pubkeys) for pubkeyhash in pubkeyhashes]
    return script.construct_array(signatures_required, pubkeys, signatures_possible)

def init_mempool_cache():
    """prime the mempool cache, so that functioning is faster...
    """
    global MEMPOOL_CACHE_INITIALIZED
    logger.debug('Initializing mempool cache...')
    start = time.time()

    mempool_txhash_list = getrawmempool()

    #with this function, don't try to load in more than BACKEND_RAW_TRANSACTIONS_CACHE_SIZE entries
    num_tx = min(len(mempool_txhash_list), config.BACKEND_RAW_TRANSACTIONS_CACHE_SIZE)
    mempool_tx = BACKEND().getrawtransaction_batch(mempool_txhash_list[:num_tx], skip_missing=True, verbose=True)

    vin_txhash_list = []
    max_remaining_num_tx = config.BACKEND_RAW_TRANSACTIONS_CACHE_SIZE - num_tx
    if max_remaining_num_tx:
        for txid in mempool_tx:
            tx = mempool_tx[txid]
            vin_txhash_list += [vin['txid'] for vin in tx['vin']]
        BACKEND().getrawtransaction_batch(vin_txhash_list[:max_remaining_num_tx], skip_missing=True, verbose=True)

    MEMPOOL_CACHE_INITIALIZED = True
    logger.info('Mempool cache initialized: {:.2f}s for {:,} transactions'.format(time.time() - start, num_tx + min(max_remaining_num_tx, len(vin_txhash_list))))

# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
<|endoftext|>


<commit_before>#! /usr/bin/env python3.4

from loadData import loadcsv

data = loadcsv("data.csv")
m = len(data)
n = len(data[0])
reserve = [True]*n
attr = {}
for i in range(n):
	attr[data[0][i]] = i

print("Constant attributes:")
for j in range(n):
	same = True
	p = 1
	while p<n and data[p][j]=="":
		p += 1
	for i in range(p+1, n):
		if data[i][j]!="" and data[i][j]!=data[p][j]:
			same = False
			break
	if same:
		if p<n:
			value = data[p][j]
		else:
			value = ""
		print('{} = "{}"'.format(data[0][j], value))
		reserve[j] = False
print()

print("Duplicate attributes:")
for i in range(n):
	for j in range(i+1, n):
		if data[1][i]=="" or data[1][j]=="":
			continue
		same = True
		for l in range(1, m):
			if data[l][i]!=data[l][j]:
				same = False
				break
		if same:
			print(data[0][i], data[0][j])
			reserve[j] = False
print()

attrFile = open("attr_list.txt", "w")
lines = []
for i in range(n):
	if reserve[i]:
		lines.append(data[0][i]+"\n")
attrFile.writelines(lines)

dependFile = open("dependences.txt", "r")
lines = dependFile.readlines()
for k in range(len(lines)):
	items = lines[k].split()
	depend = [attr[items[i]] for i in range(len(items))]
	for j in range(1, len(depend)):
		ok = True
		check = {}
		print(items[0]+": "+items[j])
		for i in range(1, m):
			if data[i][depend[0]] in check:
				if check[data[i][depend[0]]]!=data[i][depend[j]]:
					ok = False
					print(i, data[i][depend[0]], check[data[i][depend[0]]], data[i][depend[j]])
			else:
				check[data[i][depend[0]]] = data[i][depend[j]]
		print(ok)

<commit_msg>fix a mistake<commit_after>#! /usr/bin/env python3.4

from loadData import loadcsv

data = loadcsv("data.csv")
m = len(data)
n = len(data[0])
reserve = [True]*n
attr = {}
for i in range(n):
	attr[data[0][i]] = i

print("Constant attributes:")
for j in range(n):
	same = True
	p = 1
	while p<m and data[p][j]=="":
		p += 1
	for i in range(p+1, m):
		if data[i][j]!="" and data[i][j]!=data[p][j]:
			same = False
			break
	if same:
		if p<m:
			value = data[p][j]
		else:
			value = ""
		print('{} = "{}"'.format(data[0][j], value))
		reserve[j] = False
print()

print("Duplicate attributes:")
for i in range(n):
	for j in range(i+1, n):
		if data[1][i]=="" or data[1][j]=="":
			continue
		same = True
		for l in range(1, m):
			if data[l][i]!=data[l][j]:
				same = False
				break
		if same:
			print(data[0][i], data[0][j])
			reserve[j] = False
print()

attrFile = open("attr_list.txt", "w")
lines = []
for i in range(n):
	if reserve[i]:
		lines.append(data[0][i]+"\n")
attrFile.writelines(lines)

dependFile = open("dependences.txt", "r")
lines = dependFile.readlines()
for k in range(len(lines)):
	items = lines[k].split()
	depend = [attr[items[i]] for i in range(len(items))]
	for j in range(1, len(depend)):
		ok = True
		check = {}
		print(items[0]+": "+items[j])
		for i in range(1, m):
			if data[i][depend[0]] in check:
				if check[data[i][depend[0]]]!=data[i][depend[j]]:
					ok = False
					print(i, data[i][depend[0]], check[data[i][depend[0]]], data[i][depend[j]])
			else:
				check[data[i][depend[0]]] = data[i][depend[j]]
		print(ok)

<|endoftext|>


<commit_before>from django.contrib import admin

from publishing.models import PublishingProfile, PublishingRegion, PublishingLanguage

class PublishingProfileAdmin(admin.ModelAdmin):
    pass

class PublishingRegionAdmin(admin.ModelAdmin):
    pass

class PublishingLanguageAdmin(admin.ModelAdmin):
    pass

admin.site.register(PublishingProfile, PublishingProfileAdmin)
admin.site.register(PublishingRegion, PublishingRegionAdmin)
admin.site.register(PublishingLanguage, PublishingLanguageAdmin)<commit_msg>admin<commit_after>from django.contrib import admin

from publishing.models import PublishingProfile, PublishingRegion, PublishingLanguage, PublishingCountry

class PublishingProfileAdmin(admin.ModelAdmin):
    pass

class PublishingRegionAdmin(admin.ModelAdmin):
    pass

class PublishingLanguageAdmin(admin.ModelAdmin):
    pass

class PublishingCountryAdmin(admin.ModelAdmin):
    pass

admin.site.register(PublishingProfile, PublishingProfileAdmin)
admin.site.register(PublishingRegion, PublishingRegionAdmin)
admin.site.register(PublishingLanguage, PublishingLanguageAdmin)
admin.site.register(PublishingCountry, PublishingCountryAdmin)<|endoftext|>


<commit_before>#!/bin/env jython
import java
import math
import json
import sys

import org.gavrog.joss.pgraphs as pgraphs
import org.gavrog.joss.geometry as geometry

def run():
    def writeln(s=''):
        print s

    for source_path in sys.argv[1:]:
        index_in_source = 0

        for net in pgraphs.io.Net.iterator(source_path):
            index_in_source += 1

            writeln('{')
            writeln('  "source_path": "%s",' % source_path)
            writeln('  "index_in_source": %d,' % index_in_source)

            warnings, errors = process_net(net, writeln=writeln)
            status = "Error" if errors else "OK"

            writeln('  "warnings": %s,' % json.dumps(warnings))
            writeln('  "errors": %s,' % json.dumps(errors))
            writeln('  "status": "%s"' % status)
            writeln('}')
            writeln()

def process_net(net, writeln):
    writeln('  "net_name": "%s",' % net.name)

    warnings = list(net.warnings)
    errors = list(net.errors)

    if errors:
        return warnings, errors

    if not net.isConnected():
        errors.append("disconnected net")
    elif not net.isLocallyStable():
        errors.append("next-nearest neighbor collisions")
    elif not net.isStable():
        warnings.append("net has collisions")

        if net.isLadder():
            warnings.append("ladder net")
        elif net.hasSecondOrderCollisions():
            warnings.append("possible ladder net")

    if errors:
        return warnings, errors

    if not net.isMinimal():
        warnings.append("reduced from supercell")

    minMap = net.minimalImageMap()
    net = net.minimalImage()

    finder = geometry.SpaceGroupFinder(net.getSpaceGroup())
    writeln('  "spacegroup_name": "%s",' % finder.groupName)
    if finder.extension:
        writeln('  "spacegroup_extension": "%s",' % finder.extension)

    orbit_reps = [list(orb)[0] for orb in net.nodeOrbits()]
    seqs = [coordination_sequence(net, v, 10) for v in orbit_reps]
    psyms = [net.pointSymbol(v) for v in orbit_reps]

    writeln('  "net_nodes_unitcell": %s,' % net.numberOfNodes())
    writeln('  "net_nodes_asym": %s,' % len(orbit_reps))

    writeln('  "net_valency": %s,' % json.dumps([s[0] for s in seqs]))
    writeln('  "net_coordination_seqs": [')
    writeln('      %s' % ',\n      '.join(json.dumps(s) for s in seqs))
    writeln('    ],')
    writeln('  "net_wells_point_symbols": %s,' % json.dumps(psyms))

    writeln('  "net_systre_key": "%s",' % net.systreKey)

    nodeToName, mergedNames = node_name_mapping(minMap)

    try:
        success = write_embedding(
            net, 'net_barycentric', nodeToName, finder, writeln
        )
        if success == False:
            errors.append("barycentric embedding failed verification")
    except:
        errors.append("barycentric embedding crashed")

    try:
        success = write_embedding(
            net, 'net_maximal', nodeToName, finder, writeln
        )
        if success == False:
            errors.append("maximal embedding failed verification")
    except:
        errors.append("maximal embedding crashed")

    try:
        success = write_embedding(
            net, 'net_relaxed', nodeToName, finder, writeln
        )
        if success == False:
            errors.append("relaxed embedding failed verification")
    except:
        errors.append("relaxed embedding crashed")

    return warnings, errors

def node_name_mapping(phi):
    imageNode2Orbit = {}
    for orbit in phi.imageGraph.nodeOrbits():
        orbit = frozenset(orbit)
        for v in orbit:
            imageNode2Orbit[v] = orbit

    orbit2name = {}
    node2name = {}
    mergedNames = []
    mergedNamesSeen = set()

    for v in phi.sourceGraph.nodes():
        name = phi.sourceGraph.getNodeName(v)
        w = phi.getImage(v)
        orbit = imageNode2Orbit[w]

        if name != orbit2name.get(orbit, name):
            pair = (name, orbit2name[orbit])
            if pair not in mergedNamesSeen:
                mergedNames.append(pair)
                mergedNamesSeen.add(pair)
        else:
            orbit2name[orbit] = name

        node2name[w] = orbit2name[orbit]

    return node2name, mergedNames

def write_embedding(graph, prefix, nodeToName, finder, writeln):
    embedder = pgraphs.embed.Embedder(graph, None, False)

    if prefix == 'net_maximal':
        embedder.setMaximizeVolume(True)
        embedder.setRelaxPositions(False)
        embedder.setPasses(0)
        embedder.go(10000)
        embedder.normalize()
    else:
        embedder.setMaximizeVolume(False)
        embedder.setRelaxPositions(False)
        embedder.setPasses(0)
        embedder.go(500)
        embedder.normalize()

        embedder.setMaximizeVolume(False)
        embedder.setRelaxPositions(prefix == 'net_relaxed')
        embedder.setPasses(3)
        embedder.go(10000)
        embedder.normalize()

    if not verifyEmbedding(graph, nodeToName, finder, embedder):
        return False

    dim = graph.dimension
    net = pgraphs.embed.ProcessedNet(graph, 'X', nodeToName, finder, embedder)
    cgd = serializedNet(net, asCGD=True)

    nodes = []
    edges = []

    for line in cgd.split('\n'):
        fields = line.strip().split()
        if len(fields) == 0:
            continue

        if fields[0] == 'CELL':
            if dim == 3:
                a, b, c, alpha, beta, gamma = map(float, fields[1:])
                writeln('  "%s_unitcell_a": %s,' % (prefix, a))
                writeln('  "%s_unitcell_b": %s,' % (prefix, b))
                writeln('  "%s_unitcell_c": %s,' % (prefix, c))
                writeln('  "%s_unitcell_alpha": %s,' % (prefix, alpha))
                writeln('  "%s_unitcell_beta": %s,' % (prefix, beta))
                writeln('  "%s_unitcell_gamma": %s,' % (prefix, gamma))
            elif dim == 2:
                a, b, gamma = map(float, fields[1:])
                writeln('  "%s_unitcell_a": %s,' % (prefix, a))
                writeln('  "%s_unitcell_b": %s,' % (prefix, b))
                writeln('  "%s_unitcell_gamma": %s,' % (prefix, gamma))
        elif fields[0] == 'CELL_VOLUME':
            volume = float(fields[1])
            writeln('  "%s_unitcell_volume": %s,' % (prefix, volume))
        elif fields[0] == 'NODE':
            nodes.append(map(float, fields[3:]))
        elif fields[0] == 'EDGE':
            edges.append(map(float, fields[1:]))
        elif fields[0] == '#' and len(fields) > 1:
            if fields[1] == 'EDGE_MIN':
                minEdge = float(fields[2])
                writeln('  "%s_shortest_edge_length": %s,' % (prefix, minEdge))
            elif fields[1] == 'EDGE_AVG':
                avgEdge = float(fields[2])
                writeln('  "%s_average_edge_length": %s,' % (prefix, avgEdge))
            elif fields[1] == 'EDGE_MAX':
                maxEdge = float(fields[2])
                writeln('  "%s_longest_edge_length": %s,' % (prefix, maxEdge))
            if fields[1] == 'ANGLE_MIN':
                minAngle = float(fields[2])
                writeln('  "%s_smallest_angle": %s,' % (prefix, minAngle))
            elif fields[1] == 'ANGLE_AVG':
                avgAngle = float(fields[2])
                writeln('  "%s_average_angle": %s,' % (prefix, avgAngle))
            elif fields[1] == 'ANGLE_MAX':
                maxAngle = float(fields[2])
                writeln('  "%s_largest_angle": %s,' % (prefix, maxAngle))
            elif fields[1] == 'SMALLEST_SEPARATION':
                sep = float(fields[2])
                writeln('  "%s_smallest_atom_separation": %s,' % (prefix, sep))
            elif fields[1] == 'DEGREES_OF_FREEDOM':
                dof = int(fields[2])
                writeln('  "%s_degrees_of_freedom": %s,' % (prefix, dof))

    writeln('  "%s_atoms": %s,' % (prefix, nodes))
    writeln('  "%s_edges": %s,' % (prefix, edges))

def serializedNet(net, asCGD=False, writeFullCell=False, prefix=''):
    stringWriter = java.io.StringWriter()
    writer = java.io.PrintWriter(stringWriter)
    net.writeEmbedding(writer, asCGD, writeFullCell, prefix)
    return stringWriter.toString()

def verifyEmbedding(graph, nodeToName, finder, embedder):
    gram = embedder.gramMatrix

    if gram.get(0, 0).doubleValue() < 0.001:
        return False

    if gram.getSubMatrix(0, 0, 2, 2).determinant().doubleValue() < 0.001:
        return False

    if gram.determinant().doubleValue() < 0.001:
        return False

    if not graph.isStable():
        return True

    net = pgraphs.embed.ProcessedNet(graph, 'X', nodeToName, finder, embedder)
    cgd = serializedNet(net, asCGD=True)
    test = pgraphs.io.NetParser.stringToNet(cgd)

    if not test.minimalImage().equals(graph):
        return False

    return True

def coordination_sequence(net, v, n):
    cs = net.coordinationSequence(v)
    cs.next()

    out = []
    for i in range(n):
        out.append(cs.next())

    return out

if __name__ == "__main__":
    run()
<commit_msg>analyzeNetsForEpinet: refactor computation and output of embeddings<commit_after>#!/bin/env jython
import java
import math
import json
import sys

import org.gavrog.joss.pgraphs as pgraphs
import org.gavrog.joss.geometry as geometry

def run():
    def writeln(s=''):
        print s

    for source_path in sys.argv[1:]:
        index_in_source = 0

        for net in pgraphs.io.Net.iterator(source_path):
            index_in_source += 1

            writeln('{')
            writeln('  "source_path": "%s",' % source_path)
            writeln('  "index_in_source": %d,' % index_in_source)

            warnings, errors = process_net(net, writeln=writeln)
            status = "Error" if errors else "OK"

            writeln('  "warnings": %s,' % json.dumps(warnings))
            writeln('  "errors": %s,' % json.dumps(errors))
            writeln('  "status": "%s"' % status)
            writeln('}')
            writeln()

def process_net(net, writeln):
    writeln('  "net_name": "%s",' % net.name)

    warnings = list(net.warnings)
    errors = list(net.errors)

    if errors:
        return warnings, errors

    if not net.isConnected():
        errors.append("disconnected net")
    elif not net.isLocallyStable():
        errors.append("next-nearest neighbor collisions")
    elif not net.isStable():
        warnings.append("net has collisions")

        if net.isLadder():
            warnings.append("ladder net")
        elif net.hasSecondOrderCollisions():
            warnings.append("possible ladder net")

    if errors:
        return warnings, errors

    if not net.isMinimal():
        warnings.append("reduced from supercell")

    minMap = net.minimalImageMap()
    net = net.minimalImage()

    finder = geometry.SpaceGroupFinder(net.getSpaceGroup())
    writeln('  "spacegroup_name": "%s",' % finder.groupName)
    if finder.extension:
        writeln('  "spacegroup_extension": "%s",' % finder.extension)

    orbit_reps = [list(orb)[0] for orb in net.nodeOrbits()]
    seqs = [coordination_sequence(net, v, 10) for v in orbit_reps]
    psyms = [net.pointSymbol(v) for v in orbit_reps]

    writeln('  "net_nodes_unitcell": %s,' % net.numberOfNodes())
    writeln('  "net_nodes_asym": %s,' % len(orbit_reps))

    writeln('  "net_valency": %s,' % json.dumps([s[0] for s in seqs]))
    writeln('  "net_coordination_seqs": [')
    writeln('      %s' % ',\n      '.join(json.dumps(s) for s in seqs))
    writeln('    ],')
    writeln('  "net_wells_point_symbols": %s,' % json.dumps(psyms))

    writeln('  "net_systre_key": "%s",' % net.systreKey)

    nodeToName, mergedNames = node_name_mapping(minMap)

    for kind in ['barycentric', 'maximal', 'relaxed']:
        try:
            e = make_embedding(net, kind)
            try:
                if not verify_embedding(net, nodeToName, finder, e):
                    warnings.append("%s embedding failed verification" % kind)
            except:
                errors.append("%s embedding verification crashed" % kind)

            pnet = pgraphs.embed.ProcessedNet(net, 'X', nodeToName, finder, e)
            write_embedding(pnet, "net_%s" % kind, writeln)
        except:
            errors.append("%s embedding crashed" % kind)

    return warnings, errors

def node_name_mapping(phi):
    imageNode2Orbit = {}
    for orbit in phi.imageGraph.nodeOrbits():
        orbit = frozenset(orbit)
        for v in orbit:
            imageNode2Orbit[v] = orbit

    orbit2name = {}
    node2name = {}
    mergedNames = []
    mergedNamesSeen = set()

    for v in phi.sourceGraph.nodes():
        name = phi.sourceGraph.getNodeName(v)
        w = phi.getImage(v)
        orbit = imageNode2Orbit[w]

        if name != orbit2name.get(orbit, name):
            pair = (name, orbit2name[orbit])
            if pair not in mergedNamesSeen:
                mergedNames.append(pair)
                mergedNamesSeen.add(pair)
        else:
            orbit2name[orbit] = name

        node2name[w] = orbit2name[orbit]

    return node2name, mergedNames

def make_embedding(graph, kind):
    embedder = pgraphs.embed.Embedder(graph, None, False)

    if kind == 'maximal':
        embedder.setMaximizeVolume(True)
        embedder.setRelaxPositions(False)
        embedder.setPasses(0)
        embedder.go(10000)
        embedder.normalize()
    else:
        embedder.setMaximizeVolume(False)
        embedder.setRelaxPositions(False)
        embedder.setPasses(0)
        embedder.go(500)
        embedder.normalize()

        embedder.setMaximizeVolume(False)
        embedder.setRelaxPositions(kind == 'relaxed')
        embedder.setPasses(3)
        embedder.go(10000)
        embedder.normalize()

    return embedder

def write_embedding(net, prefix, writeln):
    dim = net.graph.dimension
    cgd = serialized_net(net, asCGD=True)

    nodes = []
    edges = []

    for line in cgd.split('\n'):
        fields = line.strip().split()
        if len(fields) == 0:
            continue

        if fields[0] == 'CELL':
            if dim == 3:
                a, b, c, alpha, beta, gamma = map(float, fields[1:])
                writeln('  "%s_unitcell_a": %s,' % (prefix, a))
                writeln('  "%s_unitcell_b": %s,' % (prefix, b))
                writeln('  "%s_unitcell_c": %s,' % (prefix, c))
                writeln('  "%s_unitcell_alpha": %s,' % (prefix, alpha))
                writeln('  "%s_unitcell_beta": %s,' % (prefix, beta))
                writeln('  "%s_unitcell_gamma": %s,' % (prefix, gamma))
            elif dim == 2:
                a, b, gamma = map(float, fields[1:])
                writeln('  "%s_unitcell_a": %s,' % (prefix, a))
                writeln('  "%s_unitcell_b": %s,' % (prefix, b))
                writeln('  "%s_unitcell_gamma": %s,' % (prefix, gamma))
        elif fields[0] == 'CELL_VOLUME':
            volume = float(fields[1])
            writeln('  "%s_unitcell_volume": %s,' % (prefix, volume))
        elif fields[0] == 'NODE':
            nodes.append(map(float, fields[3:]))
        elif fields[0] == 'EDGE':
            edges.append(map(float, fields[1:]))
        elif fields[0] == '#' and len(fields) > 1:
            if fields[1] == 'EDGE_MIN':
                minEdge = float(fields[2])
                writeln('  "%s_shortest_edge_length": %s,' % (prefix, minEdge))
            elif fields[1] == 'EDGE_AVG':
                avgEdge = float(fields[2])
                writeln('  "%s_average_edge_length": %s,' % (prefix, avgEdge))
            elif fields[1] == 'EDGE_MAX':
                maxEdge = float(fields[2])
                writeln('  "%s_longest_edge_length": %s,' % (prefix, maxEdge))
            if fields[1] == 'ANGLE_MIN':
                minAngle = float(fields[2])
                writeln('  "%s_smallest_angle": %s,' % (prefix, minAngle))
            elif fields[1] == 'ANGLE_AVG':
                avgAngle = float(fields[2])
                writeln('  "%s_average_angle": %s,' % (prefix, avgAngle))
            elif fields[1] == 'ANGLE_MAX':
                maxAngle = float(fields[2])
                writeln('  "%s_largest_angle": %s,' % (prefix, maxAngle))
            elif fields[1] == 'SMALLEST_SEPARATION':
                sep = float(fields[2])
                writeln('  "%s_smallest_atom_separation": %s,' % (prefix, sep))
            elif fields[1] == 'DEGREES_OF_FREEDOM':
                dof = int(fields[2])
                writeln('  "%s_degrees_of_freedom": %s,' % (prefix, dof))

    writeln('  "%s_atoms": %s,' % (prefix, nodes))
    writeln('  "%s_edges": %s,' % (prefix, edges))

def serialized_net(net, asCGD=False, writeFullCell=False, prefix=''):
    stringWriter = java.io.StringWriter()
    writer = java.io.PrintWriter(stringWriter)
    net.writeEmbedding(writer, asCGD, writeFullCell, prefix)
    return stringWriter.toString()

def verify_embedding(graph, nodeToName, finder, embedder):
    gram = embedder.gramMatrix

    if gram.get(0, 0).doubleValue() < 0.001:
        return False

    if gram.getSubMatrix(0, 0, 2, 2).determinant().doubleValue() < 0.001:
        return False

    if gram.determinant().doubleValue() < 0.001:
        return False

    if not graph.isStable():
        return True

    net = pgraphs.embed.ProcessedNet(graph, 'X', nodeToName, finder, embedder)
    cgd = serialized_net(net, asCGD=True)
    test = pgraphs.io.NetParser.stringToNet(cgd)

    if not test.minimalImage().equals(graph):
        return False

    return True

def coordination_sequence(net, v, n):
    cs = net.coordinationSequence(v)
    cs.next()

    out = []
    for i in range(n):
        out.append(cs.next())

    return out

if __name__ == "__main__":
    run()
<|endoftext|>


<commit_before>#  File       :	binary.py
#  Purpose    :	Write a program that will convert a binary number, represented as a string (e.g. '101010'), to its decimal equivalent using first principles
#  Programmer : Amal Shehu
#  Course     :	Exercism
#  Date       :	Saturday 12 October 2016, 03:02 PM

def parse_binary(bin_value):
    sum_ = 0
    for i in bin_value:
        sum_ = sum_ * 2 + int(i)
    return sum_
<commit_msg>Add regex<commit_after>#  File       :	binary.py
#  Purpose    :	Write a program that will convert a binary number, represented as a string (e.g. '101010'), to its decimal equivalent using first principles
#  Programmer : Amal Shehu
#  Course     :	Exercism
#  Date       :	Saturday 12 October 2016, 03:02 PM
import re
def parse_binary(bin_value):
    sum_ = 0
    for i in bin_value:
        matches = re.match('[01]*$',bin_value)
        if not matches:
            print i
            raise ValueError('Not a valid binary number.')
        sum_ = sum_ * 2 + int(i)
    return sum_

print parse_binary('1011')
<|endoftext|>


<commit_before>#
# Authors: Barney Desmond   <barneydesmond@gmail.com>
#          Katie McLaughlin <katie@glasnt.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Publish a sample (metric data point) to Vaultaire
"""

from ceilometer import publisher

from marquise import Marquise

from pprint import pformat
import datetime
from dateutil.parser import parse
from dateutil.tz import tzutc

from ceilometer.openstack.common.gettextutils import _
from ceilometer.openstack.common import log

LOG = log.getLogger(__name__)

def sanitize(v):
    """Sanitize a value into something that Marquise can use. This weeds
    out None keys/values, and ensures that timestamps are consistently
    formatted.
    """
    if v is None:
        return ""
    if v in (True,False):
        return 1 if v is True else 0
    if type(v) is unicode:
        v = str(v)
    if type(v) is str: # XXX: will be incorrect/ambiguous in Python 3.
        v = v.replace(":","-")
        v = v.replace(",","-")
    return v

def flatten(n, prefix=""):
    """Take a (potentially) nested dictionary and flatten it into a single
    level. Also remove any keys/values that Marquise/Vaultaire can't handle.
    """
    flattened_dict = {}
    for k,v in n.items():
        k = sanitize(k)

        # Vaultaire doesn't care about generated URLs for Ceilometer
        # API references.
        if str(k) == "links":
            continue
        if str(k).endswith('_url'):
            continue

        # Vaultaire doesn't want values if they have no contents.
        if v is None:
            continue

        # If key has a parent, concatenate it into the new keyname.
        if prefix:
            k = "{}-{}".format(prefix,k)

        # This was previously a check for __iter__, but strings have those now,
        # so let's just check for dict-ness instead. No good on lists anyway.
        if type(v) is not dict:
            v = sanitize(v)
            flattened_dict[k] = v
        else:
            flattened_dict.update(flatten(v, k))
    return flattened_dict

def sanitize_timestamp(v):
    """Convert a timestamp value into a standard form. Does no exception
    handling, as we really want to know if this fails."""
    if type(v) is unicode:
        v = str(v)
    # Try and take a value and use dateutil to parse it.  If there's no TZ
    # spec in the string, assume it's UTC because that's what Ceilometer
    # uses.
    # Eg. 2014-08-10T12:14:13Z      # timezone-aware
    # Eg. 2014-08-10 12:14:13       # timezone-naive
    # Eg. 2014-08-10 12:14:13+1000  # timezone-aware
    NANOSECONDS_PER_SECOND = 10**9
    if type(v) is datetime.datetime:
        timestamp = v
    else: # We have a string.
        timestamp = parse(v)
        if timestamp.tzinfo is None:
            timestamp = timestamp.replace(tzinfo=tzutc())
    # If we get here, we've successfully grabbed a datetime.
    epoch = datetime.datetime(1970, 1, 1, tzinfo=tzutc())
    time_since_epoch = (timestamp - epoch).total_seconds() # total_seconds() is in Py2.7 and later.
    return int(time_since_epoch * NANOSECONDS_PER_SECOND)

# pylint: disable=too-few-public-methods
class VaultairePublisher(publisher.PublisherBase):
    # We will delete keys which are likely to change frequently, or that
    # are otherwise undesirable.
    keys_to_delete = [
        "timestamp",
        "volume",
    ]

    """Implements the Publisher interface for Ceilometer."""
    def __init__(self, parsed_url):
        super(VaultairePublisher, self).__init__(parsed_url)

        self.marquise = None
        namespace = parsed_url.netloc
        if not namespace:
            LOG.error(_('The namespace for the vaultaire publisher is required'))
            return

        LOG.info(_("Marquise loaded with namespace %s" % namespace))
        self.marquise = Marquise(namespace)

    def publish_samples(self, dummy_context, samples):
        """Reconstruct a metering message for publishing to Vaultaire via Marquise

        :param dummy_context: Execution context from the service or RPC call. (Unused)
        :param samples: Samples from pipeline after transformation
        """
        if self.marquise:
            marq = self.marquise
            for sample in samples:
                sample = sample.as_dict()

                # Generate the unique identifer for the sample
                identifier = sample["resource_id"] + sample["project_id"] + \
                             sample["name"] + sample["type"] + sample["unit"]
                address = Marquise.hash_identifier(identifier)

                # Sanitize timestamp (will parse timestamp to nanoseconds since epoch)
                timestamp = sanitize_timestamp(sample["timestamp"])

                # Our payload is the volume (later parsed to "counter_volume" in ceilometer)
                payload = sanitize(sample["volume"])

                # Rebuild the sample as a source dict
                sourcedict = dict(sample)

                # Vaultaire cares about the datatype of the payload
                if type(payload) == float:
                    sourcedict["_float"] = 1
                elif type(payload) == str:
                    sourcedict["_extended"] = 1

                # Cast unit as a special metadata type
                sourcedict["_unit"] = sanitize(sourcedict.pop("unit"))

                # If it's a cumulative value, we need to tell vaultaire
                if sourcedict["type"] == "cumulative":
                    sourcedict["_counter"] = 1

                # Cast Identifier sections with unique names, in case of metadata overlap
                sourcedict["counter_name"] = sanitize(sourcedict.pop("name"))
                sourcedict["counter_type"] = sanitize(sourcedict.pop("type"))

                self._remove_extraneous(sourcedict)

                # Remove the original resource_metadata and substitute our own flattened version
                sourcedict.update(flatten(sourcedict.pop("resource_metadata")))

                # Finally, send it all off to marquise
                LOG.info(_("Marquise Send Simple: %s %s %s") % (address, timestamp, payload))
                marq.send_simple(address=address, timestamp=timestamp, value=payload)

                LOG.debug(_("Marquise Update Source Dict for %s - %s") % (address, pformat(sourcedict)))
                marq.update_source(address, sourcedict)

    def _remove_extraneous(self, sourcedict):
        for k in keys_to_delete:
            try:
                del[sourcedict[k]]
            except KeyError:
                pass
<commit_msg>Delete more cruft<commit_after>#
# Authors: Barney Desmond   <barneydesmond@gmail.com>
#          Katie McLaughlin <katie@glasnt.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Publish a sample (metric data point) to Vaultaire
"""

from ceilometer import publisher

from marquise import Marquise

from pprint import pformat
import datetime
from dateutil.parser import parse
from dateutil.tz import tzutc

from ceilometer.openstack.common.gettextutils import _
from ceilometer.openstack.common import log

LOG = log.getLogger(__name__)

def sanitize(v):
    """Sanitize a value into something that Marquise can use. This weeds
    out None keys/values, and ensures that timestamps are consistently
    formatted.
    """
    if v is None:
        return ""
    if v in (True,False):
        return 1 if v is True else 0
    if type(v) is unicode:
        v = str(v)
    if type(v) is str: # XXX: will be incorrect/ambiguous in Python 3.
        v = v.replace(":","-")
        v = v.replace(",","-")
    return v

def flatten(n, prefix=""):
    """Take a (potentially) nested dictionary and flatten it into a single
    level. Also remove any keys/values that Marquise/Vaultaire can't handle.
    """
    flattened_dict = {}
    for k,v in n.items():
        k = sanitize(k)

        # Vaultaire doesn't care about generated URLs for Ceilometer
        # API references.
        if str(k) == "links":
            continue
        if str(k).endswith('_url'):
            continue

        # Vaultaire doesn't want values if they have no contents.
        if v is None:
            continue

        # If key has a parent, concatenate it into the new keyname.
        if prefix:
            k = "{}-{}".format(prefix,k)

        # This was previously a check for __iter__, but strings have those now,
        # so let's just check for dict-ness instead. No good on lists anyway.
        if type(v) is not dict:
            v = sanitize(v)
            flattened_dict[k] = v
        else:
            flattened_dict.update(flatten(v, k))
    return flattened_dict

def sanitize_timestamp(v):
    """Convert a timestamp value into a standard form. Does no exception
    handling, as we really want to know if this fails."""
    if type(v) is unicode:
        v = str(v)
    # Try and take a value and use dateutil to parse it.  If there's no TZ
    # spec in the string, assume it's UTC because that's what Ceilometer
    # uses.
    # Eg. 2014-08-10T12:14:13Z      # timezone-aware
    # Eg. 2014-08-10 12:14:13       # timezone-naive
    # Eg. 2014-08-10 12:14:13+1000  # timezone-aware
    NANOSECONDS_PER_SECOND = 10**9
    if type(v) is datetime.datetime:
        timestamp = v
    else: # We have a string.
        timestamp = parse(v)
        if timestamp.tzinfo is None:
            timestamp = timestamp.replace(tzinfo=tzutc())
    # If we get here, we've successfully grabbed a datetime.
    epoch = datetime.datetime(1970, 1, 1, tzinfo=tzutc())
    time_since_epoch = (timestamp - epoch).total_seconds() # total_seconds() is in Py2.7 and later.
    return int(time_since_epoch * NANOSECONDS_PER_SECOND)

# pylint: disable=too-few-public-methods
class VaultairePublisher(publisher.PublisherBase):
    # We will delete keys which are likely to change frequently, or that
    # are otherwise undesirable.
    keys_to_delete = [
        "timestamp",
        "volume",
        "created_at",
        "updated_at",
    ]

    """Implements the Publisher interface for Ceilometer."""
    def __init__(self, parsed_url):
        super(VaultairePublisher, self).__init__(parsed_url)

        self.marquise = None
        namespace = parsed_url.netloc
        if not namespace:
            LOG.error(_('The namespace for the vaultaire publisher is required'))
            return

        LOG.info(_("Marquise loaded with namespace %s" % namespace))
        self.marquise = Marquise(namespace)

    def publish_samples(self, dummy_context, samples):
        """Reconstruct a metering message for publishing to Vaultaire via Marquise

        :param dummy_context: Execution context from the service or RPC call. (Unused)
        :param samples: Samples from pipeline after transformation
        """
        if self.marquise:
            marq = self.marquise
            for sample in samples:
                sample = sample.as_dict()

                # Generate the unique identifer for the sample
                identifier = sample["resource_id"] + sample["project_id"] + \
                             sample["name"] + sample["type"] + sample["unit"]
                address = Marquise.hash_identifier(identifier)

                # Sanitize timestamp (will parse timestamp to nanoseconds since epoch)
                timestamp = sanitize_timestamp(sample["timestamp"])

                # Our payload is the volume (later parsed to "counter_volume" in ceilometer)
                payload = sanitize(sample["volume"])

                # Rebuild the sample as a source dict
                sourcedict = dict(sample)

                # Vaultaire cares about the datatype of the payload
                if type(payload) == float:
                    sourcedict["_float"] = 1
                elif type(payload) == str:
                    sourcedict["_extended"] = 1

                # Cast unit as a special metadata type
                sourcedict["_unit"] = sanitize(sourcedict.pop("unit"))

                # If it's a cumulative value, we need to tell vaultaire
                if sourcedict["type"] == "cumulative":
                    sourcedict["_counter"] = 1

                # Cast Identifier sections with unique names, in case of metadata overlap
                sourcedict["counter_name"] = sanitize(sourcedict.pop("name"))
                sourcedict["counter_type"] = sanitize(sourcedict.pop("type"))

                self._remove_extraneous(sourcedict)

                # Remove the original resource_metadata and substitute our own flattened version
                sourcedict.update(flatten(sourcedict.pop("resource_metadata")))

                # Finally, send it all off to marquise
                LOG.info(_("Marquise Send Simple: %s %s %s") % (address, timestamp, payload))
                marq.send_simple(address=address, timestamp=timestamp, value=payload)

                LOG.debug(_("Marquise Update Source Dict for %s - %s") % (address, pformat(sourcedict)))
                marq.update_source(address, sourcedict)

    def _remove_extraneous(self, sourcedict):
        for k in keys_to_delete:
            try:
                del[sourcedict[k]]
            except KeyError:
                pass
<|endoftext|>


<commit_before># Dissemin: open access policy enforcement tool
# Copyright (C) 2014 Antonin Delpeuch
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Affero General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
#

from papers.models import Paper
from publishers.tests.test_romeo import RomeoAPIStub

class TestJournalPage():

    def test_escaping(self, db, check_page):
        # issue #115
        # in the meantime the journal has been deleted from sherpa
        api = RomeoAPIStub()
        journal = api.fetch_journal({'issn': '0302-9743'})
        # Small hack to make the journal appear in the publisher's journal list
        journal.update_stats()
        journal.stats.num_tot = 1
        journal.stats.save()
        publisher = journal.publisher
        check_page(200, 'publisher', kwargs={'pk': publisher.pk, 'slug': publisher.slug})

    def test_publisher_url(self, db, check_page):
        api = RomeoAPIStub()
        api.fetch_journal({'issn':'1860-949X'})
        p = Paper.create_by_doi('10.1007/978-3-642-14363-2_7')
        for publi in p.publications:
            check_page(200, 'publisher', kwargs={'pk': publi.publisher_id, 'slug': publi.publisher.slug})
<commit_msg>Test shall use mocking<commit_after># Dissemin: open access policy enforcement tool
# Copyright (C) 2014 Antonin Delpeuch
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Affero General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
#

import pytest

from papers.models import Paper
from publishers.tests.test_romeo import RomeoAPIStub

class TestJournalPage():

    def test_escaping(self, db, check_page):
        # issue #115
        # in the meantime the journal has been deleted from sherpa
        api = RomeoAPIStub()
        journal = api.fetch_journal({'issn': '0302-9743'})
        # Small hack to make the journal appear in the publisher's journal list
        journal.update_stats()
        journal.stats.num_tot = 1
        journal.stats.save()
        publisher = journal.publisher
        check_page(200, 'publisher', kwargs={'pk': publisher.pk, 'slug': publisher.slug})

    @pytest.mark.usefixtures('mock_doi')
    def test_publisher_url(self, db, check_page):
        api = RomeoAPIStub()
        api.fetch_journal({'issn':'1860-949X'})
        p = Paper.create_by_doi('10.1007/978-3-642-14363-2_7')
        for publi in p.publications:
            check_page(200, 'publisher', kwargs={'pk': publi.publisher_id, 'slug': publi.publisher.slug})
<|endoftext|>


<commit_before>"""Module for linters."""
# The less we need to code, the better!
# pylint: disable=too-few-public-methods
import re

from .base import Linter, LinterOutput

__all__ = ('Isort', 'Pycodestyle', 'Pydocstyle', 'Pyflakes', 'Pylint',
           'RadonCC', 'RadonMI')

class Isort(Linter):
    """Isort parser."""

    name = 'isort'

    def parse(self, lines):
        """Get full path and message from each output line."""
        # E.g. "ERROR: /my/path/main.py Imports are incorrectly sorted."
        pattern = re.compile(r'''
                             ^.+?
                             :\ (?P<full_path>.+?)
                             \ (?P<msg>.+)$''', re.VERBOSE)
        return self._parse_by_pattern(lines, pattern)

    def _create_output_from_match(self, match_result):
        """As isort outputs full path, we change it to relative path."""
        full_path = match_result['full_path']
        path = self._get_relative_path(full_path)
        return LinterOutput(self.name, path, match_result['msg'])

class Pycodestyle(Linter):
    """Pycodestyle parser."""

    name = 'pycodestyle'

    def parse(self, lines):
        """Get :class:`base.Result` parameters using regex."""
        pattern = re.compile(r'''
                             ^(?P<path>.+?)
                             :(?P<line_nr>\d+?)
                             :(?P<col>\d+?)
                             :\ (?P<msg>.+)$''', re.VERBOSE)
        return self._parse_by_pattern(lines, pattern)

class Pydocstyle(Linter):
    """Pydocstyle parser."""

    name = 'pydocstyle'

    def parse(self, lines):
        """Get :class:`base.Result` parameters using regex.

        There are 2 lines for each pydocstyle result:
            1. Filename and line number;
            2. Message for the problem found.
        """
        patterns = [re.compile(r'^(.+?):(\d+)'),
                    re.compile(r'^\s+(.+)$')]
        for i, line in enumerate(lines):
            if i % 2 == 0:
                path, line_nr = patterns[0].match(line).groups()
            else:
                msg = patterns[1].match(line).group(1)
                yield LinterOutput(self.name, path, msg, line_nr)

class Pyflakes(Linter):
    """Pyflakes parser."""

    name = 'pyflakes'

    def parse(self, lines):
        """Get :class:`base.Result` parameters using regex."""
        pattern = re.compile(r'''
                             ^(?P<path>.+?)
                             :(?P<line_nr>\d+?)
                             :\ (?P<msg>.+)$''', re.VERBOSE)
        return self._parse_by_pattern(lines, pattern)

class Pylint(Linter):
    """Pylint parser."""

    name = 'pylint'

    def parse(self, lines):
        """Get :class:`base.Result` parameters using regex."""
        pattern = re.compile(r"""^(?P<path>.+?)
                                 :(?P<msg>.+)
                                 :(?P<line_nr>\d+?)
                                 :(?P<col>\d+?)$""", re.VERBOSE)
        return self._parse_by_pattern(lines, pattern)

class RadonCC(Linter):
    """Parser for radon ciclomatic complexity."""

    name = 'radon cc'

    def parse(self, lines):
        """Get :class:`base.Result` parameters using regex.

        The output has one line with the file path followed by others with
        one problem per line.
        """
        # E.g. 'relative/path/to/file.py'
        pattern_path = re.compile(r'^(\S.*$)')
        # E.g. '    C 19:0 RadonCC - A'
        pattern_result = re.compile(r'\s+\w (\d+):(\d+) (.+)$')
        path = None
        for line in lines:
            match = pattern_path.match(line)
            if match:
                # We have the file path. Skip the remainder of the loop
                path = line
                continue
            match = pattern_result.match(line)
            if match:
                # Output found for the file stored in ``path``.
                line_nr, col, msg = match.groups()
                yield LinterOutput(self.name, path, msg, line_nr, col)

class RadonMI(Linter):
    """Parser for radon maintainability index."""

    name = 'radon mi'

    def parse(self, lines):
        """Get :class:`base.Result` parameters using regex."""
        pattern = re.compile(r'''
                             ^(?P<path>.+)
                             \ -\ (?P<msg>[A-F])$''', re.VERBOSE)
        return self._parse_by_pattern(lines, pattern)
<commit_msg>Add mypy support<commit_after>"""Module for linters."""
# The less we need to code, the better!
# pylint: disable=too-few-public-methods
import re

from .base import Linter, LinterOutput

__all__ = ('Isort', 'Pycodestyle', 'Pydocstyle', 'Pyflakes', 'Pylint',
           'RadonCC', 'RadonMI')

class Isort(Linter):
    """Isort parser."""

    name = 'isort'

    def parse(self, lines):
        """Get full path and message from each output line."""
        # E.g. "ERROR: /my/path/main.py Imports are incorrectly sorted."
        pattern = re.compile(r'''
                             ^.+?
                             :\ (?P<full_path>.+?)
                             \ (?P<msg>.+)$''', re.VERBOSE)
        return self._parse_by_pattern(lines, pattern)

    def _create_output_from_match(self, match_result):
        """As isort outputs full path, we change it to relative path."""
        full_path = match_result['full_path']
        path = self._get_relative_path(full_path)
        return LinterOutput(self.name, path, match_result['msg'])

class Pycodestyle(Linter):
    """Pycodestyle parser."""

    name = 'pycodestyle'

    def parse(self, lines):
        """Get :class:`base.Result` parameters using regex."""
        pattern = re.compile(r'''
                             ^(?P<path>.+?)
                             :(?P<line_nr>\d+?)
                             :(?P<col>\d+?)
                             :\ (?P<msg>.+)$''', re.VERBOSE)
        return self._parse_by_pattern(lines, pattern)

class Mypy(Linter):
    """Mypy parser."""

    name = 'mypy'

    def parse(self, lines):
        """Get :class:`base.Result` parameters using regex."""
        pattern = re.compile(r'''
                             ^(?P<path>.+?)
                             :(?P<line_nr>\d+?)
                             :\ (?P<msg>.+)$''', re.VERBOSE)
        return self._parse_by_pattern(lines, pattern)

class Pydocstyle(Linter):
    """Pydocstyle parser."""

    name = 'pydocstyle'

    def parse(self, lines):
        """Get :class:`base.Result` parameters using regex.

        There are 2 lines for each pydocstyle result:
            1. Filename and line number;
            2. Message for the problem found.
        """
        patterns = [re.compile(r'^(.+?):(\d+)'),
                    re.compile(r'^\s+(.+)$')]
        for i, line in enumerate(lines):
            if i % 2 == 0:
                path, line_nr = patterns[0].match(line).groups()
            else:
                msg = patterns[1].match(line).group(1)
                yield LinterOutput(self.name, path, msg, line_nr)

class Pyflakes(Linter):
    """Pyflakes parser."""

    name = 'pyflakes'

    def parse(self, lines):
        """Get :class:`base.Result` parameters using regex."""
        pattern = re.compile(r'''
                             ^(?P<path>.+?)
                             :(?P<line_nr>\d+?)
                             :\ (?P<msg>.+)$''', re.VERBOSE)
        return self._parse_by_pattern(lines, pattern)

class Pylint(Linter):
    """Pylint parser."""

    name = 'pylint'

    def parse(self, lines):
        """Get :class:`base.Result` parameters using regex."""
        pattern = re.compile(r"""^(?P<path>.+?)
                                 :(?P<msg>.+)
                                 :(?P<line_nr>\d+?)
                                 :(?P<col>\d+?)$""", re.VERBOSE)
        return self._parse_by_pattern(lines, pattern)

class RadonCC(Linter):
    """Parser for radon ciclomatic complexity."""

    name = 'radon cc'

    def parse(self, lines):
        """Get :class:`base.Result` parameters using regex.

        The output has one line with the file path followed by others with
        one problem per line.
        """
        # E.g. 'relative/path/to/file.py'
        pattern_path = re.compile(r'^(\S.*$)')
        # E.g. '    C 19:0 RadonCC - A'
        pattern_result = re.compile(r'\s+\w (\d+):(\d+) (.+)$')
        path = None
        for line in lines:
            match = pattern_path.match(line)
            if match:
                # We have the file path. Skip the remainder of the loop
                path = line
                continue
            match = pattern_result.match(line)
            if match:
                # Output found for the file stored in ``path``.
                line_nr, col, msg = match.groups()
                yield LinterOutput(self.name, path, msg, line_nr, col)

class RadonMI(Linter):
    """Parser for radon maintainability index."""

    name = 'radon mi'

    def parse(self, lines):
        """Get :class:`base.Result` parameters using regex."""
        pattern = re.compile(r'''
                             ^(?P<path>.+)
                             \ -\ (?P<msg>[A-F])$''', re.VERBOSE)
        return self._parse_by_pattern(lines, pattern)
<|endoftext|>


<commit_before>import json
import sys
import traceback
import requests

class Updater():
    """ This class enables updating the index of available images.
    """

    def __init__(self,global_conf_filename):
        """ Creates an `Updater` based on the values in the json file `global_conf_file`.

        :param global_conf_file: path to json file with parameters configuration.
        """
        self.indexer = None
        self.global_conf_filename = global_conf_filename
        self.global_conf = json.load(open(self.global_conf_filename,'rt'))
        self.init_indexer()
        self.has_indexed = False
        # could/should be loaded from conf
        self.refresh_URL = 'http://127.0.0.1:5000/cu_image_search/refresh'

    def init_indexer(self):
        """ Initialize `indexer` from `global_conf['UP_indexer']` value.

        Currently supported indexer types are:
        - hbase_indexer
        """
        field = 'UP_indexer'
        if field not in self.global_conf:
            raise ValueError("[Updater: error] "+field+" is not defined in configuration file.")
        elif self.global_conf[field]=="hbase_indexer":
            from ..indexer.hbase_indexer import HBaseIndexer
            self.indexer = HBaseIndexer(self.global_conf_filename)
        else:
            raise ValueError("[Updater: error] unkown 'indexer' {}.".format(self.global_conf[field]))

    def refresh_API(self):
        """ Forces a refresh of the API index.
        """
        r = requests.get(self.refresh_URL)
        if r.status_code == 200:
            print("[Updater: refresh_API] refresh call return: {}".format(r.content))
        return True

    def run_update(self):
        """ Runs an update.
        """
        try:
            # needs to read table 'escorts_images_updates', rows starting with 'index_update_' and not marked as indexed.
            update_id, str_list_sha1s = self.indexer.get_next_batch()
            if update_id:
                list_sha1s = str_list_sha1s.split(',')
                print("[Updater.run_update: log] Update {} has {} images.".format(update_id, len(list_sha1s)))
                # also get 'info:image' 'info:featnorm_cu', and 'info:hash256_cu' or all self.extractions_columns
                rows_batch = self.indexer.get_columns_from_sha1_rows(list_sha1s, columns=["info:s3_url"])
                # deal with precomputed features (and hashcodes)

                if rows_batch:
                    clean_batch = [(row[0], row[1]["info:s3_url"]) for row in rows_batch]
                    batch_indexed = self.indexer.index_batch_sha1(clean_batch, update_id)
                    if batch_indexed:
                        self.has_indexed = True
                else:
                    print("[Updater.run_update: log] Did not get any urls for this update ({}) images.".format(update_id))
                    print("[Updater.run_update: log] We were looking for the images urls in table {}.".format(self.indexer.table_sha1infos_name))
            else:
                print("[Updater.run_update: log] Nothing to update!")
            # when done mark update_id as processed.
        except timeout:
            print("[Updater.run_update] caught timeout error or TTransportException. Trying to refresh indexer connection pool.")
            self.indexer.pool = happybase.ConnectionPool(size=self.indexer.nb_threads,host=self.indexer.hbase_host)
        except Exception as inst:
            print "[Updater.run_udpate: error] {}".format(inst)
            exc_type, exc_value, exc_traceback = sys.exc_info()
            print "*** print_tb:"
            traceback.print_tb(exc_traceback, file=sys.stdout)
            print "*** print_exception:"
            traceback.print_exception(exc_type, exc_value, exc_traceback, file=sys.stdout)
<commit_msg>import timeout<commit_after>import json
import sys
import traceback
import requests
from socket import timeout

class Updater():
    """ This class enables updating the index of available images.
    """

    def __init__(self,global_conf_filename):
        """ Creates an `Updater` based on the values in the json file `global_conf_file`.

        :param global_conf_file: path to json file with parameters configuration.
        """
        self.indexer = None
        self.global_conf_filename = global_conf_filename
        self.global_conf = json.load(open(self.global_conf_filename,'rt'))
        self.init_indexer()
        self.has_indexed = False
        # could/should be loaded from conf
        self.refresh_URL = 'http://127.0.0.1:5000/cu_image_search/refresh'

    def init_indexer(self):
        """ Initialize `indexer` from `global_conf['UP_indexer']` value.

        Currently supported indexer types are:
        - hbase_indexer
        """
        field = 'UP_indexer'
        if field not in self.global_conf:
            raise ValueError("[Updater: error] "+field+" is not defined in configuration file.")
        elif self.global_conf[field]=="hbase_indexer":
            from ..indexer.hbase_indexer import HBaseIndexer
            self.indexer = HBaseIndexer(self.global_conf_filename)
        else:
            raise ValueError("[Updater: error] unkown 'indexer' {}.".format(self.global_conf[field]))

    def refresh_API(self):
        """ Forces a refresh of the API index.
        """
        r = requests.get(self.refresh_URL)
        if r.status_code == 200:
            print("[Updater: refresh_API] refresh call return: {}".format(r.content))
        return True

    def run_update(self):
        """ Runs an update.
        """
        try:
            # needs to read table 'escorts_images_updates', rows starting with 'index_update_' and not marked as indexed.
            update_id, str_list_sha1s = self.indexer.get_next_batch()
            if update_id:
                list_sha1s = str_list_sha1s.split(',')
                print("[Updater.run_update: log] Update {} has {} images.".format(update_id, len(list_sha1s)))
                # also get 'info:image' 'info:featnorm_cu', and 'info:hash256_cu' or all self.extractions_columns
                rows_batch = self.indexer.get_columns_from_sha1_rows(list_sha1s, columns=["info:s3_url"])
                # deal with precomputed features (and hashcodes)

                if rows_batch:
                    clean_batch = [(row[0], row[1]["info:s3_url"]) for row in rows_batch]
                    batch_indexed = self.indexer.index_batch_sha1(clean_batch, update_id)
                    if batch_indexed:
                        self.has_indexed = True
                else:
                    print("[Updater.run_update: log] Did not get any urls for this update ({}) images.".format(update_id))
                    print("[Updater.run_update: log] We were looking for the images urls in table {}.".format(self.indexer.table_sha1infos_name))
            else:
                print("[Updater.run_update: log] Nothing to update!")
            # when done mark update_id as processed.
        except timeout:
            print("[Updater.run_update] caught timeout error or TTransportException. Trying to refresh indexer connection pool.")
            self.indexer.pool = happybase.ConnectionPool(size=self.indexer.nb_threads,host=self.indexer.hbase_host)
        except Exception as inst:
            print "[Updater.run_udpate: error] {}".format(inst)
            exc_type, exc_value, exc_traceback = sys.exc_info()
            print "*** print_tb:"
            traceback.print_tb(exc_traceback, file=sys.stdout)
            print "*** print_exception:"
            traceback.print_exception(exc_type, exc_value, exc_traceback, file=sys.stdout)
<|endoftext|>