<commit_before>#
#  Created by Greg Landrum (greg.landrum@t5informatics.com), Feb 2016
#
from flask import Flask,make_response,request
from rdkit import Chem
from rdkit import rdBase
from rdkit.Chem import rdDepictor
from rdkit.Chem.Draw import rdMolDraw2D
import json

app = Flask(__name__)

@app.route('/')
def health():
    res = dict(rdkitVersion=rdBase.rdkitVersion,boostVersion=rdBase.boostVersion)
    return json.dumps(res)

@app.route('/canon_smiles/<smiles>')
def canon_smiles(smiles):
    " returns canonical SMILES for an input SMILES "
    return Chem.CanonSmiles(smiles)

def _prepareMol(mol,kekulize):
    mc = Chem.Mol(mol.ToBinary())
    if kekulize:
        try:
            Chem.Kekulize(mc)
        except:
            mc = Chem.Mol(mol.ToBinary())
    if not mc.GetNumConformers():
        rdDepictor.Compute2DCoords(mc)
    return mc
def _moltosvg(mol,molSize=(450,200),kekulize=True,drawer=None,**kwargs):
    mc = _prepareMol(mol,kekulize)
    if drawer is None:
        drawer = rdMolDraw2D.MolDraw2DSVG(molSize[0],molSize[1])
    drawer.DrawMolecule(mc,**kwargs)
    drawer.FinishDrawing()
    svg = drawer.GetDrawingText()
    #return svg
    return svg.replace('svg:','')
def _moltopng(mol,molSize=(450,200),kekulize=True,drawer=None,**kwargs):
    mc = _prepareMol(mol,kekulize)
    if drawer is None:
        drawer = rdMolDraw2D.MolDraw2DCairo(molSize[0],molSize[1])
    drawer.DrawMolecule(mc,**kwargs)
    drawer.FinishDrawing()
    return drawer.GetDrawingText()

def _render(mol,renderer,size=(150,100),**kwargs):
    sz = int(request.values.get('w',size[0])),int(request.values.get('h',size[1]))
    return renderer(mol,molSize=sz,**kwargs)

@app.route('/mol_to_png/mol.png', methods=['GET', 'POST'])
def mol_to_png():
    " returns a PNG for a mol block "
    molblock = request.values.get('mol')
    m = Chem.MolFromMolBlock(molblock)
    response = make_response(_render(m,_moltopng))
    response.headers['Content-Type'] = 'image/png'
    return response
@app.route('/mol_to_svg/mol.svg', methods=['GET', 'POST'])
def mol_to_svg():
    " returns an SVG for a mol block "
    molblock = request.values.get('mol')
    m = Chem.MolFromMolBlock(molblock)
    response = make_response(_render(m,_moltosvg))
    return response

@app.route('/smiles_to_png/<smiles>.png', methods=['GET', 'POST'])
def smiles_to_png(smiles):
    " returns a PNG for a SMILES "
    m = Chem.MolFromSmiles(smiles)
    response = make_response(_render(m,_moltopng))
    response.headers['Content-Type'] = 'image/png'
    return response

@app.route('/smiles_to_svg/<smiles>.svg', methods=['GET', 'POST'])
def smiles_to_svg(smiles):
    " returns an SVG for a SMILES "
    m = Chem.MolFromSmiles(smiles)
    response = make_response(_render(m,_moltosvg))
    #response.headers['Content-Type'] = 'image/svg+xml' # SVG doesn't render properly in chrome if we send this
    return response

if __name__ == '__main__':
    # FIX: turn this off pre-deployment
    app.debug  = True
    app.run()
<commit_msg>switch to using the new PrepareMolForDrawing() that is on master<commit_after>#
#  Created by Greg Landrum (greg.landrum@t5informatics.com), Feb 2016
#
from flask import Flask,make_response,request
from rdkit import Chem
from rdkit import rdBase
from rdkit.Chem import rdDepictor
from rdkit.Chem.Draw import rdMolDraw2D
import json

app = Flask(__name__)

@app.route('/')
def health():
    res = dict(rdkitVersion=rdBase.rdkitVersion,boostVersion=rdBase.boostVersion)
    return json.dumps(res)

@app.route('/canon_smiles/<smiles>')
def canon_smiles(smiles):
    " returns canonical SMILES for an input SMILES "
    return Chem.CanonSmiles(smiles)

def _moltosvg(mol,molSize=(450,200),kekulize=True,drawer=None,**kwargs):
    mc = rdMolDraw2D.PrepareMolForDrawing(mol,kekulize=kekulize)
    if drawer is None:
        drawer = rdMolDraw2D.MolDraw2DSVG(molSize[0],molSize[1])
    drawer.DrawMolecule(mc,**kwargs)
    drawer.FinishDrawing()
    svg = drawer.GetDrawingText()
    #return svg
    return svg.replace('svg:','')
def _moltopng(mol,molSize=(450,200),kekulize=True,drawer=None,**kwargs):
    mc = rdMolDraw2D.PrepareMolForDrawing(mol,kekulize=kekulize)
    if drawer is None:
        drawer = rdMolDraw2D.MolDraw2DCairo(molSize[0],molSize[1])
    drawer.DrawMolecule(mc,**kwargs)
    drawer.FinishDrawing()
    return drawer.GetDrawingText()

def _render(mol,renderer,size=(150,100),**kwargs):
    sz = int(request.values.get('w',size[0])),int(request.values.get('h',size[1]))
    return renderer(mol,molSize=sz,**kwargs)

@app.route('/mol_to_png/mol.png', methods=['GET', 'POST'])
def mol_to_png():
    " returns a PNG for a mol block "
    molblock = request.values.get('mol')
    m = Chem.MolFromMolBlock(molblock)
    response = make_response(_render(m,_moltopng))
    response.headers['Content-Type'] = 'image/png'
    return response
@app.route('/mol_to_svg/mol.svg', methods=['GET', 'POST'])
def mol_to_svg():
    " returns an SVG for a mol block "
    molblock = request.values.get('mol')
    m = Chem.MolFromMolBlock(molblock)
    response = make_response(_render(m,_moltosvg))
    return response

@app.route('/smiles_to_png/<smiles>.png', methods=['GET', 'POST'])
def smiles_to_png(smiles):
    " returns a PNG for a SMILES "
    m = Chem.MolFromSmiles(smiles)
    response = make_response(_render(m,_moltopng))
    response.headers['Content-Type'] = 'image/png'
    return response

@app.route('/smiles_to_svg/<smiles>.svg', methods=['GET', 'POST'])
def smiles_to_svg(smiles):
    " returns an SVG for a SMILES "
    m = Chem.MolFromSmiles(smiles)
    response = make_response(_render(m,_moltosvg))
    #response.headers['Content-Type'] = 'image/svg+xml' # SVG doesn't render properly in chrome if we send this
    return response

if __name__ == '__main__':
    # FIX: turn this off pre-deployment
    app.debug  = True
    app.run()
<|endoftext|>


<commit_before># Copyright (c) 2013, Web Notes Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt

from __future__ import unicode_literals
"""
	Execute Patch Files

	To run directly

	python lib/wnf.py patch patch1, patch2 etc
	python lib/wnf.py patch -f patch1, patch2 etc

	where patch1, patch2 is module name
"""
import frappe, os, frappe.permissions

class PatchError(Exception): pass

def run_all():
	"""run all pending patches"""
	executed = [p[0] for p in frappe.db.sql("""select patch from `tabPatch Log`""")]

	for patch in get_all_patches():
		if patch and (patch not in executed):
			if not run_single(patchmodule = patch):
				log(patch + ': failed: STOPPED')
				raise PatchError(patch)

def get_all_patches():
	patches = []
	for app in frappe.get_installed_apps():
		# 3-to-4 fix
		if app=="webnotes":
			app="frappe"
		patches.extend(frappe.get_file_items(frappe.get_pymodule_path(app, "patches.txt")))

	return patches

def reload_doc(args):
	import frappe.modules
	run_single(method = frappe.modules.reload_doc, methodargs = args)

def run_single(patchmodule=None, method=None, methodargs=None, force=False):
	from frappe import conf

	# don't write txt files
	conf.developer_mode = 0

	if force or method or not executed(patchmodule):
		return execute_patch(patchmodule, method, methodargs)
	else:
		return True

def execute_patch(patchmodule, method=None, methodargs=None):
	"""execute the patch"""
	block_user(True)
	frappe.db.begin()

	try:
		log('Executing {patch} in {site} ({db})'.format(patch=patchmodule or str(methodargs),
			site=frappe.local.site, db=frappe.db.cur_db_name))
		if patchmodule:
			if patchmodule.startswith("execute:"):
				exec patchmodule.split("execute:")[1] in globals()
			else:
				frappe.get_attr(patchmodule.split()[0] + ".execute")()
			update_patch_log(patchmodule)
		elif method:
			method(**methodargs)

	except Exception:
		frappe.db.rollback()
		raise

	else:
		frappe.db.commit()
		block_user(False)
		log('Success')

	return True

def update_patch_log(patchmodule):
	"""update patch_file in patch log"""
	frappe.get_doc({"doctype": "Patch Log", "patch": patchmodule}).insert()

def executed(patchmodule):
	"""return True if is executed"""
	done = frappe.db.get_value("Patch Log", {"patch": patchmodule})
	# if done:
	# 	print "Patch %s already executed in %s" % (patchmodule, frappe.db.cur_db_name)
	return done

def block_user(block):
	"""stop/start execution till patch is run"""
	frappe.local.flags.in_patch = block
	frappe.db.begin()
	msg = "Patches are being executed in the system. Please try again in a few moments."
	frappe.db.set_global('__session_status', block and 'stop' or None)
	frappe.db.set_global('__session_status_message', block and msg or None)
	frappe.db.commit()

def check_session_stopped():
	if frappe.db.get_global("__session_status")=='stop':
		frappe.msgprint(frappe.db.get_global("__session_status_message"))
		raise frappe.SessionStopped('Session Stopped')

def log(msg):
	print msg
<commit_msg>bypass shopping_cart in patch handler<commit_after># Copyright (c) 2013, Web Notes Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt

from __future__ import unicode_literals
"""
	Execute Patch Files

	To run directly

	python lib/wnf.py patch patch1, patch2 etc
	python lib/wnf.py patch -f patch1, patch2 etc

	where patch1, patch2 is module name
"""
import frappe, os, frappe.permissions

class PatchError(Exception): pass

def run_all():
	"""run all pending patches"""
	executed = [p[0] for p in frappe.db.sql("""select patch from `tabPatch Log`""")]

	for patch in get_all_patches():
		if patch and (patch not in executed):
			if not run_single(patchmodule = patch):
				log(patch + ': failed: STOPPED')
				raise PatchError(patch)

def get_all_patches():
	patches = []
	for app in frappe.get_installed_apps():
		if app == "shopping_cart":
			continue
		# 3-to-4 fix
		if app=="webnotes":
			app="frappe"
		patches.extend(frappe.get_file_items(frappe.get_pymodule_path(app, "patches.txt")))

	return patches

def reload_doc(args):
	import frappe.modules
	run_single(method = frappe.modules.reload_doc, methodargs = args)

def run_single(patchmodule=None, method=None, methodargs=None, force=False):
	from frappe import conf

	# don't write txt files
	conf.developer_mode = 0

	if force or method or not executed(patchmodule):
		return execute_patch(patchmodule, method, methodargs)
	else:
		return True

def execute_patch(patchmodule, method=None, methodargs=None):
	"""execute the patch"""
	block_user(True)
	frappe.db.begin()

	try:
		log('Executing {patch} in {site} ({db})'.format(patch=patchmodule or str(methodargs),
			site=frappe.local.site, db=frappe.db.cur_db_name))
		if patchmodule:
			if patchmodule.startswith("execute:"):
				exec patchmodule.split("execute:")[1] in globals()
			else:
				frappe.get_attr(patchmodule.split()[0] + ".execute")()
			update_patch_log(patchmodule)
		elif method:
			method(**methodargs)

	except Exception:
		frappe.db.rollback()
		raise

	else:
		frappe.db.commit()
		block_user(False)
		log('Success')

	return True

def update_patch_log(patchmodule):
	"""update patch_file in patch log"""
	frappe.get_doc({"doctype": "Patch Log", "patch": patchmodule}).insert()

def executed(patchmodule):
	"""return True if is executed"""
	done = frappe.db.get_value("Patch Log", {"patch": patchmodule})
	# if done:
	# 	print "Patch %s already executed in %s" % (patchmodule, frappe.db.cur_db_name)
	return done

def block_user(block):
	"""stop/start execution till patch is run"""
	frappe.local.flags.in_patch = block
	frappe.db.begin()
	msg = "Patches are being executed in the system. Please try again in a few moments."
	frappe.db.set_global('__session_status', block and 'stop' or None)
	frappe.db.set_global('__session_status_message', block and msg or None)
	frappe.db.commit()

def check_session_stopped():
	if frappe.db.get_global("__session_status")=='stop':
		frappe.msgprint(frappe.db.get_global("__session_status_message"))
		raise frappe.SessionStopped('Session Stopped')

def log(msg):
	print msg
<|endoftext|>


<commit_before>def compare_string(string1: str, string2: str) -> str:
    """
    >>> compare_string('0010','0110')
    '0_10'

    >>> compare_string('0110','1101')
    -1
    """
    l1 = list(string1)
    l2 = list(string2)
    count = 0
    for i in range(len(l1)):
        if l1[i] != l2[i]:
            count += 1
            l1[i] = "_"
    if count > 1:
        return -1
    else:
        return "".join(l1)

def check(binary: [str]) -> [str]:
    """
    >>> check(['0.00.01.5'])
    ['0.00.01.5']
    """
    pi = []
    while 1:
        check1 = ["$"] * len(binary)
        temp = []
        for i in range(len(binary)):
            for j in range(i + 1, len(binary)):
                k = compare_string(binary[i], binary[j])
                if k != -1:
                    check1[i] = "*"
                    check1[j] = "*"
                    temp.append(k)
        for i in range(len(binary)):
            if check1[i] == "$":
                pi.append(binary[i])
        if len(temp) == 0:
            return pi
        binary = list(set(temp))

def decimal_to_binary(no_of_variable: int, minterms: [float]) -> [str]:
    """
    >>> decimal_to_binary(3,[1.5])
    ['0.00.01.5']
    """
    temp = []
    s = ""
    for m in minterms:
        for i in range(no_of_variable):
            s = str(m % 2) + s
            m //= 2
        temp.append(s)
        s = ""
    return temp

def is_for_table(string1: str, string2: str, count: int) -> bool:
    """
    >>> is_for_table('__1','011',2)
    True

    >>> is_for_table('01_','001',1)
    False
    """
    l1 = list(string1)
    l2 = list(string2)
    count_n = 0
    for i in range(len(l1)):
        if l1[i] != l2[i]:
            count_n += 1
    if count_n == count:
        return True
    else:
        return False

def selection(chart: [[int]], prime_implicants: [str]) -> [str]:
    """
    >>> selection([[1]],['0.00.01.5'])
    ['0.00.01.5']

    >>> selection([[1]],['0.00.01.5'])
    ['0.00.01.5']
    """
    temp = []
    select = [0] * len(chart)
    for i in range(len(chart[0])):
        count = 0
        rem = -1
        for j in range(len(chart)):
            if chart[j][i] == 1:
                count += 1
                rem = j
        if count == 1:
            select[rem] = 1
    for i in range(len(select)):
        if select[i] == 1:
            for j in range(len(chart[0])):
                if chart[i][j] == 1:
                    for k in range(len(chart)):
                        chart[k][j] = 0
            temp.append(prime_implicants[i])
    while 1:
        max_n = 0
        rem = -1
        count_n = 0
        for i in range(len(chart)):
            count_n = chart[i].count(1)
            if count_n > max_n:
                max_n = count_n
                rem = i

        if max_n == 0:
            return temp

        temp.append(prime_implicants[rem])

        for i in range(len(chart[0])):
            if chart[rem][i] == 1:
                for j in range(len(chart)):
                    chart[j][i] = 0

def prime_implicant_chart(prime_implicants: [str], binary: [str]) -> [[int]]:
    """
    >>> prime_implicant_chart(['0.00.01.5'],['0.00.01.5'])
    [[1]]
    """
    chart = [[0 for x in range(len(binary))] for x in range(len(prime_implicants))]
    for i in range(len(prime_implicants)):
        count = prime_implicants[i].count("_")
        for j in range(len(binary)):
            if is_for_table(prime_implicants[i], binary[j], count):
                chart[i][j] = 1

    return chart

def main():
    no_of_variable = int(input("Enter the no. of variables\n"))
    minterms = [
        int(x)
        for x in input(
            "Enter the decimal representation of Minterms 'Spaces Separated'\n"
        ).split()
    ]
    binary = decimal_to_binary(no_of_variable, minterms)

    prime_implicants = check(binary)
    print("Prime Implicants are:")
    print(prime_implicants)
    chart = prime_implicant_chart(prime_implicants, binary)

    essential_prime_implicants = selection(chart, prime_implicants)
    print("Essential Prime Implicants are:")
    print(essential_prime_implicants)

if __name__ == "__main__":
    import doctest

    doctest.testmod()
    main()
<commit_msg>[mypy] Add/fix type annotations for boolean_algebra (#4172)<commit_after>from typing import List

def compare_string(string1: str, string2: str) -> str:
    """
    >>> compare_string('0010','0110')
    '0_10'

    >>> compare_string('0110','1101')
    'X'
    """
    l1 = list(string1)
    l2 = list(string2)
    count = 0
    for i in range(len(l1)):
        if l1[i] != l2[i]:
            count += 1
            l1[i] = "_"
    if count > 1:
        return "X"
    else:
        return "".join(l1)

def check(binary: List[str]) -> List[str]:
    """
    >>> check(['0.00.01.5'])
    ['0.00.01.5']
    """
    pi = []
    while 1:
        check1 = ["$"] * len(binary)
        temp = []
        for i in range(len(binary)):
            for j in range(i + 1, len(binary)):
                k = compare_string(binary[i], binary[j])
                if k != "X":
                    check1[i] = "*"
                    check1[j] = "*"
                    temp.append(k)
        for i in range(len(binary)):
            if check1[i] == "$":
                pi.append(binary[i])
        if len(temp) == 0:
            return pi
        binary = list(set(temp))

def decimal_to_binary(no_of_variable: int, minterms: List[float]) -> List[str]:
    """
    >>> decimal_to_binary(3,[1.5])
    ['0.00.01.5']
    """
    temp = []
    s = ""
    for m in minterms:
        for i in range(no_of_variable):
            s = str(m % 2) + s
            m //= 2
        temp.append(s)
        s = ""
    return temp

def is_for_table(string1: str, string2: str, count: int) -> bool:
    """
    >>> is_for_table('__1','011',2)
    True

    >>> is_for_table('01_','001',1)
    False
    """
    l1 = list(string1)
    l2 = list(string2)
    count_n = 0
    for i in range(len(l1)):
        if l1[i] != l2[i]:
            count_n += 1
    if count_n == count:
        return True
    else:
        return False

def selection(chart: List[List[int]], prime_implicants: List[str]) -> List[str]:
    """
    >>> selection([[1]],['0.00.01.5'])
    ['0.00.01.5']

    >>> selection([[1]],['0.00.01.5'])
    ['0.00.01.5']
    """
    temp = []
    select = [0] * len(chart)
    for i in range(len(chart[0])):
        count = 0
        rem = -1
        for j in range(len(chart)):
            if chart[j][i] == 1:
                count += 1
                rem = j
        if count == 1:
            select[rem] = 1
    for i in range(len(select)):
        if select[i] == 1:
            for j in range(len(chart[0])):
                if chart[i][j] == 1:
                    for k in range(len(chart)):
                        chart[k][j] = 0
            temp.append(prime_implicants[i])
    while 1:
        max_n = 0
        rem = -1
        count_n = 0
        for i in range(len(chart)):
            count_n = chart[i].count(1)
            if count_n > max_n:
                max_n = count_n
                rem = i

        if max_n == 0:
            return temp

        temp.append(prime_implicants[rem])

        for i in range(len(chart[0])):
            if chart[rem][i] == 1:
                for j in range(len(chart)):
                    chart[j][i] = 0

def prime_implicant_chart(
    prime_implicants: List[str], binary: List[str]
) -> List[List[int]]:
    """
    >>> prime_implicant_chart(['0.00.01.5'],['0.00.01.5'])
    [[1]]
    """
    chart = [[0 for x in range(len(binary))] for x in range(len(prime_implicants))]
    for i in range(len(prime_implicants)):
        count = prime_implicants[i].count("_")
        for j in range(len(binary)):
            if is_for_table(prime_implicants[i], binary[j], count):
                chart[i][j] = 1

    return chart

def main():
    no_of_variable = int(input("Enter the no. of variables\n"))
    minterms = [
        int(x)
        for x in input(
            "Enter the decimal representation of Minterms 'Spaces Separated'\n"
        ).split()
    ]
    binary = decimal_to_binary(no_of_variable, minterms)

    prime_implicants = check(binary)
    print("Prime Implicants are:")
    print(prime_implicants)
    chart = prime_implicant_chart(prime_implicants, binary)

    essential_prime_implicants = selection(chart, prime_implicants)
    print("Essential Prime Implicants are:")
    print(essential_prime_implicants)

if __name__ == "__main__":
    import doctest

    doctest.testmod()
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: latin-1 -*-
import argparse
import os
import datetime
import encodedcc
import logging
import sys

EPILOG = '''takes a file with a list of experiment accessions, a query,
or a single accession and checks all associated objects
and sets them to their released status'''
logger = logging.getLogger(__name__)

def getArgs():
    parser = argparse.ArgumentParser(epilog=EPILOG)
    parser.add_argument('--infile',
                        help="File containing accessions or single accession")
    parser.add_argument('--outfile',
                        help="Output file name", default='Release_report.txt')
    parser.add_argument('--key',
                        help="The keypair identifier from the keyfile.",
                        default='default')
    parser.add_argument('--keyfile',
                        help="The keyfile",
                        default=os.path.expanduser('~/keypairs.json'))
    parser.add_argument('--update',
                        help="Run script and update the objects. Default is off",
                        action='store_true', default=False)
    parser.add_argument('--query',
                        help="Custom query for accessions")
    parser.add_argument('--force',
                        help="Forces release of experiments that did not pass audit. Default is off",
                        action='store_true', default=False)
    parser.add_argument('--debug',
                        help="Run script in debug mode.  Default is off",
                        action='store_true', default=False)
    parser.add_argument('--logall',
                        help="Adds status of 'released' objects to output file. Default is off",
                        action='store_true', default=False)
    args = parser.parse_args()
    if args.debug:
        logging.basicConfig(filename=args.outfile, filemode="w",
                            format='%(levelname)s:%(message)s',
                            level=logging.DEBUG)
    else:  # use the default logging level
        logging.basicConfig(filename=args.outfile, filemode="w",
                            format='%(message)s',
                            level=logging.INFO)
    logging.getLogger("requests").setLevel(logging.WARNING)

    return args

class Data_Release():
    def __init__(self, args, connection):
        self.infile = args.infile
        self.outfile = args.outfile
        self.QUERY = args.query
        self.LOGALL = args.logall
        self.FORCE = args.force
        self.UPDATE = args.update
        self.keysLink = []
        self.PROFILES = {}
        self.ACCESSIONS = []
        self.statusDict = {}
        self.connection = connection
        temp = encodedcc.get_ENCODE("/profiles/", self.connection)
        remove = ["AnalysisStepRun", "AnalysisStepVersion", "AnalysisStep", "Lab", "User", "Award"]
        self.profilesJSON = list(set(temp.keys()) - set(remove))

        for item in self.profilesJSON:
            profile = temp[item]
            self.keysLink = []  # if a key is in this list, it points to a link and will be embedded in the final product
            self.make_profile(profile)
            self.PROFILES[item] = self.keysLink
        #print(self.PROFILES)

        self.current = []
        self.finished = []
        for item in temp.keys():
            status = temp[item]["properties"]["status"]["enum"]
            if "current" in status:
                self.current.append(item)
            if "finished" in status:
                self.finished.append(item)

    def make_profile(self, dictionary):
        '''builds the PROFILES reference dictionary
        keysLink is the list of keys that point to links, used in the PROFILES'''
        d = dictionary["properties"]
        for prop in d.keys():
            if d[prop].get("linkTo") or d[prop].get("linkFrom"):
                self.keysLink.append(prop)
            else:
                if d[prop].get("items"):
                    i = d[prop].get("items")
                    if i.get("linkTo") or i.get("linkFrom"):
                        self.keysLink.append(prop)

    def expand_links(self, obj):
        '''uses PROFILES to build the expanded instance of the object fed in'''
        self.searched.append(obj["@id"])
        name = obj["@type"][0]
        for key in obj.keys():
            if key in self.PROFILES[name]:
                newLinkValues = []
                if type(obj[key]) is list:
                    for link in obj[key]:  # loop through keys
                        if link not in self.searched:  # if we have not checked this link yet we continue
                            temp = encodedcc.get_ENCODE(link, self.connection)
                            newLinkValues.append(temp)
                            #print(link)
                            self.expand_links(temp)
                    obj[key] = newLinkValues
                elif type(obj[key]) is str:
                    if obj[key] not in self.searched:
                        #print(key)
                        temp = encodedcc.get_ENCODE(obj[key], self.connection)
                        obj[key] = temp
                        self.expand_links(temp)

    def get_status(self, obj):
        '''take object get status, @type, @id, uuid
        {@id : [@type, uuid, status]}'''
        print(obj["@type"][0])
        name = obj["@type"][0]
        self.searched.append(obj["@id"])
        if self.PROFILES.get(name):
            self.statusDict[obj["@id"]] = [name, obj["uuid"], obj["status"]]
            for key in obj.keys():
                # loop through object properties
                if key in self.PROFILES[name]:
                    # if the key is in profiles it's a link
                    if type(obj[key]) is list:
                        for link in obj[key]:
                            subobj = encodedcc.get_ENCODE(link, self.connection)
                            if subobj["@type"][0] in self.PROFILES:
                                if subobj["@id"] not in self.searched:
                                    print("list", subobj["@id"])
                                    self.get_status(subobj)
                    else:
                        subobj = encodedcc.get_ENCODE(obj[key], self.connection)
                        if subobj["@type"][0] in self.PROFILES:
                            if subobj["@id"] not in self.searched:
                                print("string", subobj["@id"])
                                # expand subobject
                                self.get_status(subobj)

    def get_status_old(self, obj):
        '''take object get status, @type, @id, uuid
        {@id : [@type, uuid, status]}'''
        name = obj["@type"][0]
        self.searched.append(obj["@id"])
        print(name)
        self.statusDict[obj["@id"]] = [name, obj["uuid"], obj["status"]]
        for key in obj.keys():
            # loop through object properties
            if key in self.PROFILES[name]:
                # if the key is in profiles it's a link
                print(obj[key])
                subobj = encodedcc.get_ENCODE(obj[key], self.connection)
                if key in self.noExpand:
                    # don't expand, just get subobj data
                    self.statusDict[subobj["@id"]] = [subobj["@type"][0], subobj["uuid"], subobj["status"]]
                else:
                    # expand subobject
                    self.get_status(subobj)

    def find_status(self, dictionary):
        '''takes the object created by expand_links and makes a new dict,
        with each subobject and its status {@id : [@type, uuid, status]}'''
        d = dictionary
        name = d["@type"][0]
        self.statusDict[d["@id"]] = [name, d["uuid"], d["status"]]
        for key in d.keys():
            if key in self.PROFILES.get(name, []):
                if type(d[key]) is list:
                    for item in d[key]:
                        self.find_status(item)
                elif type(d[key]) is dict:
                    self.find_status(d[key])

    def releasinator(self, name, uuid, status, audit):
        '''releases objects into their equivalent released states'''
        stats = {}
        if name in self.current:
            logger.info('%s' % "UPDATING: {} {} with status {} is now current".format(name, uuid, status))
            stats = {"status": "current"}
        elif name in self.finished:
            logger.info('%s' % "UPDATING: {} {} with status {} is now finished".format(name, uuid, status))
            stats = {"status": "finished"}
        else:
            log = "UPDATING: {} {} with status {} is now released".format(name, uuid, status)
            stats = {"status": "released"}
            if audit:
                now = datetime.datetime.now().date()
                stats = {"date_released": str(now), "status": "released"}
                log = "UPDATING: {} {} with status {} is now released with date {}".format(name, uuid, status, now)
            logger.info('%s' % log)
        encodedcc.patch_ENCODE(uuid, self.connection, stats)

    def run_script(self):
        if self.UPDATE:
            print("WARNING: This run is an UPDATE run objects will be released.")
        else:
            print("Object status will be checked but not changed")
        if self.FORCE:
            print("WARNING: Objects that do not pass audit will be FORCE-released")
        if self.LOGALL:
            print("Logging all statuses")
        if self.infile:
            if os.path.isfile(self.infile):
                self.ACCESSIONS = [line.rstrip('\n') for line in open(self.infile)]
            else:
                self.ACCESSIONS = [self.infile]
        elif self.QUERY:
            if "search" in self.QUERY:
                temp = encodedcc.get_ENCODE(self.QUERY, self.connection).get("@graph", [])
            else:
                temp = [encodedcc.get_ENCODE(self.QUERY, self.connection)]
            if any(temp):
                for obj in temp:
                    if obj.get("accession"):
                        self.ACCESSIONS.append(obj["accession"])
                    elif obj.get("uuid"):
                        self.ACCESSIONS.append(obj["uuid"])
                    elif obj.get("@id"):
                        self.ACCESSIONS.append(obj["@id"])
                    elif obj.get("aliases"):
                        self.ACCESSIONS.append(obj["aliases"][0])
        if len(self.ACCESSIONS) == 0:
            print("ERROR: object has no identifier", file=sys.stderr)
            sys.exit(1)

        good = ["released", "current", "disabled", "published", "finished", "virtual"]
        bad = ["replaced", "revoked", "deleted", "upload failed",
               "format check failed", "uploading", "error"]
        ignore = ["User", "AntibodyCharacterization", "Publication"]
        for accession in self.ACCESSIONS:
            self.searched = []
            expandedDict = encodedcc.get_ENCODE(accession, self.connection)
            #self.expand_links(expandedDict)
            objectStatus = expandedDict.get("status")
            obj = expandedDict["@type"][0]
            audit = encodedcc.get_ENCODE(accession, self.connection, "page").get("audit", {})
            passAudit = True
            logger.info('%s' % "{}: {} Status: {}".format(obj, accession, objectStatus))
            if audit.get("ERROR", ""):
                logger.warning('%s' % "WARNING: Audit status: ERROR")
                passAudit = False
            if audit.get("NOT_COMPLIANT", ""):
                logger.warning('%s' % "WARNING: Audit status: NOT COMPLIANT")
                passAudit = False
            self.statusDict = {}
            self.get_status(expandedDict)
            #print(expandedDict)
            #sys.exit(1)
            #self.find_status(expandedDict)
            if self.FORCE:
                passAudit = True
            named = []
            for key in sorted(self.statusDict.keys()):
                name = self.statusDict[key][0]
                uuid = self.statusDict[key][1]
                status = self.statusDict[key][2]
                if name not in ignore:
                    if name not in named:
                        logger.info('%s' % name.upper())
                    if status in good:
                        if self.LOGALL:
                            logger.info('%s' % "{} {} has status {}".format(name, uuid, status))
                    elif status in bad:
                        logger.warning('%s' % "WARNING: {} {} has status {}".format(name, uuid, status))
                    else:
                        logger.info('%s' % "{} {} has status {}".format(name, uuid, status))
                        if self.UPDATE:
                            if passAudit:
                                self.releasinator(name, uuid, status, passAudit)
                    named.append(name)
        print("Data written to file", self.outfile)

def main():
    args = getArgs()
    key = encodedcc.ENC_Key(args.keyfile, args.key)
    connection = encodedcc.ENC_Connection(key)
    print("Running on", key.server)
    # build the PROFILES reference dictionary
    release = Data_Release(args, connection)
    #sys.exit(1)
    release.run_script()

if __name__ == "__main__":
    main()
<commit_msg>printing out @ids instead of uuids for clarity<commit_after>#!/usr/bin/env python
# -*- coding: latin-1 -*-
import argparse
import os
import datetime
import encodedcc
import logging
import sys

EPILOG = '''takes a file with a list of experiment accessions, a query,
or a single accession and checks all associated objects
and sets them to their released status'''
logger = logging.getLogger(__name__)

def getArgs():
    parser = argparse.ArgumentParser(epilog=EPILOG)
    parser.add_argument('--infile',
                        help="File containing accessions or single accession")
    parser.add_argument('--outfile',
                        help="Output file name", default='Release_report.txt')
    parser.add_argument('--key',
                        help="The keypair identifier from the keyfile.",
                        default='default')
    parser.add_argument('--keyfile',
                        help="The keyfile",
                        default=os.path.expanduser('~/keypairs.json'))
    parser.add_argument('--update',
                        help="Run script and update the objects. Default is off",
                        action='store_true', default=False)
    parser.add_argument('--query',
                        help="Custom query for accessions")
    parser.add_argument('--force',
                        help="Forces release of experiments that did not pass audit. Default is off",
                        action='store_true', default=False)
    parser.add_argument('--debug',
                        help="Run script in debug mode.  Default is off",
                        action='store_true', default=False)
    parser.add_argument('--logall',
                        help="Adds status of 'released' objects to output file. Default is off",
                        action='store_true', default=False)
    args = parser.parse_args()
    if args.debug:
        logging.basicConfig(filename=args.outfile, filemode="w",
                            format='%(levelname)s:%(message)s',
                            level=logging.DEBUG)
    else:  # use the default logging level
        logging.basicConfig(filename=args.outfile, filemode="w",
                            format='%(message)s',
                            level=logging.INFO)
    logging.getLogger("requests").setLevel(logging.WARNING)

    return args

class Data_Release():
    def __init__(self, args, connection):
        self.infile = args.infile
        self.outfile = args.outfile
        self.QUERY = args.query
        self.LOGALL = args.logall
        self.FORCE = args.force
        self.UPDATE = args.update
        self.keysLink = []
        self.PROFILES = {}
        self.ACCESSIONS = []
        self.statusDict = {}
        self.connection = connection
        temp = encodedcc.get_ENCODE("/profiles/", self.connection)
        ignore = ["AnalysisStepRun", "AnalysisStepVersion", "AnalysisStep",
                  "Lab", "Document", "Award", "AntibodyCharacterization",
                  "Publication", "Organism", "Reference", "AccessKey", "User"]
        self.profilesJSON = list(set(temp.keys()) - set(ignore))
        self.profiles_ref = []
        for profile in self.profilesJSON:
            if profile.endswith("y"):
                # this is a hack to find words like "Library"
                profile = profile.rstrip("y") + "ies"
                self.profiles_ref.append(profile.lower())
            elif profile.endswith("s"):
                # check for things with "Series" type endings
                self.profiles_ref.append(profile.lower())
            else:
                self.profiles_ref.append(profile.lower() + "s")

        for item in self.profilesJSON:
            profile = temp[item]
            self.keysLink = []  # if a key is in this list, it points to a link and will be embedded in the final product
            self.make_profile(profile)
            self.PROFILES[item] = self.keysLink

        self.current = []
        self.finished = []
        for item in temp.keys():
            status = temp[item]["properties"]["status"]["enum"]
            if "current" in status:
                self.current.append(item)
            if "finished" in status:
                self.finished.append(item)

    def make_profile(self, dictionary):
        '''builds the PROFILES reference dictionary
        keysLink is the list of keys that point to links, used in the PROFILES'''
        d = dictionary["properties"]
        for prop in d.keys():
            if d[prop].get("linkTo") or d[prop].get("linkFrom"):
                self.keysLink.append(prop)
            else:
                if d[prop].get("items"):
                    i = d[prop].get("items")
                    if i.get("linkTo") or i.get("linkFrom"):
                        self.keysLink.append(prop)

    def get_status(self, obj):
        '''take object get status, @type, @id, uuid
        {@id : [@type, status]}'''
        name = obj["@type"][0]
        self.searched.append(obj["@id"])
        if self.PROFILES.get(name):
            self.statusDict[obj["@id"]] = [name, obj["status"]]
            for key in obj.keys():
                # loop through object properties
                if key in self.PROFILES[name]:
                    # if the key is in profiles it's a link
                    if type(obj[key]) is list:
                        for link in obj[key]:
                            item = link.split("/")[1].replace("-", "")
                            #print(item)
                            if item in self.profiles_ref and link not in self.searched:
                                # expand subobject
                                subobj = encodedcc.get_ENCODE(link, self.connection)
                                self.get_status(subobj)
                    else:
                        item = obj[key].split("/")[1].replace("-", "")
                        #print(item)
                        if item in self.profiles_ref and obj[key] not in self.searched:
                            # expand subobject
                            subobj = encodedcc.get_ENCODE(obj[key], self.connection)
                            self.get_status(subobj)

    def releasinator(self, name, identifier, status, audit):
        '''releases objects into their equivalent released states'''
        patch_dict = {}
        if name in self.current:
            logger.info('%s' % "UPDATING: {} {} with status {} is now current".format(name, identifier, status))
            patch_dict = {"status": "current"}
        elif name in self.finished:
            logger.info('%s' % "UPDATING: {} {} with status {} is now finished".format(name, identifier, status))
            patch_dict = {"status": "finished"}
        else:
            log = "UPDATING: {} {} with status {} is now released".format(name, identifier, status)
            patch_dict = {"status": "released"}
            if audit:
                now = datetime.datetime.now().date()
                patch_dict = {"date_released": str(now), "status": "released"}
                log = "UPDATING: {} {} with status {} is now released with date {}".format(name, identifier, status, now)
            logger.info('%s' % log)
        encodedcc.patch_ENCODE(identifier, self.connection, patch_dict)

    def run_script(self):
        if self.UPDATE:
            print("WARNING: This run is an UPDATE run objects will be released.")
        else:
            print("Object status will be checked but not changed")
        if self.FORCE:
            print("WARNING: Objects that do not pass audit will be FORCE-released")
        if self.LOGALL:
            print("Logging all statuses")
        if self.infile:
            if os.path.isfile(self.infile):
                self.ACCESSIONS = [line.rstrip('\n') for line in open(self.infile)]
            else:
                self.ACCESSIONS = [self.infile]
        elif self.QUERY:
            if "search" in self.QUERY:
                temp = encodedcc.get_ENCODE(self.QUERY, self.connection).get("@graph", [])
            else:
                temp = [encodedcc.get_ENCODE(self.QUERY, self.connection)]
            if any(temp):
                for obj in temp:
                    if obj.get("accession"):
                        self.ACCESSIONS.append(obj["accession"])
                    elif obj.get("uuid"):
                        self.ACCESSIONS.append(obj["uuid"])
                    elif obj.get("@id"):
                        self.ACCESSIONS.append(obj["@id"])
                    elif obj.get("aliases"):
                        self.ACCESSIONS.append(obj["aliases"][0])
        if len(self.ACCESSIONS) == 0:
            print("ERROR: object has no identifier", file=sys.stderr)
            sys.exit(1)

        good = ["released", "current", "disabled", "published", "finished", "virtual"]
        bad = ["replaced", "revoked", "deleted", "upload failed",
               "format check failed", "uploading", "error"]
        ignore = ["User", "AntibodyCharacterization", "Publication"]
        for accession in self.ACCESSIONS:
            self.searched = []
            expandedDict = encodedcc.get_ENCODE(accession, self.connection)
            objectStatus = expandedDict.get("status")
            obj = expandedDict["@type"][0]

            audit = encodedcc.get_ENCODE(accession, self.connection, "page").get("audit", {})
            passAudit = True
            logger.info('%s' % "{}: {} Status: {}".format(obj, accession, objectStatus))
            if audit.get("ERROR", ""):
                logger.warning('%s' % "WARNING: Audit status: ERROR")
                passAudit = False
            if audit.get("NOT_COMPLIANT", ""):
                logger.warning('%s' % "WARNING: Audit status: NOT COMPLIANT")
                passAudit = False
            self.statusDict = {}
            self.get_status(expandedDict)
            if self.FORCE:
                passAudit = True

            named = []
            for key in sorted(self.statusDict.keys()):
                name = self.statusDict[key][0]
                status = self.statusDict[key][1]
                if name not in ignore:
                    if name not in named:
                        logger.info('%s' % name.upper())
                    if status in good:
                        if self.LOGALL:
                            logger.info('%s' % "{} has status {}".format(key, status))
                    elif status in bad:
                        logger.warning('%s' % "WARNING: {} has status {}".format(key, status))
                    else:
                        logger.info('%s' % "{} has status {}".format(key, status))
                        if self.UPDATE:
                            if passAudit:
                                self.releasinator(name, key, status, passAudit)
                    named.append(name)
        print("Data written to file", self.outfile)

def main():
    args = getArgs()
    key = encodedcc.ENC_Key(args.keyfile, args.key)
    connection = encodedcc.ENC_Connection(key)
    print("Running on", key.server)
    # build the PROFILES reference dictionary
    release = Data_Release(args, connection)
    release.run_script()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
from __future__ import print_function
from optparse import OptionParser
import os
import random
import sys
import time

import h5py
import joblib
import numpy as np
from sklearn.linear_model import LinearRegression, LogisticRegression
from sklearn.metrics import roc_auc_score
import tensorflow as tf

import basenji

################################################################################
# basenji_test.py
#
#
################################################################################

################################################################################
# main
################################################################################
def main():
    usage = 'usage: %prog [options] <params_file> <model_file> <data_file>'
    parser = OptionParser(usage)
    parser.add_option('-b', dest='batch_size', default=None, type='int', help='Batch size')
    parser.add_option('-d', dest='down_sample', default=1, type='int', help='Down sample test computation by taking uniformly spaced positions [Default: %default]')
    parser.add_option('-o', dest='out_dir', default='test_out', help='Output directory for test statistics [Default: %default]')
    parser.add_option('-p', dest='peaks_hdf5', help='Compute AUC for sequence peak calls [Default: %default]')
    parser.add_option('-s', dest='scent_file', help='Dimension reduction model file')
    parser.add_option('-v', dest='valid', default=False, action='store_true', help='Process the validation set [Default: %default]')
    parser.add_option('-w', dest='pool_width', default=1, type='int', help='Max pool width for regressing nucleotide predictions to predict peak calls [Default: %default]')
    (options,args) = parser.parse_args()

    if len(args) != 3:
    	parser.error('Must provide paramters, model, and test data HDF5')
    else:
        params_file = args[0]
        model_file = args[1]
        data_file = args[2]

    if not os.path.isdir(options.out_dir):
        os.mkdir(options.out_dir)

    #######################################################
    # load data
    #######################################################
    data_open = h5py.File(data_file)

    if not options.valid:
        test_seqs = data_open['test_in']
        test_targets = data_open['test_out']
    else:
        test_seqs = data_open['valid_in']
        test_targets = data_open['valid_out']

    #######################################################
    # model parameters and placeholders
    #######################################################
    job = basenji.dna_io.read_job_params(params_file)

    job['batch_length'] = test_seqs.shape[1]
    job['seq_depth'] = test_seqs.shape[2]
    job['num_targets'] = test_targets.shape[2]
    job['target_pool'] = int(np.array(data_open.get('pool_width', 1)))

    t0 = time.time()
    dr = basenji.rnn.RNN()
    dr.build(job)
    print('Model building time %ds' % (time.time()-t0))

    if options.batch_size is None:
        options.batch_size = dr.batch_size

    # adjust for fourier
    job['fourier'] = 'train_out_imag' in data_open
    if job['fourier']:
        test_targets_imag = data_open['test_out_imag']
        if options.valid:
            test_targets_imag = data_open['valid_out_imag']

    # adjust for factors
    if options.scent_file is not None:
        t0 = time.time()
        test_targets_full = data_open['test_out_full']
        model = joblib.load(options.scent_file)

    #######################################################
    # test
    #######################################################
    # initialize batcher
    if job['fourier']:
        batcher_test = basenji.batcher.BatcherF(test_seqs, test_targets, test_targets_imag, options.batch_size, job['pool_width'])
    else:
        batcher_test = basenji.batcher.Batcher(test_seqs, test_targets, options.batch_size, job['pool_width'])

    # initialize saver
    saver = tf.train.Saver()

    with tf.Session() as sess:
        # load variables into session
        saver.restore(sess, model_file)

        # test
        t0 = time.time()
        test_loss, test_r2_list, test_preds = dr.test(sess, batcher_test, return_preds=True, down_sample=options.down_sample)
        print('RNN test: %ds' % (time.time()-t0))

        # print
        print('Test loss: %7.5f' % test_loss)
        print('Test R2:   %7.5f' % test_r2_list.mean())

        r2_out = open('%s/r2.txt' % options.out_dir, 'w')
        for ti in range(len(test_r2_list)):
            print('%4d  %.4f' % (ti,test_r2_list[ti]), file=r2_out)
        r2_out.close()

        if options.scent_file is not None:
            full_targets = test_targets_full.shape[2]

            # uniformly sample indexes
            ds_indexes = np.arange(0, dr.batch_length-2*dr.batch_buffer, options.down_sample)

            # filter down full test targets
            test_targets_full_ds = test_targets_full[:,dr.batch_buffer+ds_indexes,:]

            # inverse transform in length batches
            t0 = time.time()
            test_preds_full = np.zeros((test_preds.shape[0], test_preds.shape[1], full_targets), dtype='float16')
            for li in range(test_preds.shape[1]):
                test_preds_full[:,li,:] = model.inverse_transform(test_preds[:,li,:])
            print('PCA transform: %ds' % (time.time()-t0))

            print(test_preds_full.shape)
            print(test_targets_full.shape)

            # compute R2 by target
            t0 = time.time()
            test_r2_full = np.zeros(full_targets)
            for ti in range(full_targets):
                # flatten
                preds_ti = test_preds_full[:,:,ti].flatten()
                targets_ti = test_targets_full_ds[:,:,ti].flatten()

                # compute R2
                tmean = targets_ti.mean(dtype='float64')
                tvar = (targets_ti-tmean).var(dtype='float64')
                pvar = (targets_ti-preds_ti).var(dtype='float64')
                test_r2_full[ti] = 1.0 - pvar/tvar
            print('Compute full R2: %d' % (time.time()-t0))

            print('Test full R2: %7.5f' % test_r2_full.mean())

            r2_out = open('%s/r2_full.txt' % options.out_dir, 'w')
            for ti in range(len(test_r2_full)):
                print('%4d  %.4f' % (ti,test_r2_full[ti]), file=r2_out)
            r2_out.close()

        #######################################################
        # peaks AUC
        #######################################################
        if options.peaks_hdf5:
            # use validation set to train
            valid_seqs = data_open['valid_in']
            valid_targets = data_open['valid_out']

            # initialize batcher
            if job['fourier']:
                valid_targets_imag = data_open['valid_out_imag']
                batcher_valid = basenji.batcher.BatcherF(valid_seqs, valid_targets, valid_targets_imag, options.batch_size)
            else:
                batcher_valid = basenji.batcher.Batcher(valid_seqs, valid_targets, options.batch_size)

            # make predictions
            _, _, valid_preds = dr.test(sess, batcher_valid, return_preds=True, down_sample=options.down_sample)

            print(valid_preds.shape)

            # max pool
            if options.pool_width > 1:
                valid_preds = max_pool(valid_preds, options.pool_width)
                test_preds = max_pool(test_preds, options.pool_width)
                print(valid_preds.shape)

            # load peaks
            peaks_open = h5py.File(options.peaks_hdf5)
            valid_peaks = np.array(peaks_open['valid_out'])
            test_peaks = np.array(peaks_open['test_out'])
            peaks_open.close()

            # compute target AUCs
            target_aucs = []
            model_coefs = []
            for ti in range(test_peaks.shape[1]):
                # balance positive and negative examples
                # valid_preds_ti, valid_peaks_ti = balance(valid_preds, valid_peaks[:,ti])
                valid_preds_ti, valid_peaks_ti = valid_preds, valid_peaks[:,ti]

                # train a predictor for peak calls
                model = LogisticRegression()
                if options.scent_file is not None:
                    valid_preds_ti_flat = valid_preds_ti.reshape((valid_preds_ti.shape[0],-1))
                    model.fit(valid_preds_ti_flat, valid_peaks_ti)
                else:
                    model.fit(valid_preds_ti[:,:,ti], valid_peaks_ti)
                model_coefs.append(model.coef_)

                # predict peaks for test set
                if options.scent_file is not None:
                    test_preds_flat = test_preds.reshape((test_preds.shape[0],-1))
                    test_peaks_preds = model.predict_proba(test_preds_flat)[:,1]
                else:
                    test_peaks_preds = model.predict_proba(test_preds[:,:,ti])[:,1]

                # compute AUC
                auc = roc_auc_score(test_peaks[:,ti], test_peaks_preds)
                target_aucs.append(auc)
                print('%d AUC: %f' % (ti,auc))

            print('AUC: %f' % np.mean(target_aucs))

            model_coefs = np.vstack(model_coefs)
            np.save('%s/coefs.npy' % options.out_dir, model_coefs)

    data_open.close()

def balance(X, Y):
    # determine positive and negative indexes
    indexes_pos = [i for i in range(len(Y)) if Y[i] == 1]
    indexes_neg = [i for i in range(len(Y)) if Y[i] == 0]

    # sample down negative
    indexes_neg_sample = random.sample(indexes_neg, len(indexes_pos))

    # combine
    indexes = sorted(indexes_pos + indexes_neg_sample)

    return X[indexes], Y[indexes]

def max_pool(preds, pool):
    # group by pool
    preds_pool = preds.reshape((preds.shape[0], preds.shape[1]//pool, pool, preds.shape[2]), order='C')

    # max
    return preds_pool.max(axis=2)

################################################################################
# __main__
################################################################################
if __name__ == '__main__':
    main()
<commit_msg>pool_width name bug<commit_after>#!/usr/bin/env python
from __future__ import print_function
from optparse import OptionParser
import os
import random
import sys
import time

import h5py
import joblib
import numpy as np
from sklearn.linear_model import LinearRegression, LogisticRegression
from sklearn.metrics import roc_auc_score
import tensorflow as tf

import basenji

################################################################################
# basenji_test.py
#
#
################################################################################

################################################################################
# main
################################################################################
def main():
    usage = 'usage: %prog [options] <params_file> <model_file> <data_file>'
    parser = OptionParser(usage)
    parser.add_option('-b', dest='batch_size', default=None, type='int', help='Batch size')
    parser.add_option('-d', dest='down_sample', default=1, type='int', help='Down sample test computation by taking uniformly spaced positions [Default: %default]')
    parser.add_option('-o', dest='out_dir', default='test_out', help='Output directory for test statistics [Default: %default]')
    parser.add_option('-p', dest='peaks_hdf5', help='Compute AUC for sequence peak calls [Default: %default]')
    parser.add_option('-s', dest='scent_file', help='Dimension reduction model file')
    parser.add_option('-v', dest='valid', default=False, action='store_true', help='Process the validation set [Default: %default]')
    parser.add_option('-w', dest='pool_width', default=1, type='int', help='Max pool width for regressing nucleotide predictions to predict peak calls [Default: %default]')
    (options,args) = parser.parse_args()

    if len(args) != 3:
    	parser.error('Must provide paramters, model, and test data HDF5')
    else:
        params_file = args[0]
        model_file = args[1]
        data_file = args[2]

    if not os.path.isdir(options.out_dir):
        os.mkdir(options.out_dir)

    #######################################################
    # load data
    #######################################################
    data_open = h5py.File(data_file)

    if not options.valid:
        test_seqs = data_open['test_in']
        test_targets = data_open['test_out']
    else:
        test_seqs = data_open['valid_in']
        test_targets = data_open['valid_out']

    #######################################################
    # model parameters and placeholders
    #######################################################
    job = basenji.dna_io.read_job_params(params_file)

    job['batch_length'] = test_seqs.shape[1]
    job['seq_depth'] = test_seqs.shape[2]
    job['num_targets'] = test_targets.shape[2]
    job['target_pool'] = int(np.array(data_open.get('pool_width', 1)))

    t0 = time.time()
    dr = basenji.rnn.RNN()
    dr.build(job)
    print('Model building time %ds' % (time.time()-t0))

    if options.batch_size is None:
        options.batch_size = dr.batch_size

    # adjust for fourier
    job['fourier'] = 'train_out_imag' in data_open
    if job['fourier']:
        test_targets_imag = data_open['test_out_imag']
        if options.valid:
            test_targets_imag = data_open['valid_out_imag']

    # adjust for factors
    if options.scent_file is not None:
        t0 = time.time()
        test_targets_full = data_open['test_out_full']
        model = joblib.load(options.scent_file)

    #######################################################
    # test
    #######################################################
    # initialize batcher
    if job['fourier']:
        batcher_test = basenji.batcher.BatcherF(test_seqs, test_targets, test_targets_imag, options.batch_size, job['target_pool'])
    else:
        batcher_test = basenji.batcher.Batcher(test_seqs, test_targets, options.batch_size, job['target_pool'])

    # initialize saver
    saver = tf.train.Saver()

    with tf.Session() as sess:
        # load variables into session
        saver.restore(sess, model_file)

        # test
        t0 = time.time()
        test_loss, test_r2_list, test_preds = dr.test(sess, batcher_test, return_preds=True, down_sample=options.down_sample)
        print('RNN test: %ds' % (time.time()-t0))

        # print
        print('Test loss: %7.5f' % test_loss)
        print('Test R2:   %7.5f' % test_r2_list.mean())

        r2_out = open('%s/r2.txt' % options.out_dir, 'w')
        for ti in range(len(test_r2_list)):
            print('%4d  %.4f' % (ti,test_r2_list[ti]), file=r2_out)
        r2_out.close()

        if options.scent_file is not None:
            full_targets = test_targets_full.shape[2]

            # uniformly sample indexes
            ds_indexes = np.arange(0, dr.batch_length-2*dr.batch_buffer, options.down_sample)

            # filter down full test targets
            test_targets_full_ds = test_targets_full[:,dr.batch_buffer+ds_indexes,:]

            # inverse transform in length batches
            t0 = time.time()
            test_preds_full = np.zeros((test_preds.shape[0], test_preds.shape[1], full_targets), dtype='float16')
            for li in range(test_preds.shape[1]):
                test_preds_full[:,li,:] = model.inverse_transform(test_preds[:,li,:])
            print('PCA transform: %ds' % (time.time()-t0))

            print(test_preds_full.shape)
            print(test_targets_full.shape)

            # compute R2 by target
            t0 = time.time()
            test_r2_full = np.zeros(full_targets)
            for ti in range(full_targets):
                # flatten
                preds_ti = test_preds_full[:,:,ti].flatten()
                targets_ti = test_targets_full_ds[:,:,ti].flatten()

                # compute R2
                tmean = targets_ti.mean(dtype='float64')
                tvar = (targets_ti-tmean).var(dtype='float64')
                pvar = (targets_ti-preds_ti).var(dtype='float64')
                test_r2_full[ti] = 1.0 - pvar/tvar
            print('Compute full R2: %d' % (time.time()-t0))

            print('Test full R2: %7.5f' % test_r2_full.mean())

            r2_out = open('%s/r2_full.txt' % options.out_dir, 'w')
            for ti in range(len(test_r2_full)):
                print('%4d  %.4f' % (ti,test_r2_full[ti]), file=r2_out)
            r2_out.close()

        #######################################################
        # peaks AUC
        #######################################################
        if options.peaks_hdf5:
            # use validation set to train
            valid_seqs = data_open['valid_in']
            valid_targets = data_open['valid_out']

            # initialize batcher
            if job['fourier']:
                valid_targets_imag = data_open['valid_out_imag']
                batcher_valid = basenji.batcher.BatcherF(valid_seqs, valid_targets, valid_targets_imag, options.batch_size)
            else:
                batcher_valid = basenji.batcher.Batcher(valid_seqs, valid_targets, options.batch_size)

            # make predictions
            _, _, valid_preds = dr.test(sess, batcher_valid, return_preds=True, down_sample=options.down_sample)

            print(valid_preds.shape)

            # max pool
            if options.pool_width > 1:
                valid_preds = max_pool(valid_preds, options.pool_width)
                test_preds = max_pool(test_preds, options.pool_width)
                print(valid_preds.shape)

            # load peaks
            peaks_open = h5py.File(options.peaks_hdf5)
            valid_peaks = np.array(peaks_open['valid_out'])
            test_peaks = np.array(peaks_open['test_out'])
            peaks_open.close()

            # compute target AUCs
            target_aucs = []
            model_coefs = []
            for ti in range(test_peaks.shape[1]):
                # balance positive and negative examples
                # valid_preds_ti, valid_peaks_ti = balance(valid_preds, valid_peaks[:,ti])
                valid_preds_ti, valid_peaks_ti = valid_preds, valid_peaks[:,ti]

                # train a predictor for peak calls
                model = LogisticRegression()
                if options.scent_file is not None:
                    valid_preds_ti_flat = valid_preds_ti.reshape((valid_preds_ti.shape[0],-1))
                    model.fit(valid_preds_ti_flat, valid_peaks_ti)
                else:
                    model.fit(valid_preds_ti[:,:,ti], valid_peaks_ti)
                model_coefs.append(model.coef_)

                # predict peaks for test set
                if options.scent_file is not None:
                    test_preds_flat = test_preds.reshape((test_preds.shape[0],-1))
                    test_peaks_preds = model.predict_proba(test_preds_flat)[:,1]
                else:
                    test_peaks_preds = model.predict_proba(test_preds[:,:,ti])[:,1]

                # compute AUC
                auc = roc_auc_score(test_peaks[:,ti], test_peaks_preds)
                target_aucs.append(auc)
                print('%d AUC: %f' % (ti,auc))

            print('AUC: %f' % np.mean(target_aucs))

            model_coefs = np.vstack(model_coefs)
            np.save('%s/coefs.npy' % options.out_dir, model_coefs)

    data_open.close()

def balance(X, Y):
    # determine positive and negative indexes
    indexes_pos = [i for i in range(len(Y)) if Y[i] == 1]
    indexes_neg = [i for i in range(len(Y)) if Y[i] == 0]

    # sample down negative
    indexes_neg_sample = random.sample(indexes_neg, len(indexes_pos))

    # combine
    indexes = sorted(indexes_pos + indexes_neg_sample)

    return X[indexes], Y[indexes]

def max_pool(preds, pool):
    # group by pool
    preds_pool = preds.reshape((preds.shape[0], preds.shape[1]//pool, pool, preds.shape[2]), order='C')

    # max
    return preds_pool.max(axis=2)

################################################################################
# __main__
################################################################################
if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before> #!/usr/bin/env python
import urllib
import urllib2
import json
import re
import os.path
import readline, glob
# import subprocess
import sys
import pprint
from mutagen.mp3 import MP3
from mutagen.id3 import ID3NoHeaderError
from mutagen.id3 import ID3, APIC, TIT2, ID3, TALB, TPE1, TPE2, TRCK, COMM, USLT, TCOM, TCON, TDRC, error

def validate_url(url):
    #TODO: validate url, duh
    return False

def change_song_details(audio_path, title, artist, art_path, album_name, track_number, year, genre):
    mp3 = MP3(audio_path, ID3=ID3)
    set_tags(mp3)
    set_album_art(mp3, art_path, 'image/jpg')
    mp3.save()

    id3 = ID3(audio_path)
    set_album_name(id3, album_name)
    set_title(id3, title)
    set_artist(id3, artist)
    set_track_number(id3, track_number)
    set_year(id3, year)
    set_genre(id3, genre)
    id3.save()

    print 'Track: %s' % id3['TIT2'].text[0]
    print 'Artist: %s' % id3['TPE2'].text[0]
    print 'Album: %s' % id3['TALB'].text[0]
    print 'Year: %s' % id3['TDRC'].text[0]
    print 'Number: %s' % id3['TRCK'].text[0]

def set_tags(mp3):
    try:
        mp3.add_tags()
    except error as e:
        print e
        pass

def set_album_art(mp3, art_path, image_type):
    try:
        mp3.tags.add(
            APIC(
                encoding = 3, # 3 is for utf-8
                mime = image_type, # image/jpeg or image/png
                type = 3, # 3 is for the cover image
                desc = u'Cover',
                data = open(art_path).read()
            )
        )
    except error as e:
        print e
        pass

def set_album_name(id3, album_name):
    try:
        id3.add(TALB(encoding = 3, text = album_name))
    except error as e:
        print e
        pass

def set_title(id3, title):
    try:
        id3.add(TIT2(encoding = 3, text = title))
    except error as e:
        print e
        pass

def set_artist(id3, artist):
    try:
        id3.add(TPE2(encoding = 3, text = artist))
    except error as e:
        print e
        pass

def set_track_number(id3, track_number):
    try:
        id3.add(TRCK(encoding = 3, text = track_number))
    except error as e:
        print e
        pass

def set_year(id3, year):
    try:
        id3.add(TDRC(encoding = 3, text = year))
    except error as e:
        print e
        pass

def set_genre(id3, genre):
    try:
        id3.add(TCON(encoding = 3, text = genre))
    except error as e:
        print e
        pass

def rip_it_up(album_url):

    response = urllib2.urlopen(album_url)
    if(response.getcode() == 200):
        garbage = response.read()

        print "here it is"
        regex = re.compile("var\sTralbumData\s\=\s\{")

        for m in regex.finditer(garbage):
            start = m.start()

        i = start
        end = None
        while not end:
            if garbage[i] is ";" and garbage[i-1] is "}":
                end = i
            i += 1

        good_stuff = garbage[start:end]
    else:
        return False
    #download this shizz
    return True

if len(sys.argv) > 1:
    album_url = sys.argv[1]

    if validate_url(album_url):
        print "Now ripping " + album_url #TODO: grab album title from json
        if(rip_it_up(album_url)):
            print "Aw yiz"
        else:
            print "Ah, that's a real ding there"
    else:
        print "What this is?"
        change_song_details('tests/sample.mp3', 'Side A', 'Kappa Chow', 'tests/albumart.jpg', 'Summer Tour Tape', '1', '2015', 'sackville')
else:
    print "Gimmie something to rip bud!"

<commit_msg>Added basic URL validation<commit_after> #!/usr/bin/env python
import urllib
import urllib2
import json
import re
import os.path
import readline, glob
# import subprocess
import sys
import pprint
from mutagen.mp3 import MP3
from mutagen.id3 import ID3, APIC, TIT2, ID3, TALB, TPE1, TPE2, TRCK, TCON, TDRC, error

def validate_url(url):
    try:
        urllib2.urlopen(url)
        return True
    except:
        print "Crap url: %s" % url
        return False
    return False

def change_song_details(audio_path, title, artist, art_path, album_name, track_number, year, genre):
    mp3 = MP3(audio_path, ID3=ID3)
    set_tags(mp3)
    set_album_art(mp3, art_path, 'image/jpg')
    mp3.save()

    id3 = ID3(audio_path)
    set_album_name(id3, album_name)
    set_title(id3, title)
    set_artist(id3, artist)
    set_track_number(id3, track_number)
    set_year(id3, year)
    set_genre(id3, genre)
    id3.save()

    print 'Track: %s' % id3['TIT2'].text[0]
    print 'Artist: %s' % id3['TPE2'].text[0]
    print 'Album: %s' % id3['TALB'].text[0]
    print 'Year: %s' % id3['TDRC'].text[0]
    print 'Number: %s' % id3['TRCK'].text[0]

def set_tags(mp3):
    try:
        mp3.add_tags()
    except error as e:
        print e
        pass

def set_album_art(mp3, art_path, image_type):
    try:
        mp3.tags.add(
            APIC(
                encoding = 3, # 3 is for utf-8
                mime = image_type, # image/jpeg or image/png
                type = 3, # 3 is for the cover image
                desc = u'Cover',
                data = open(art_path).read()
            )
        )
    except error as e:
        print e
        pass

def set_album_name(id3, album_name):
    try:
        id3.add(TALB(encoding = 3, text = album_name))
    except error as e:
        print e
        pass

def set_title(id3, title):
    try:
        id3.add(TIT2(encoding = 3, text = title))
    except error as e:
        print e
        pass

def set_artist(id3, artist):
    try:
        id3.add(TPE2(encoding = 3, text = artist))
    except error as e:
        print e
        pass

def set_track_number(id3, track_number):
    try:
        id3.add(TRCK(encoding = 3, text = track_number))
    except error as e:
        print e
        pass

def set_year(id3, year):
    try:
        id3.add(TDRC(encoding = 3, text = year))
    except error as e:
        print e
        pass

def set_genre(id3, genre):
    try:
        id3.add(TCON(encoding = 3, text = genre))
    except error as e:
        print e
        pass

def rip_it_up(album_url):

    response = urllib2.urlopen(album_url)
    if(response.getcode() == 200):
        garbage = response.read()

        print "here it is"
        regex = re.compile("var\sTralbumData\s\=\s\{")

        for m in regex.finditer(garbage):
            start = m.start()

        i = start
        end = None
        while not end:
            if garbage[i] is ";" and garbage[i-1] is "}":
                end = i
            i += 1

        good_stuff = garbage[start:end]
    else:
        return False
    #download this shizz
    return True

if len(sys.argv) > 1:
    album_url = sys.argv[1]

    if validate_url(album_url):
        print "Now ripping " + album_url #TODO: grab album title from json
        if(rip_it_up(album_url)):
            print "Aw yiz"
        else:
            print "Ah, that's a real ding there"
    else:
        print "What this is?"
        change_song_details('tests/sample.mp3', 'Side A', 'Kappa Chow', 'tests/albumart.jpg', 'Summer Tour Tape', '1', '2015', 'sackville')
else:
    print "Gimmie something to rip bud!"

<|endoftext|>


<commit_before>'''
Created on Jan 11, 2013

__author__ = "Elizabeth 'pidge' Flanagan"
__copyright__ = "Copyright 2012-2013, Intel Corp."
__credits__ = ["Elizabeth Flanagan"]
__license__ = "GPL"
__version__ = "2.0"
__maintainer__ = "Elizabeth Flanagan"
__email__ = "elizabeth.flanagan@intel.com"
'''

from buildbot.steps.shell import ShellCommand
from twisted.python import log
import os, datetime
from autobuilder.config import *

class PublishArtifacts(ShellCommand):

    haltOnFailure = False
    flunkOnFailure = True
    name = "Publishing Artifacts"
    def __init__(self, factory, argdict=None, **kwargs):
        self.factory = factory
        self.description = "Publishing Artifacts"
        self.slavedir=os.path.join(os.path.join(YOCTO_ABBASE, "yocto-slave"))
        for k, v in argdict.iteritems():
            setattr(self, k, v)
        ShellCommand.__init__(self, **kwargs)

    def start(self):
        self.timeout=15000
        DEST=self.getProperty("DEST")
        buildername=self.getProperty("buildername")
        self.basedir=os.path.join(os.path.join(
                                    self.slavedir, buildername),
                                    "build/build/")
        command=""
        DATESTAMP=datetime.datetime.now().strftime("%Y%m%d")
        log.msg("Publishing " + str(self.artifacts))
        if str(os.environ.get('PUBLISH_BUILDS')) == "True":
            for artifact in self.artifacts:
                log.msg("Publishing " + str(artifact))
                if artifact == "adt_installer":
                    command=command+"mkdir -p " + os.path.join(DEST, "adt_installer") + ";"
                    command=command+"cp -R --no-dereference --preserve=links " + \
                                    os.path.join(self.basedir, "tmp/deploy/images/") + \
                                    "*adt* " + os.path.join(DEST, "adt_installer")
                elif artifact == "adt_installer-QA":
                    command=command+"mkdir -p " + os.path.join(DEST, "adt_installer-QA") + ";"
                    command=command+"cp -R --no-dereference --preserve=links " + \
                                    os.path.join(self.basedir, "tmp/deploy/images/") + \
                                    "*adt* " + os.path.join(DEST, "adt_installer-QA")
                elif artifact == "adtrepo-dev":
                    command=command+"SDKVERSION=`cat ./build/meta-yocto/conf/distro/poky.conf |grep 'DISTRO_VERSION ='|sed 's/DISTRO_VERSION = //'|sed 's/-${DATE}//'|sed 's/\"//g'`;"
                    command=command+"mkdir -p " + os.path.join(DEST, "adt_installer-QA") + ";"
                    command=command+"rm -rf " + os.environ.get("ADTREPO_PATH") + "/$SDKVERSION/" + DATESTAMP
                elif "qemu" in artifact:
                    if artifact == "qemux86-tiny":
                        command=command+"mkdir -p " + DEST + "/machines/qemu/" + artifact + ";"
                        command=command+"cp -R --no-dereference --preserve=links " + DEST + "/machines/qemu/qemux86-tiny;"
                    else:
                        command=command+"mkdir -p " + DEST + "/machines/qemu/" + artifact + ";"
                        command=command+"cp -R --no-dereference --preserve=links " + \
                                        os.path.join(self.basedir, "tmp/deploy/images/") + \
                                        "*"+artifact+"* " + DEST + "/machines/qemu/" + artifact + ";"
                else:
                    command=command+"mkdir -p " + DEST + "/machines/qemu/" + artifact + ";"
                    command=command+"cp -R --no-dereference --preserve=links " + \
                                     os.path.join(self.basedir, "tmp/deploy/images/") + \
                                     "*"+artifact+"* " + DEST + "/machines/qemu/" + artifact + ";"
            self.command = command
        else:
            self.command = "echo 'Skipping Step.'"
        ShellCommand.start(self)

    def describe(self, done=False):
        description = ShellCommand.describe(self,done)
        return description

<commit_msg>PublishArtifacts.py: simplifying qemu publish<commit_after>'''
Created on Jan 11, 2013

__author__ = "Elizabeth 'pidge' Flanagan"
__copyright__ = "Copyright 2012-2013, Intel Corp."
__credits__ = ["Elizabeth Flanagan"]
__license__ = "GPL"
__version__ = "2.0"
__maintainer__ = "Elizabeth Flanagan"
__email__ = "elizabeth.flanagan@intel.com"
'''

from buildbot.steps.shell import ShellCommand
from twisted.python import log
import os, datetime
from autobuilder.config import *

class PublishArtifacts(ShellCommand):

    haltOnFailure = False
    flunkOnFailure = True
    name = "Publishing Artifacts"
    def __init__(self, factory, argdict=None, **kwargs):
        self.factory = factory
        self.description = "Publishing Artifacts"
        self.slavedir=os.path.join(os.path.join(YOCTO_ABBASE, "yocto-slave"))
        for k, v in argdict.iteritems():
            setattr(self, k, v)
        ShellCommand.__init__(self, **kwargs)

    def start(self):
        self.timeout=15000
        DEST=self.getProperty("DEST")
        buildername=self.getProperty("buildername")
        self.basedir=os.path.join(os.path.join(
                                    self.slavedir, buildername),
                                    "build/build/")
        command=""
        DATESTAMP=datetime.datetime.now().strftime("%Y%m%d")
        log.msg("Publishing " + str(self.artifacts))
        if str(os.environ.get('PUBLISH_BUILDS')) == "True":
            for artifact in self.artifacts:
                log.msg("Publishing " + str(artifact))
                if artifact == "adt_installer":
                    command=command+"mkdir -p " + os.path.join(DEST, "adt_installer") + ";"
                    command=command+"cp -R --no-dereference --preserve=links " + \
                                    os.path.join(self.basedir, "tmp/deploy/images/") + \
                                    "*adt* " + os.path.join(DEST, "adt_installer")
                elif artifact == "adt_installer-QA":
                    command=command+"mkdir -p " + os.path.join(DEST, "adt_installer-QA") + ";"
                    command=command+"cp -R --no-dereference --preserve=links " + \
                                    os.path.join(self.basedir, "tmp/deploy/images/") + \
                                    "*adt* " + os.path.join(DEST, "adt_installer-QA")
                elif artifact == "adtrepo-dev":
                    command=command+"SDKVERSION=`cat ./build/meta-yocto/conf/distro/poky.conf |grep 'DISTRO_VERSION ='|sed 's/DISTRO_VERSION = //'|sed 's/-${DATE}//'|sed 's/\"//g'`;"
                    command=command+"mkdir -p " + os.path.join(DEST, "adt_installer-QA") + ";"
                    command=command+"rm -rf " + os.environ.get("ADTREPO_PATH") + "/$SDKVERSION/" + DATESTAMP
                elif "qemu" in artifact:
                    command=command+"mkdir -p " + DEST + "/machines/qemu/" + artifact + ";"
                    command=command+"cp -R --no-dereference --preserve=links " + \
                                    os.path.join(self.basedir, "tmp/deploy/images/") + \
                                    "* " + DEST + "/machines/qemu/" + artifact + ";"
                else:
                    command=command+"mkdir -p " + DEST + "/machines/qemu/" + artifact + ";"
                    command=command+"cp -R --no-dereference --preserve=links " + \
                                     os.path.join(self.basedir, "tmp/deploy/images/") + \
                                     "*"+artifact+"* " + DEST + "/machines/qemu/" + artifact + ";"
            self.command = command
        else:
            self.command = "echo 'Skipping Step.'"
        ShellCommand.start(self)

    def describe(self, done=False):
        description = ShellCommand.describe(self,done)
        return description

<|endoftext|>


<commit_before>'''
Iedereen houdt van koekjes, in het echt en de virtuele variant.
'''
from datetime import datetime

import sqlalchemy as sa
from sqlalchemy.ext.declarative import declarative_base
from stickord.registry import Command, get_easy_logger, role_whitelist

Base = declarative_base() # pylint: disable=invalid-name
LOGGER = get_easy_logger('commands.koekjes')
KOEKTROMMEL = 40

class Koekje(Base):
    ''' Represent a single cookie '''
    __tablename__ = 'koekjes'

    id = sa.Column(sa.Integer, primary_key=True) # pylint: disable=invalid-name
    #save the new cookie count
    count = sa.Column(sa.Integer, nullable=False)
    author = sa.Column(sa.String, nullable=False)
    created_at = sa.Column(sa.DateTime, default=datetime.utcnow)

    def __repr__(self):
        ''' Represent as <Koekje(id, count, author, created_at)>. '''
        return (f'<Koekje({self.id}, {self.count}, {self.author},'
                f' {self.created_at})>')

@Command('koekje', category='Sticky')
async def cookie(_cont, mesg, client, sessionmaker, *_args, **_kwargs):
    ''' Gives the user a cookie. This cookie has no discernible function,
    representation or value. '''
    session = sessionmaker()
    koektrommel = session.query(Koekje).order_by(Koekje.created_at.desc()).first()

    if not koektrommel:
        LOGGER.info('No koektrommel yet, making one')
        resetkoekjes(session, client.user)
        count = KOEKTROMMEL
        _author = client.user.id
        _when = datetime.utcnow()
    else:
        count, _author, _when = (
            koektrommel.count,
            koektrommel.author,
            koektrommel.created_at
        )

    num = count - 1

    if num < 0:
        LOGGER.info(
            '%s (%s) tried to get a cookie but couldn\'t',
            mesg.author.mention, mesg.author.id
        )
        response = (f'Oh no! There\'s no cookie in the koektrommel'
                    f' have someone from @Bestuur refill it.')
    else:
        setkoekjes(session, num, mesg.author)
        await client.add_reaction(mesg, '\U0001f36a')
        response = None

    session.commit()
    return response

@Command(['vulkoekjes', 'koekjesbijvullen'], category='Sticky')
@role_whitelist(['Admin', 'Bestuur'])
async def vultrommel(_cont, msg, _client, sessionmaker, *_args, **_kwargs):
    ''' Refill the infamous koektrommel (Bestuur only). '''
    session = sessionmaker()
    resetkoekjes(session, msg.author)
    session.commit()
    return f'Geweldig, {msg.author.mention} heeft de koektrommel bijgevuld!'

def resetkoekjes(session, author):
    ''' Reset koektrommel to 40. '''
    setkoekjes(session, KOEKTROMMEL, author)

def setkoekjes(session, count, author):
    ''' Set the koektrommel to a particular count'''
    LOGGER.debug(
        'Koektrommel set to %s cookies by %s (%s)',
        count, author.mention, author.id
    )

    author_id = author.id
    session.add(Koekje(
        count=count,
        author=author_id
    ))
<commit_msg>Prevent refill of cookies of not empty<commit_after>'''
Iedereen houdt van koekjes, in het echt en de virtuele variant.
'''
from datetime import datetime

import sqlalchemy as sa
from sqlalchemy.ext.declarative import declarative_base
from stickord.registry import Command, get_easy_logger, role_whitelist

Base = declarative_base() # pylint: disable=invalid-name
LOGGER = get_easy_logger('commands.koekjes')
KOEKTROMMEL = 40

class Koekje(Base):
    ''' Represent a single cookie '''
    __tablename__ = 'koekjes'

    id = sa.Column(sa.Integer, primary_key=True) # pylint: disable=invalid-name
    #save the new cookie count
    count = sa.Column(sa.Integer, nullable=False)
    author = sa.Column(sa.String, nullable=False)
    created_at = sa.Column(sa.DateTime, default=datetime.utcnow)

    def __repr__(self):
        ''' Represent as <Koekje(id, count, author, created_at)>. '''
        return (f'<Koekje({self.id}, {self.count}, {self.author},'
                f' {self.created_at})>')

@Command('koekje', category='Sticky')
async def cookie(_cont, mesg, client, sessionmaker, *_args, **_kwargs):
    ''' Gives the user a cookie. This cookie has no discernible function,
    representation or value. '''
    session = sessionmaker()
    koektrommel = session.query(Koekje).order_by(Koekje.created_at.desc()).first()

    if not koektrommel:
        LOGGER.info('No koektrommel yet, making one')
        resetkoekjes(session, client.user)
        count = KOEKTROMMEL
        _author = client.user.id
        _when = datetime.utcnow()
    else:
        count, _author, _when = (
            koektrommel.count,
            koektrommel.author,
            koektrommel.created_at
        )

    num = count - 1

    if num < 0:
        LOGGER.debug(
            '%s (%s) tried to get a cookie but couldn\'t',
            mesg.author.mention, mesg.author.id
        )
        response = (f'Oh no! There\'s no cookie in the koektrommel'
                    f' have someone from @Bestuur refill it.')
    else:
        setkoekjes(session, num, mesg.author)
        await client.add_reaction(mesg, '\U0001f36a')
        response = None

    session.commit()
    return response

@Command(['vulkoekjes', 'koekjesbijvullen'], category='Sticky')
@role_whitelist(['Admin', 'Bestuur'])
async def vultrommel(_cont, msg, _client, sessionmaker, *_args, **_kwargs):
    ''' Refill the infamous koektrommel (Bestuur only). '''
    session = sessionmaker()
    koektrommel = session.query(Koekje).order_by(Koekje.created_at.desc()).first()
    if koektrommel.count == 0:
        resetkoekjes(session, msg.author)
        session.commit()
        return f'Geweldig, {msg.author.mention} heeft de koektrommel bijgevuld!'
    return f'Oh nee! Er passen niet zoveel koekjes in de koektrommel want hij is nog niet leeg!'

def resetkoekjes(session, author):
    ''' Reset koektrommel to 40. '''
    setkoekjes(session, KOEKTROMMEL, author)

def setkoekjes(session, count, author):
    ''' Set the koektrommel to a particular count'''
    LOGGER.debug(
        'Koektrommel set to %s cookies by %s (%s)',
        count, author.mention, author.id
    )

    author_id = author.id
    session.add(Koekje(
        count=count,
        author=author_id
    ))
<|endoftext|>