<commit_before>#!/usr/bin/env python

import logging
import os
import putio
import pyinotify
import sqlite3

from flask import *
from flask.ext.script import Manager

app = Flask(__name__)

app.logger.setLevel(logging.DEBUG)
app.config.from_object('config')

logging.basicConfig(filename=app.config.get('LOG_FILENAME'), level=app.config.get('LOG_LEVEL', logging.WARNING))

client = putio.Client(app.config['PUTIO_TOKEN'])

manager = Manager(app)

@manager.command
def transfers_cancel_seeding():
    transfer_ids = []
    for transfer in client.Transfer.list():
        if transfer.status == 'SEEDING':
            transfer_ids.append(transfer.id)

    if len(transfer_ids):
        client.Transfer.cancel(transfer_ids)

@manager.command
def transfers_clean():
    client.Transfer.clean()

@manager.command
def transfers_groom():
    transfers_cancel_seeding()
    transfers_clean()

@manager.command
def torrents_add():
    files = os.listdir(app.config['TORRENTS'])

    if len(files):
        with sqlite3.connect(app.config['DATABASE']) as connection:
            c = connection.cursor()

            for name in os.listdir(app.config['TORRENTS']):
                path = os.path.join(app.config['TORRENTS'], name)
                size = os.path.getsize(path)

                c.execute("select datetime(created_at, 'localtime') from torrents where name = ? and size = ?", (name, size))
                row = c.fetchone()

                if row is None:
                    app.logger.debug('adding torrent: %s' % name)

                    try:
                        app.logger.info('adding torrent: %s' % path)
                        transfer = client.Transfer.add_torrent(path)
                        os.unlink(path)
                        app.logger.info('deleted torrent, added: %s' % transfer)
                    except Exception, e:
                        if e.message == 'BadRequest':
                            # Assume it's already added
                            os.unlink(path)
                            app.logger.warning('deleted torrent, already added : %s' % (name,))
                        else:
                            raise e

                    c.execute('insert into torrents (name, size) values (?, ?)', (name, size))
                    connection.commit()
                else:
                    os.unlink(path)
                    app.logger.warning('deleted torrent, added at %s : %s' % (row[0], name))

@manager.command
def torrents_watch(add_existing=True):
    torrents_add()

    class EventHandler(pyinotify.ProcessEvent):
        def process_IN_CLOSE_WRITE(self, event):
            app.logger.debug('received event: %s' % event)
            transfer = client.Transfer.add_torrent(event.pathname)
            app.logger.info('transfer added: %s' % transfer)

    wm = pyinotify.WatchManager()
    mask = pyinotify.IN_CLOSE_WRITE

    handler = EventHandler()
    notifier = pyinotify.Notifier(wm, handler)

    wdd = wm.add_watch(app.config['TORRENTS'], mask, rec=True)
    app.logger.debug('added watch: %s' % wdd)

    notifier.loop()

@manager.command
def files_download(limit=None):
    files = client.File.list()

    if len(files):
        with sqlite3.connect(app.config['DATABASE']) as connection:
            if limit is not None:
                downloaded = 0

            c = connection.cursor()

            for file in files:
                c.execute("select datetime(created_at, 'localtime') from downloads where name = ? and size = ?", (file.name, file.size))
                row = c.fetchone()

                if row is None:
                    app.logger.debug('downloading file: %s' % file)
                    file.download(dest=app.config['INCOMPLETE'], delete_after_download=True)
                    app.logger.info('downloaded file: %s' % file)

                    path = os.path.join(app.config['INCOMPLETE'], file.name)
                    shutil.move(path, app.config['DOWNLOADS'])

                    c.execute('insert into downloads (id, name, size) values (?, ?, ?)', (file.id, file.name, file.size))
                    connection.commit()

                    if limit is not None:
                        downloaded = downloaded + 1

                        if downloaded > limit:
                            break
                else:
                    app.logger.warning('file downloaded at %s : %s' % (row[0], file))

if __name__ == '__main__':
    manager.run()
<commit_msg>Blush<commit_after>#!/usr/bin/env python

import logging
import os
import putio
import pyinotify
import shutil
import sqlite3

from flask import *
from flask.ext.script import Manager

app = Flask(__name__)

app.logger.setLevel(logging.DEBUG)
app.config.from_object('config')

logging.basicConfig(filename=app.config.get('LOG_FILENAME'), level=app.config.get('LOG_LEVEL', logging.WARNING))

client = putio.Client(app.config['PUTIO_TOKEN'])

manager = Manager(app)

@manager.command
def transfers_cancel_seeding():
    transfer_ids = []
    for transfer in client.Transfer.list():
        if transfer.status == 'SEEDING':
            transfer_ids.append(transfer.id)

    if len(transfer_ids):
        client.Transfer.cancel(transfer_ids)

@manager.command
def transfers_clean():
    client.Transfer.clean()

@manager.command
def transfers_groom():
    transfers_cancel_seeding()
    transfers_clean()

@manager.command
def torrents_add():
    files = os.listdir(app.config['TORRENTS'])

    if len(files):
        with sqlite3.connect(app.config['DATABASE']) as connection:
            c = connection.cursor()

            for name in os.listdir(app.config['TORRENTS']):
                path = os.path.join(app.config['TORRENTS'], name)
                size = os.path.getsize(path)

                c.execute("select datetime(created_at, 'localtime') from torrents where name = ? and size = ?", (name, size))
                row = c.fetchone()

                if row is None:
                    app.logger.debug('adding torrent: %s' % name)

                    try:
                        app.logger.info('adding torrent: %s' % path)
                        transfer = client.Transfer.add_torrent(path)
                        os.unlink(path)
                        app.logger.info('deleted torrent, added: %s' % transfer)
                    except Exception, e:
                        if e.message == 'BadRequest':
                            # Assume it's already added
                            os.unlink(path)
                            app.logger.warning('deleted torrent, already added : %s' % (name,))
                        else:
                            raise e

                    c.execute('insert into torrents (name, size) values (?, ?)', (name, size))
                    connection.commit()
                else:
                    os.unlink(path)
                    app.logger.warning('deleted torrent, added at %s : %s' % (row[0], name))

@manager.command
def torrents_watch(add_existing=True):
    torrents_add()

    class EventHandler(pyinotify.ProcessEvent):
        def process_IN_CLOSE_WRITE(self, event):
            app.logger.debug('received event: %s' % event)
            transfer = client.Transfer.add_torrent(event.pathname)
            app.logger.info('transfer added: %s' % transfer)

    wm = pyinotify.WatchManager()
    mask = pyinotify.IN_CLOSE_WRITE

    handler = EventHandler()
    notifier = pyinotify.Notifier(wm, handler)

    wdd = wm.add_watch(app.config['TORRENTS'], mask, rec=True)
    app.logger.debug('added watch: %s' % wdd)

    notifier.loop()

@manager.command
def files_download(limit=None):
    files = client.File.list()

    if len(files):
        with sqlite3.connect(app.config['DATABASE']) as connection:
            if limit is not None:
                downloaded = 0

            c = connection.cursor()

            for file in files:
                c.execute("select datetime(created_at, 'localtime') from downloads where name = ? and size = ?", (file.name, file.size))
                row = c.fetchone()

                if row is None:
                    app.logger.debug('downloading file: %s' % file)
                    file.download(dest=app.config['INCOMPLETE'], delete_after_download=True)
                    app.logger.info('downloaded file: %s' % file)

                    path = os.path.join(app.config['INCOMPLETE'], file.name)
                    shutil.move(path, app.config['DOWNLOADS'])

                    c.execute('insert into downloads (id, name, size) values (?, ?, ?)', (file.id, file.name, file.size))
                    connection.commit()

                    if limit is not None:
                        downloaded = downloaded + 1

                        if downloaded > limit:
                            break
                else:
                    app.logger.warning('file downloaded at %s : %s' % (row[0], file))

if __name__ == '__main__':
    manager.run()
<|endoftext|>


<commit_before>#!/usr/bin/env python
import os
import sys

if __name__ == "__main__":
    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "rpspot.settings")

    from django.core.management import execute_from_command_line

    execute_from_command_line(sys.argv)
<commit_msg>another change related to settings reorg<commit_after>#!/usr/bin/env python
import os
import sys

if __name__ == "__main__":
    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "rpspot.settings.dev")

    from django.core.management import execute_from_command_line

    execute_from_command_line(sys.argv)
<|endoftext|>


<commit_before>"""Perform joint genotyping using GATK HaplotypeCaller with gVCF inputs.

For GATK4, merges into a shared database using GenomicsDBImport. For GATK3
handles merging of large batch sizes using CombineGVCFs.
For both, follows this with joint variant calling using GenotypeGVCFs.
"""
import math
import os
import toolz as tz

from bcbio import broad, utils
from bcbio.distributed.transaction import file_transaction
from bcbio.pipeline import datadict as dd
from bcbio.variation import bamprep, vcfutils

def run_region(data, region, vrn_files, out_file):
    """Perform variant calling on gVCF inputs in a specific genomic region.
    """
    broad_runner = broad.runner_from_config(data["config"])
    if broad_runner.gatk_type() == "gatk4":
        genomics_db = _run_genomicsdb_import(vrn_files, region, out_file, data)
        return _run_genotype_gvcfs_genomicsdb(genomics_db, region, out_file, data)
    else:
        vrn_files = _batch_gvcfs(data, region, vrn_files, dd.get_ref_file(data), out_file)
        return _run_genotype_gvcfs_gatk3(data, region, vrn_files, dd.get_ref_file(data), out_file)

# ## gVCF joint calling -- GATK4

def _run_genomicsdb_import(vrn_files, region, out_file, data):
    """Create a GenomicsDB reference for all the variation files: GATK4.

    Not yet tested as scale, need to explore --batchSize to reduce memory
    usage if needed.
    """
    out_dir = "%s_genomicsdb" % utils.splitext_plus(out_file)[0]
    if not os.path.exists(out_dir):
        with file_transaction(data, out_dir) as tx_out_dir:
            broad_runner = broad.runner_from_config(data["config"])
            params = ["-T", "GenomicsDBImport",
                      "--genomicsDBWorkspace", tx_out_dir,
                      "-L", bamprep.region_to_gatk(region)]
            for vrn_file in vrn_files:
                params += ["--variant", vrn_file]
            cores = dd.get_cores(data)
            memscale = {"magnitude": 0.9 * cores, "direction": "increase"} if cores > 1 else None
            broad_runner.run_gatk(params, memscale)
    return out_dir

def _run_genotype_gvcfs_genomicsdb(genomics_db, region, out_file, data):
    """GenotypeGVCFs from a merged GenomicsDB input: GATK4.

    No core scaling -- not yet supported in GATK4.
    """
    if not utils.file_exists(out_file):
        with file_transaction(data, out_file) as tx_out_file:
            broad_runner = broad.runner_from_config(data["config"])
            params = ["-T", "GenotypeGVCFs",
                      "--variant", "gendb://%s" % genomics_db,
                      "-R", dd.get_ref_file(data),
                      "--output", tx_out_file,
                      "-L", bamprep.region_to_gatk(region)]
            cores = dd.get_cores(data)
            memscale = {"magnitude": 0.9 * cores, "direction": "increase"} if cores > 1 else None
            broad_runner.run_gatk(params, memscale)
    return vcfutils.bgzip_and_index(out_file, data["config"])

# ## gVCF joint genotype calling -- GATK3

def _run_genotype_gvcfs_gatk3(data, region, vrn_files, ref_file, out_file):
    """Performs genotyping of gVCFs into final VCF files.
    """
    if not utils.file_exists(out_file):
        broad_runner = broad.runner_from_config(data["config"])
        with file_transaction(data, out_file) as tx_out_file:
            assoc_files = tz.get_in(("genome_resources", "variation"), data, {})
            if not assoc_files: assoc_files = {}
            params = ["-T", "GenotypeGVCFs",
                      "-R", ref_file, "-o", tx_out_file,
                      "-L", bamprep.region_to_gatk(region),
                      "--max_alternate_alleles", "4"]
            for vrn_file in vrn_files:
                params += ["--variant", vrn_file]
            if assoc_files.get("dbsnp"):
                params += ["--dbsnp", assoc_files["dbsnp"]]
            broad_runner.new_resources("gatk-haplotype")
            cores = dd.get_cores(data)
            if cores > 1:
                # GATK performs poorly with memory usage when parallelizing
                # with a large number of cores but makes use of extra memory,
                # so we cap at 6 cores.
                # See issue #1565 for discussion
                # Recent GATK 3.x versions also have race conditions with multiple
                # threads, so limit to 1 and keep memory available
                # https://gatkforums.broadinstitute.org/wdl/discussion/8718/concurrentmodificationexception-in-gatk-3-7-genotypegvcfs
                # params += ["-nt", str(min(6, cores))]
                memscale = {"magnitude": 0.9 * cores, "direction": "increase"}
            else:
                memscale = None
            broad_runner.run_gatk(params, memscale=memscale)
    return vcfutils.bgzip_and_index(out_file, data["config"])

# ## gVCF batching

def _batch_gvcfs(data, region, vrn_files, ref_file, out_file=None):
    """Perform batching of gVCF files if above recommended input count.
    """
    if out_file is None:
        out_file = vrn_files[0]
    # group to get below the maximum batch size, using 200 as the baseline
    max_batch = int(dd.get_joint_group_size(data))
    if len(vrn_files) > max_batch:
        out = []
        num_batches = int(math.ceil(float(len(vrn_files)) / max_batch))
        for i, batch_vrn_files in enumerate(tz.partition_all(num_batches, vrn_files)):
            base, ext = utils.splitext_plus(out_file)
            batch_out_file = "%s-b%s%s" % (base, i, ext)
            out.append(run_combine_gvcfs(batch_vrn_files, region, ref_file, batch_out_file, data))
        return _batch_gvcfs(data, region, out, ref_file)
    else:
        return vrn_files

def run_combine_gvcfs(vrn_files, region, ref_file, out_file, data):
    if not utils.file_exists(out_file):
        broad_runner = broad.runner_from_config(data["config"])
        with file_transaction(data, out_file) as tx_out_file:
            params = ["-T", "CombineGVCFs", "-R", ref_file, "-o", tx_out_file]
            if region:
                params += ["-L", bamprep.region_to_gatk(region)]
            for vrn_file in vrn_files:
                params += ["--variant", vrn_file]
            cores = dd.get_cores(data)
            memscale = {"magnitude": 0.9 * cores, "direction": "increase"} if cores > 1 else None
            broad_runner.new_resources("gatk-haplotype")
            broad_runner.run_gatk(params, memscale=memscale)
    return vcfutils.bgzip_and_index(out_file, data["config"])
<commit_msg>GATK4: correctly pass memscale to joint callers<commit_after>"""Perform joint genotyping using GATK HaplotypeCaller with gVCF inputs.

For GATK4, merges into a shared database using GenomicsDBImport. For GATK3
handles merging of large batch sizes using CombineGVCFs.
For both, follows this with joint variant calling using GenotypeGVCFs.
"""
import math
import os
import toolz as tz

from bcbio import broad, utils
from bcbio.distributed.transaction import file_transaction
from bcbio.pipeline import datadict as dd
from bcbio.variation import bamprep, vcfutils

def run_region(data, region, vrn_files, out_file):
    """Perform variant calling on gVCF inputs in a specific genomic region.
    """
    broad_runner = broad.runner_from_config(data["config"])
    if broad_runner.gatk_type() == "gatk4":
        genomics_db = _run_genomicsdb_import(vrn_files, region, out_file, data)
        return _run_genotype_gvcfs_genomicsdb(genomics_db, region, out_file, data)
    else:
        vrn_files = _batch_gvcfs(data, region, vrn_files, dd.get_ref_file(data), out_file)
        return _run_genotype_gvcfs_gatk3(data, region, vrn_files, dd.get_ref_file(data), out_file)

# ## gVCF joint calling -- GATK4

def _run_genomicsdb_import(vrn_files, region, out_file, data):
    """Create a GenomicsDB reference for all the variation files: GATK4.

    Not yet tested as scale, need to explore --batchSize to reduce memory
    usage if needed.
    """
    out_dir = "%s_genomicsdb" % utils.splitext_plus(out_file)[0]
    if not os.path.exists(out_dir):
        with file_transaction(data, out_dir) as tx_out_dir:
            broad_runner = broad.runner_from_config(data["config"])
            params = ["-T", "GenomicsDBImport",
                      "--genomicsDBWorkspace", tx_out_dir,
                      "-L", bamprep.region_to_gatk(region)]
            for vrn_file in vrn_files:
                params += ["--variant", vrn_file]
            cores = dd.get_cores(data)
            memscale = {"magnitude": 0.9 * cores, "direction": "increase"} if cores > 1 else None
            broad_runner.run_gatk(params, memscale=memscale)
    return out_dir

def _run_genotype_gvcfs_genomicsdb(genomics_db, region, out_file, data):
    """GenotypeGVCFs from a merged GenomicsDB input: GATK4.

    No core scaling -- not yet supported in GATK4.
    """
    if not utils.file_exists(out_file):
        with file_transaction(data, out_file) as tx_out_file:
            broad_runner = broad.runner_from_config(data["config"])
            params = ["-T", "GenotypeGVCFs",
                      "--variant", "gendb://%s" % genomics_db,
                      "-R", dd.get_ref_file(data),
                      "--output", tx_out_file,
                      "-L", bamprep.region_to_gatk(region)]
            cores = dd.get_cores(data)
            memscale = {"magnitude": 0.9 * cores, "direction": "increase"} if cores > 1 else None
            broad_runner.run_gatk(params, memscale=memscale)
    return vcfutils.bgzip_and_index(out_file, data["config"])

# ## gVCF joint genotype calling -- GATK3

def _run_genotype_gvcfs_gatk3(data, region, vrn_files, ref_file, out_file):
    """Performs genotyping of gVCFs into final VCF files.
    """
    if not utils.file_exists(out_file):
        broad_runner = broad.runner_from_config(data["config"])
        with file_transaction(data, out_file) as tx_out_file:
            assoc_files = tz.get_in(("genome_resources", "variation"), data, {})
            if not assoc_files: assoc_files = {}
            params = ["-T", "GenotypeGVCFs",
                      "-R", ref_file, "-o", tx_out_file,
                      "-L", bamprep.region_to_gatk(region),
                      "--max_alternate_alleles", "4"]
            for vrn_file in vrn_files:
                params += ["--variant", vrn_file]
            if assoc_files.get("dbsnp"):
                params += ["--dbsnp", assoc_files["dbsnp"]]
            broad_runner.new_resources("gatk-haplotype")
            cores = dd.get_cores(data)
            if cores > 1:
                # GATK performs poorly with memory usage when parallelizing
                # with a large number of cores but makes use of extra memory,
                # so we cap at 6 cores.
                # See issue #1565 for discussion
                # Recent GATK 3.x versions also have race conditions with multiple
                # threads, so limit to 1 and keep memory available
                # https://gatkforums.broadinstitute.org/wdl/discussion/8718/concurrentmodificationexception-in-gatk-3-7-genotypegvcfs
                # params += ["-nt", str(min(6, cores))]
                memscale = {"magnitude": 0.9 * cores, "direction": "increase"}
            else:
                memscale = None
            broad_runner.run_gatk(params, memscale=memscale)
    return vcfutils.bgzip_and_index(out_file, data["config"])

# ## gVCF batching

def _batch_gvcfs(data, region, vrn_files, ref_file, out_file=None):
    """Perform batching of gVCF files if above recommended input count.
    """
    if out_file is None:
        out_file = vrn_files[0]
    # group to get below the maximum batch size, using 200 as the baseline
    max_batch = int(dd.get_joint_group_size(data))
    if len(vrn_files) > max_batch:
        out = []
        num_batches = int(math.ceil(float(len(vrn_files)) / max_batch))
        for i, batch_vrn_files in enumerate(tz.partition_all(num_batches, vrn_files)):
            base, ext = utils.splitext_plus(out_file)
            batch_out_file = "%s-b%s%s" % (base, i, ext)
            out.append(run_combine_gvcfs(batch_vrn_files, region, ref_file, batch_out_file, data))
        return _batch_gvcfs(data, region, out, ref_file)
    else:
        return vrn_files

def run_combine_gvcfs(vrn_files, region, ref_file, out_file, data):
    if not utils.file_exists(out_file):
        broad_runner = broad.runner_from_config(data["config"])
        with file_transaction(data, out_file) as tx_out_file:
            params = ["-T", "CombineGVCFs", "-R", ref_file, "-o", tx_out_file]
            if region:
                params += ["-L", bamprep.region_to_gatk(region)]
            for vrn_file in vrn_files:
                params += ["--variant", vrn_file]
            cores = dd.get_cores(data)
            memscale = {"magnitude": 0.9 * cores, "direction": "increase"} if cores > 1 else None
            broad_runner.new_resources("gatk-haplotype")
            broad_runner.run_gatk(params, memscale=memscale)
    return vcfutils.bgzip_and_index(out_file, data["config"])
<|endoftext|>


<commit_before>#!/usr/bin/env python3
"""Sample Sheet generation for BCL2FASTQ pipeline
"""
# --- standard library imports
#
import sys
import os
import logging
import argparse
from datetime import datetime

#--- third-party imports
#
import yaml
import requests
import xml.etree.ElementTree as ET

#--- project specific imports
# /

# global logger
# http://docs.python.org/library/logging.html
LOG = logging.getLogger("")
logging.basicConfig(level=logging.INFO,
                    format='%(levelname)s [%(asctime)s]: %(message)s')

SAMPLESHEET_HEADER = '[Data]'+'\n'+ 'Lane,Sample_ID,Sample_Name,Sample_Plate,Sample_Well,I7_Index_ID,index,I5_Index_ID,index2,Sample_Project,Description'

def generate_timestamp():
    """generate ISO8601 timestamp incl microsends
    """
    return datetime.isoformat(datetime.now())

def cmdline_parser():
    """
    creates a argparse instance
    """

    parser = argparse.ArgumentParser(description=__doc__)

    parser.add_argument("-v", "--verbose",
                        action="store_true", dest="verbose",
                        help="be verbose")
    parser.add_argument("--debug",
                        action="store_true", dest="debug",
                        help="debugging")
    parser.add_argument("-r", "--runIDPath",
                        dest="runIDPath",
                        required=True,
                        help="runIDPath, e.g. /mnt/seq/userrig/HS004/HS004-PE-R00139_BC6A7HANXX")
    parser.add_argument("-o", "--outdir",
                        required=True,
                        dest="outdir",
                        help="output directory, e.g.  /mnt/projects/rpd/testing/output/bcl2fastq ")
    return parser

def getdirs(args):
    runIDPath = args.runIDPath
    if not os.path.exists(runIDPath):
        LOG.fatal("runIDPath '%s' does not exist under Run directory.\n" % (runIDPath))
        sys.exit(1)

    RunInfo = os.path.join(args.runIDPath + '/RunInfo.xml')
    if not os.path.exists(RunInfo):
        LOG.fatal("RunInfo '%s' does not exist under Run directory.\n" % (RunInfo))
        sys.exit(1)

    outdir = args.outdir
    if not os.path.exists(outdir):
        LOG.fatal("output directory '%s' does not exist.\n" % (outdir))
        sys.exit(1)

    return(runIDPath,outdir,RunInfo)

def generateUseBases(barcode_lens, RunInfo):
    tree = ET.parse(RunInfo)
    root = tree.getroot()
    UB_list = []

    # for each lane and its barcode lengths
    for k, v in sorted(barcode_lens.items()):
        # v is list of barcode_len tuples
        assert len(set(v))==1, ("Different barcode length in lane {}".format(k))
        BC1, BC2 = v[0]# since all v's are the same

        UB = ""
        #if test:
        for Read in root.iter('Read'):
            NumCyc = int(Read.attrib['NumCycles'])-1
            if Read.attrib['IsIndexedRead'] == 'N':
                UB+='Y'+str(NumCyc)+'n*,'
            elif Read.attrib['IsIndexedRead'] == 'Y':
                if Read.attrib['Number'] == '2':    ### BC1
                    if BC1 > 0:
                        UB+='I'+str(BC1)+'n*,'
                    else:
                        UB+='n*'+','
                if Read.attrib['Number'] == '3':    ### BC2
                    if BC2 > 0:
                        UB+='I'+str(BC2)+'n*,'
                    else:
                        UB+='n*'+','
        UB = UB[:-1]
        UB_list.append(str(k+':'+UB))
    return (UB_list)

def main():
    """
    The main function
    """
    parser = cmdline_parser()
    args = parser.parse_args()

    if args.verbose:
        LOG.setLevel(logging.INFO)
    if args.debug:
        LOG.setLevel(logging.DEBUG)

    (runIDPath,outdirbase,RunInfo) = getdirs(args)
    runid_with_flowcellid = runIDPath.split('/')[-1]
    machine_id = runid_with_flowcellid.split('-')[0]

    # keys: lanes, values are barcode lens in lane (always two tuples, -1 if not present)
    barcode_lens = {}
    LOG.info("Generating sample sheet")
    run_num = runid_with_flowcellid.split('_')[0]

    #rest_url = 'http://dlap51v:8080/elm/rest/seqrun/illumina/' + run_num + '/detailanalysis/json'
    rest_url = 'http://qldb01:8080/rest/seqrun/illumina/' + run_num + '/detailanalysis/json'
    response = requests.get(rest_url)
    rest_data = response.json()

    outdir = os.path.join(outdirbase, machine_id,
                            runid_with_flowcellid + '_' + generate_timestamp())
    assert not os.path.exists(outdir)
    os.makedirs(outdir)
    run_id = rest_data['runId']
    counter = 0
    if rest_data['runPass'] != 'Pass':
        LOG.info("Skipping non-passed run")
        sys.exit(0)
    # this is the master samplesheet
    samplesheet = os.path.join(outdir, run_id + '_sampleSheet.csv')
    with open(samplesheet, 'w') as fh_out:
        fh_out.write(SAMPLESHEET_HEADER + '\n')
        for rows in rest_data['lanes']:
            if rows['lanePass'] == 'Pass':
                if "MUX" in rows['libraryId']:
                    # multiplexed
                    counter = 0
                    for child in rows['Children']:
                        counter += 1
                        id = 'S'+str(counter)
                        if "-" in (child['barcode']):
                            # dual index
                            #print (child['barcode'])
                            index = child['barcode'].split('-')
                            #print (index[0])
                            #print (index[1])
                            sample = rows['laneId']+',Sample_'+child['libraryId']+','+child['libraryId']+'-'+child['barcode']+',,,'+id+','+ index[0] +',,'+ index[1] + ',' +'Project_'+rows['libraryId']+','+child['libtech']
                            #index_lens = (len(index[0], len(index[1]))
                            index_lens = (len((index[0])), len((index[1])))
                        else:
                            sample = rows['laneId']+',Sample_'+child['libraryId']+','+child['libraryId']+'-'+child['barcode']+',,,'+id+','+child['barcode']+',,,'+'Project_'+rows['libraryId']+','+child['libtech']
                            index_lens = (len(child['barcode']), -1)
                        barcode_lens.setdefault(rows['laneId'], []).append(index_lens)
                        fh_out.write(sample+ '\n')
                else:
                    # non-multiplexed
                    sample = rows['laneId']+',Sample_'+rows['libraryId']+','+rows['libraryId']+'-NoIndex'+',,,,,,,'+'Project_'+rows['libraryId']+','+rows['libtech']
                    index_lens = (-1, -1)
                    barcode_lens.setdefault(rows['laneId'], []).append(index_lens)
                    fh_out.write(sample+ '\n')
    LOG.info("Generating UseBases")
    UseBases = generateUseBases(barcode_lens, RunInfo)
    UseBases_data = dict(UseBases = UseBases )
    config = os.path.join(outdir,run_id + '_config.yml')
    with open(config, 'w') as outfile:
        outfile.write( yaml.dump(UseBases_data, default_flow_style=True) )

if __name__ == "__main__":
    main()
    LOG.info("Successful program exit")
<commit_msg>Added sample_info yaml<commit_after>#!/usr/bin/env python3
"""Sample Sheet generation for BCL2FASTQ pipeline
"""
# --- standard library imports
#
import sys
import os
import logging
import argparse
from datetime import datetime

#--- third-party imports
#
import yaml
import requests
import xml.etree.ElementTree as ET

#--- project specific imports
# /

# global logger
# http://docs.python.org/library/logging.html
LOG = logging.getLogger("")
logging.basicConfig(level=logging.INFO,
                    format='%(levelname)s [%(asctime)s]: %(message)s')

SAMPLESHEET_HEADER = '[Data]'+'\n'+ 'Lane,Sample_ID,Sample_Name,Sample_Plate,Sample_Well,I7_Index_ID,index,I5_Index_ID,index2,Sample_Project,Description'

def generate_timestamp():
    """generate ISO8601 timestamp incl microsends
    """
    return datetime.isoformat(datetime.now())

def cmdline_parser():
    """
    creates a argparse instance
    """

    parser = argparse.ArgumentParser(description=__doc__)

    parser.add_argument("-v", "--verbose",
                        action="store_true", dest="verbose",
                        help="be verbose")
    parser.add_argument("--debug",
                        action="store_true", dest="debug",
                        help="debugging")
    parser.add_argument("-r", "--runIDPath",
                        dest="runIDPath",
                        required=True,
                        help="runIDPath, e.g. /mnt/seq/userrig/HS004/HS004-PE-R00139_BC6A7HANXX")
    parser.add_argument("-o", "--outdir",
                        required=True,
                        dest="outdir",
                        help="output directory, e.g.  /mnt/projects/rpd/testing/output/bcl2fastq ")
    return parser

def getdirs(args):
    runIDPath = args.runIDPath
    if not os.path.exists(runIDPath):
        LOG.fatal("runIDPath '%s' does not exist under Run directory.\n" % (runIDPath))
        sys.exit(1)

    RunInfo = os.path.join(args.runIDPath + '/RunInfo.xml')
    if not os.path.exists(RunInfo):
        LOG.fatal("RunInfo '%s' does not exist under Run directory.\n" % (RunInfo))
        sys.exit(1)

    outdir = args.outdir
    if not os.path.exists(outdir):
        LOG.fatal("output directory '%s' does not exist.\n" % (outdir))
        sys.exit(1)

    return(runIDPath,outdir,RunInfo)

def generateUseBases(barcode_lens, RunInfo):
    tree = ET.parse(RunInfo)
    root = tree.getroot()
    UB_list = []

    # for each lane and its barcode lengths
    for k, v in sorted(barcode_lens.items()):
        # v is list of barcode_len tuples
        assert len(set(v))==1, ("Different barcode length in lane {}".format(k))
        BC1, BC2 = v[0]# since all v's are the same

        UB = ""
        #if test:
        for Read in root.iter('Read'):
            NumCyc = int(Read.attrib['NumCycles'])-1
            if Read.attrib['IsIndexedRead'] == 'N':
                UB+='Y'+str(NumCyc)+'n*,'
            elif Read.attrib['IsIndexedRead'] == 'Y':
                if Read.attrib['Number'] == '2':    ### BC1
                    if BC1 > 0:
                        UB+='I'+str(BC1)+'n*,'
                    else:
                        UB+='n*'+','
                if Read.attrib['Number'] == '3':    ### BC2
                    if BC2 > 0:
                        UB+='I'+str(BC2)+'n*,'
                    else:
                        UB+='n*'+','
        UB = UB[:-1]
        UB_list.append(str(k+':'+UB))
    return (UB_list)

def main():
    """
    The main function
    """
    parser = cmdline_parser()
    args = parser.parse_args()

    if args.verbose:
        LOG.setLevel(logging.INFO)
    if args.debug:
        LOG.setLevel(logging.DEBUG)

    (runIDPath,outdirbase,RunInfo) = getdirs(args)
    runid_with_flowcellid = runIDPath.split('/')[-1]
    flowcellid = runid_with_flowcellid.split('_')[-1]
    machine_id = runid_with_flowcellid.split('-')[0]

    # keys: lanes, values are barcode lens in lane (always two tuples, -1 if not present)
    barcode_lens = {}
    sample_info = []
    LOG.info("Generating sample sheet")
    run_num = runid_with_flowcellid.split('_')[0]

    #rest_url = 'http://dlap51v:8080/elm/rest/seqrun/illumina/' + run_num + '/detailanalysis/json'
    rest_url = 'http://qldb01:8080/rest/seqrun/illumina/' + run_num + '/detailanalysis/json'
    response = requests.get(rest_url)
    rest_data = response.json()

    outdir = os.path.join(outdirbase, machine_id,
                            runid_with_flowcellid + '_' + generate_timestamp())
    assert not os.path.exists(outdir)
    os.makedirs(outdir)
    run_id = rest_data['runId']
    counter = 0
    if rest_data['runPass'] != 'Pass':
        LOG.info("Skipping non-passed run")
        sys.exit(0)
    # this is the master samplesheet
    samplesheet = os.path.join(outdir, run_id + '_sampleSheet.csv')
    with open(samplesheet, 'w') as fh_out:
        fh_out.write(SAMPLESHEET_HEADER + '\n')
        for rows in rest_data['lanes']:
            if rows['lanePass'] == 'Pass':
                if "MUX" in rows['libraryId']:
                    # multiplexed
                    counter = 0
                    for child in rows['Children']:
                        counter += 1
                        id = 'S'+str(counter)
                        if "-" in (child['barcode']):
                            # dual index
                            index = child['barcode'].split('-')
                            sample = rows['laneId']+',Sample_'+child['libraryId']+','+child['libraryId']+'-'+child['barcode']+',,,'+id+','+ index[0] +',,'+ index[1] + ',' +'Project_'+rows['libraryId']+','+child['libtech']
                            index_lens = (len((index[0])), len((index[1])))
                            sample_dir = os.path.join(outdir, machine_id, + 'Project_' + rows['libraryId']+','+ 'Sample_' + child['libraryId'])
                            #print ('TEST' + sample_dir)
                            sample_dir = outdir + '/' + machine_id + '/' + 'Project_' + rows['libraryId'] + 'Sample_' + child['libraryId']
                            sample_id = run_id + ',' + flowcellid + ',' + child['libraryId'] + ',' + rows['laneId'] + ',' + sample_dir
                            print (sample_id)

                        else:
                            sample = rows['laneId']+',Sample_'+child['libraryId']+','+child['libraryId']+'-'+child['barcode']+',,,'+id+','+child['barcode']+',,,'+'Project_'+rows['libraryId']+','+child['libtech']
                            index_lens = (len(child['barcode']), -1)
                            sample_dir = outdir + '/' + machine_id + '/' + 'Project_' + rows['libraryId'] + 'Sample_' + child['libraryId']
                            sample_id = run_id + ',' + flowcellid + ',' + child['libraryId'] + ',' + rows['laneId']+ ',' + sample_dir
                            print (sample_id)

                        sample_info.append(sample_id)
                        barcode_lens.setdefault(rows['laneId'], []).append(index_lens)
                        fh_out.write(sample+ '\n')
                else:
                    # non-multiplexed
                    sample = rows['laneId']+',Sample_'+rows['libraryId']+','+rows['libraryId']+'-NoIndex'+',,,,,,,'+'Project_'+rows['libraryId']+','+rows['libtech']
                    index_lens = (-1, -1)
                    barcode_lens.setdefault(rows['laneId'], []).append(index_lens)
                    sample_dir = outdir + '/' + machine_id + '/' + 'Project_' + rows['libraryId'] + 'Sample_' + rows['libraryId']
                    sample_id = run_id + ',' + flowcellid + ',' + child['libraryId'] + ',' + rows['laneId']+ ',' + sample_dir
                    sample_info.append(sample_id)
                    fh_out.write(sample+ '\n')
    LOG.info("Generating UseBases")
    UseBases = generateUseBases(barcode_lens, RunInfo)
    UseBases_data = dict(UseBases = UseBases )

    config_useBase = os.path.join(outdir,run_id + '_useBases.yaml')
    with open(config_useBase, 'w') as outfile:
        outfile.write( yaml.dump(UseBases_data, default_flow_style=True))

    sample_info_yaml = dict(sample_info = sample_info )
    config_sample_info = os.path.join(outdir,run_id + '_sampleInfo.yaml')
    with open(config_sample_info, 'w') as outfile:
        outfile.write( yaml.dump(sample_info_yaml, default_flow_style=True))

if __name__ == "__main__":
    main()
    LOG.info("Successful program exit")
<|endoftext|>


<commit_before>"""
Benchmarks of Non-Negative Matrix Factorization
"""

from __future__ import print_function

from collections import defaultdict
import gc
from time import time

import six

import numpy as np
from scipy.linalg import norm

from sklearn.decomposition.nmf import NMF, _initialize_nmf
from sklearn.datasets.samples_generator import make_low_rank_matrix
from sklearn.externals.six.moves import xrange

def alt_nnmf(V, r, max_iter=1000, tol=1e-3, init='random'):
    """
    A, S = nnmf(X, r, tol=1e-3, R=None)

    Implement Lee & Seung's algorithm

    Parameters
    ----------
    V : 2-ndarray, [n_samples, n_features]
        input matrix
    r : integer
        number of latent features
    max_iter : integer, optional
        maximum number of iterations (default: 1000)
    tol : double
        tolerance threshold for early exit (when the update factor is within
        tol of 1., the function exits)
    init : string
        Method used to initialize the procedure.

    Returns
    -------
    A : 2-ndarray, [n_samples, r]
        Component part of the factorization

    S : 2-ndarray, [r, n_features]
        Data part of the factorization
    Reference
    ---------
    "Algorithms for Non-negative Matrix Factorization"
    by Daniel D Lee, Sebastian H Seung
    (available at http://citeseer.ist.psu.edu/lee01algorithms.html)
    """
    # Nomenclature in the function follows Lee & Seung
    eps = 1e-5
    n, m = V.shape
    W, H = _initialize_nmf(V, r, init, random_state=0)

    for i in xrange(max_iter):
        updateH = np.dot(W.T, V) / (np.dot(np.dot(W.T, W), H) + eps)
        H *= updateH
        updateW = np.dot(V, H.T) / (np.dot(W, np.dot(H, H.T)) + eps)
        W *= updateW
        if i % 10 == 0:
            max_update = max(updateW.max(), updateH.max())
            if abs(1. - max_update) < tol:
                break
    return W, H

def report(error, time):
    print("Frobenius loss: %.5f" % error)
    print("Took: %.2fs" % time)
    print()

def benchmark(samples_range, features_range, rank=50, tolerance=1e-5):
    timeset = defaultdict(lambda: [])
    err = defaultdict(lambda: [])

    for n_samples in samples_range:
        for n_features in features_range:
            print("%2d samples, %2d features" % (n_samples, n_features))
            print('=======================')
            X = np.abs(make_low_rank_matrix(n_samples, n_features,
                       effective_rank=rank, tail_strength=0.2))

            gc.collect()
            print("benchmarking nndsvd-nmf: ")
            tstart = time()
            m = NMF(n_components=30, tol=tolerance, init='nndsvd').fit(X)
            tend = time() - tstart
            timeset['nndsvd-nmf'].append(tend)
            err['nndsvd-nmf'].append(m.reconstruction_err_)
            report(m.reconstruction_err_, tend)

            gc.collect()
            print("benchmarking nndsvda-nmf: ")
            tstart = time()
            m = NMF(n_components=30, init='nndsvda',
                    tol=tolerance).fit(X)
            tend = time() - tstart
            timeset['nndsvda-nmf'].append(tend)
            err['nndsvda-nmf'].append(m.reconstruction_err_)
            report(m.reconstruction_err_, tend)

            gc.collect()
            print("benchmarking nndsvdar-nmf: ")
            tstart = time()
            m = NMF(n_components=30, init='nndsvdar',
                    tol=tolerance).fit(X)
            tend = time() - tstart
            timeset['nndsvdar-nmf'].append(tend)
            err['nndsvdar-nmf'].append(m.reconstruction_err_)
            report(m.reconstruction_err_, tend)

            gc.collect()
            print("benchmarking random-nmf")
            tstart = time()
            m = NMF(n_components=30, init='random', max_iter=1000,
                    tol=tolerance).fit(X)
            tend = time() - tstart
            timeset['random-nmf'].append(tend)
            err['random-nmf'].append(m.reconstruction_err_)
            report(m.reconstruction_err_, tend)

            gc.collect()
            print("benchmarking alt-random-nmf")
            tstart = time()
            W, H = alt_nnmf(X, r=30, init='random', tol=tolerance)
            tend = time() - tstart
            timeset['alt-random-nmf'].append(tend)
            err['alt-random-nmf'].append(np.linalg.norm(X - np.dot(W, H)))
            report(norm(X - np.dot(W, H)), tend)

    return timeset, err

if __name__ == '__main__':
    from mpl_toolkits.mplot3d import axes3d  # register the 3d projection
    axes3d
    import matplotlib.pyplot as plt

    samples_range = np.linspace(50, 500, 3).astype(np.int)
    features_range = np.linspace(50, 500, 3).astype(np.int)
    timeset, err = benchmark(samples_range, features_range)

    for i, results in enumerate((timeset, err)):
        fig = plt.figure('scikit-learn Non-Negative Matrix Factorization'
                         'benchmark results')
        ax = fig.gca(projection='3d')
        for c, (label, timings) in zip('rbgcm', sorted(six.iteritems(results))):
            X, Y = np.meshgrid(samples_range, features_range)
            Z = np.asarray(timings).reshape(samples_range.shape[0],
                                            features_range.shape[0])
            # plot the actual surface
            ax.plot_surface(X, Y, Z, rstride=8, cstride=8, alpha=0.3,
                            color=c)
            # dummy point plot to stick the legend to since surface plot do not
            # support legends (yet?)
            ax.plot([1], [1], [1], color=c, label=label)

        ax.set_xlabel('n_samples')
        ax.set_ylabel('n_features')
        zlabel = 'Time (s)' if i == 0 else 'reconstruction error'
        ax.set_zlabel(zlabel)
        ax.legend()
        plt.show()
<commit_msg>DOC Improve benchmark on NMF (#5779)<commit_after>"""
Benchmarks of Non-Negative Matrix Factorization
"""
# Authors: Tom Dupre la Tour (benchmark)
#          Chih-Jen Linn (original projected gradient NMF implementation)
#          Anthony Di Franco (projected gradient, Python and NumPy port)
# License: BSD 3 clause

from __future__ import print_function
from time import time
import sys
import warnings
import numbers

import numpy as np
import matplotlib.pyplot as plt
import pandas

from sklearn.utils.testing import ignore_warnings
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.decomposition.nmf import NMF
from sklearn.decomposition.nmf import _initialize_nmf
from sklearn.decomposition.nmf import _beta_divergence
from sklearn.decomposition.nmf import INTEGER_TYPES, _check_init
from sklearn.externals.joblib import Memory
from sklearn.exceptions import ConvergenceWarning
from sklearn.utils.extmath import fast_dot, safe_sparse_dot, squared_norm
from sklearn.utils import check_array
from sklearn.utils.validation import check_is_fitted, check_non_negative

mem = Memory(cachedir='.', verbose=0)

###################
# Start of _PGNMF #
###################
# This class implements a projected gradient solver for the NMF.
# The projected gradient solver was removed from scikit-learn in version 0.19,
# and a simplified copy is used here for comparison purpose only.
# It is not tested, and it may change or disappear without notice.

def _norm(x):
    """Dot product-based Euclidean norm implementation
    See: http://fseoane.net/blog/2011/computing-the-vector-norm/
    """
    return np.sqrt(squared_norm(x))

def _nls_subproblem(X, W, H, tol, max_iter, alpha=0., l1_ratio=0.,
                    sigma=0.01, beta=0.1):
    """Non-negative least square solver
    Solves a non-negative least squares subproblem using the projected
    gradient descent algorithm.
    Parameters
    ----------
    X : array-like, shape (n_samples, n_features)
        Constant matrix.
    W : array-like, shape (n_samples, n_components)
        Constant matrix.
    H : array-like, shape (n_components, n_features)
        Initial guess for the solution.
    tol : float
        Tolerance of the stopping condition.
    max_iter : int
        Maximum number of iterations before timing out.
    alpha : double, default: 0.
        Constant that multiplies the regularization terms. Set it to zero to
        have no regularization.
    l1_ratio : double, default: 0.
        The regularization mixing parameter, with 0 <= l1_ratio <= 1.
        For l1_ratio = 0 the penalty is an L2 penalty.
        For l1_ratio = 1 it is an L1 penalty.
        For 0 < l1_ratio < 1, the penalty is a combination of L1 and L2.
    sigma : float
        Constant used in the sufficient decrease condition checked by the line
        search.  Smaller values lead to a looser sufficient decrease condition,
        thus reducing the time taken by the line search, but potentially
        increasing the number of iterations of the projected gradient
        procedure. 0.01 is a commonly used value in the optimization
        literature.
    beta : float
        Factor by which the step size is decreased (resp. increased) until
        (resp. as long as) the sufficient decrease condition is satisfied.
        Larger values allow to find a better step size but lead to longer line
        search. 0.1 is a commonly used value in the optimization literature.
    Returns
    -------
    H : array-like, shape (n_components, n_features)
        Solution to the non-negative least squares problem.
    grad : array-like, shape (n_components, n_features)
        The gradient.
    n_iter : int
        The number of iterations done by the algorithm.
    References
    ----------
    C.-J. Lin. Projected gradient methods for non-negative matrix
    factorization. Neural Computation, 19(2007), 2756-2779.
    http://www.csie.ntu.edu.tw/~cjlin/nmf/
    """
    WtX = safe_sparse_dot(W.T, X)
    WtW = fast_dot(W.T, W)

    # values justified in the paper (alpha is renamed gamma)
    gamma = 1
    for n_iter in range(1, max_iter + 1):
        grad = np.dot(WtW, H) - WtX
        if alpha > 0 and l1_ratio == 1.:
            grad += alpha
        elif alpha > 0:
            grad += alpha * (l1_ratio + (1 - l1_ratio) * H)

        # The following multiplication with a boolean array is more than twice
        # as fast as indexing into grad.
        if _norm(grad * np.logical_or(grad < 0, H > 0)) < tol:
            break

        Hp = H

        for inner_iter in range(20):
            # Gradient step.
            Hn = H - gamma * grad
            # Projection step.
            Hn *= Hn > 0
            d = Hn - H
            gradd = np.dot(grad.ravel(), d.ravel())
            dQd = np.dot(np.dot(WtW, d).ravel(), d.ravel())
            suff_decr = (1 - sigma) * gradd + 0.5 * dQd < 0
            if inner_iter == 0:
                decr_gamma = not suff_decr

            if decr_gamma:
                if suff_decr:
                    H = Hn
                    break
                else:
                    gamma *= beta
            elif not suff_decr or (Hp == Hn).all():
                H = Hp
                break
            else:
                gamma /= beta
                Hp = Hn

    if n_iter == max_iter:
        warnings.warn("Iteration limit reached in nls subproblem.",
                      ConvergenceWarning)

    return H, grad, n_iter

def _fit_projected_gradient(X, W, H, tol, max_iter, nls_max_iter, alpha,
                            l1_ratio):
    gradW = (np.dot(W, np.dot(H, H.T)) -
             safe_sparse_dot(X, H.T, dense_output=True))
    gradH = (np.dot(np.dot(W.T, W), H) -
             safe_sparse_dot(W.T, X, dense_output=True))

    init_grad = squared_norm(gradW) + squared_norm(gradH.T)
    # max(0.001, tol) to force alternating minimizations of W and H
    tolW = max(0.001, tol) * np.sqrt(init_grad)
    tolH = tolW

    for n_iter in range(1, max_iter + 1):
        # stopping condition as discussed in paper
        proj_grad_W = squared_norm(gradW * np.logical_or(gradW < 0, W > 0))
        proj_grad_H = squared_norm(gradH * np.logical_or(gradH < 0, H > 0))

        if (proj_grad_W + proj_grad_H) / init_grad < tol ** 2:
            break

        # update W
        Wt, gradWt, iterW = _nls_subproblem(X.T, H.T, W.T, tolW, nls_max_iter,
                                            alpha=alpha, l1_ratio=l1_ratio)
        W, gradW = Wt.T, gradWt.T

        if iterW == 1:
            tolW = 0.1 * tolW

        # update H
        H, gradH, iterH = _nls_subproblem(X, W, H, tolH, nls_max_iter,
                                          alpha=alpha, l1_ratio=l1_ratio)
        if iterH == 1:
            tolH = 0.1 * tolH

    H[H == 0] = 0   # fix up negative zeros

    if n_iter == max_iter:
        Wt, _, _ = _nls_subproblem(X.T, H.T, W.T, tolW, nls_max_iter,
                                   alpha=alpha, l1_ratio=l1_ratio)
        W = Wt.T

    return W, H, n_iter

class _PGNMF(NMF):
    """Non-Negative Matrix Factorization (NMF) with projected gradient solver.

    This class is private and for comparison purpose only.
    It may change or disappear without notice.

    """
    def __init__(self, n_components=None, solver='pg', init=None,
                 tol=1e-4, max_iter=200, random_state=None,
                 alpha=0., l1_ratio=0., nls_max_iter=10):
        self.nls_max_iter = nls_max_iter
        self.n_components = n_components
        self.init = init
        self.solver = solver
        self.tol = tol
        self.max_iter = max_iter
        self.random_state = random_state
        self.alpha = alpha
        self.l1_ratio = l1_ratio

    def fit(self, X, y=None, **params):
        self.fit_transform(X, **params)
        return self

    def transform(self, X):
        check_is_fitted(self, 'components_')
        H = self.components_
        W, _, self.n_iter_ = self._fit_transform(X, H=H, update_H=False)
        return W

    def inverse_transform(self, W):
        check_is_fitted(self, 'components_')
        return np.dot(W, self.components_)

    def fit_transform(self, X, y=None, W=None, H=None):
        W, H, self.n_iter = self._fit_transform(X, W=W, H=H, update_H=True)
        self.components_ = H
        return W

    def _fit_transform(self, X, y=None, W=None, H=None, update_H=True):
        X = check_array(X, accept_sparse=('csr', 'csc'))
        check_non_negative(X, "NMF (input X)")

        n_samples, n_features = X.shape
        n_components = self.n_components
        if n_components is None:
            n_components = n_features

        if (not isinstance(n_components, INTEGER_TYPES) or
                n_components <= 0):
            raise ValueError("Number of components must be a positive integer;"
                             " got (n_components=%r)" % n_components)
        if not isinstance(self.max_iter, INTEGER_TYPES) or self.max_iter < 0:
            raise ValueError("Maximum number of iterations must be a positive "
                             "integer; got (max_iter=%r)" % self.max_iter)
        if not isinstance(self.tol, numbers.Number) or self.tol < 0:
            raise ValueError("Tolerance for stopping criteria must be "
                             "positive; got (tol=%r)" % self.tol)

        # check W and H, or initialize them
        if self.init == 'custom' and update_H:
            _check_init(H, (n_components, n_features), "NMF (input H)")
            _check_init(W, (n_samples, n_components), "NMF (input W)")
        elif not update_H:
            _check_init(H, (n_components, n_features), "NMF (input H)")
            W = np.zeros((n_samples, n_components))
        else:
            W, H = _initialize_nmf(X, n_components, init=self.init,
                                   random_state=self.random_state)

        if update_H:  # fit_transform
            W, H, n_iter = _fit_projected_gradient(
                X, W, H, self.tol, self.max_iter, self.nls_max_iter,
                self.alpha, self.l1_ratio)
        else:  # transform
            Wt, _, n_iter = _nls_subproblem(X.T, H.T, W.T, self.tol,
                                            self.nls_max_iter,
                                            alpha=self.alpha,
                                            l1_ratio=self.l1_ratio)
            W = Wt.T

        if n_iter == self.max_iter and self.tol > 0:
            warnings.warn("Maximum number of iteration %d reached. Increase it"
                          " to improve convergence." % self.max_iter,
                          ConvergenceWarning)

        return W, H, n_iter

#################
# End of _PGNMF #
#################

def plot_results(results_df, plot_name):
    if results_df is None:
        return None

    plt.figure(figsize=(16, 6))
    colors = 'bgr'
    markers = 'ovs'
    ax = plt.subplot(1, 3, 1)
    for i, init in enumerate(np.unique(results_df['init'])):
        plt.subplot(1, 3, i + 1, sharex=ax, sharey=ax)
        for j, method in enumerate(np.unique(results_df['method'])):
            mask = np.logical_and(results_df['init'] == init,
                                  results_df['method'] == method)
            selected_items = results_df[mask]

            plt.plot(selected_items['time'], selected_items['loss'],
                     color=colors[j % len(colors)], ls='-',
                     marker=markers[j % len(markers)],
                     label=method)

        plt.legend(loc=0, fontsize='x-small')
        plt.xlabel("Time (s)")
        plt.ylabel("loss")
        plt.title("%s" % init)
    plt.suptitle(plot_name, fontsize=16)

@ignore_warnings(category=ConvergenceWarning)
# use joblib to cache the results.
# X_shape is specified in arguments for avoiding hashing X
@mem.cache(ignore=['X', 'W0', 'H0'])
def bench_one(name, X, W0, H0, X_shape, clf_type, clf_params, init,
              n_components, random_state):
    W = W0.copy()
    H = H0.copy()

    clf = clf_type(**clf_params)
    st = time()
    W = clf.fit_transform(X, W=W, H=H)
    end = time()
    H = clf.components_

    this_loss = _beta_divergence(X, W, H, 2.0, True)
    duration = end - st
    return this_loss, duration

def run_bench(X, clfs, plot_name, n_components, tol, alpha, l1_ratio):
    start = time()
    results = []
    for name, clf_type, iter_range, clf_params in clfs:
        print("Training %s:" % name)
        for rs, init in enumerate(('nndsvd', 'nndsvdar', 'random')):
            print("    %s %s: " % (init, " " * (8 - len(init))), end="")
            W, H = _initialize_nmf(X, n_components, init, 1e-6, rs)

            for max_iter in iter_range:
                clf_params['alpha'] = alpha
                clf_params['l1_ratio'] = l1_ratio
                clf_params['max_iter'] = max_iter
                clf_params['tol'] = tol
                clf_params['random_state'] = rs
                clf_params['init'] = 'custom'
                clf_params['n_components'] = n_components

                this_loss, duration = bench_one(name, X, W, H, X.shape,
                                                clf_type, clf_params,
                                                init, n_components, rs)

                init_name = "init='%s'" % init
                results.append((name, this_loss, duration, init_name))
                # print("loss: %.6f, time: %.3f sec" % (this_loss, duration))
                print(".", end="")
                sys.stdout.flush()
            print(" ")

    # Use a panda dataframe to organize the results
    results_df = pandas.DataFrame(results,
                                  columns="method loss time init".split())
    print("Total time = %0.3f sec\n" % (time() - start))

    # plot the results
    plot_results(results_df, plot_name)
    return results_df

def load_20news():
    print("Loading 20 newsgroups dataset")
    print("-----------------------------")
    from sklearn.datasets import fetch_20newsgroups
    dataset = fetch_20newsgroups(shuffle=True, random_state=1,
                                 remove=('headers', 'footers', 'quotes'))
    vectorizer = TfidfVectorizer(max_df=0.95, min_df=2, stop_words='english')
    tfidf = vectorizer.fit_transform(dataset.data)
    return tfidf

def load_faces():
    print("Loading Olivetti face dataset")
    print("-----------------------------")
    from sklearn.datasets import fetch_olivetti_faces
    faces = fetch_olivetti_faces(shuffle=True)
    return faces.data

def build_clfs(cd_iters, pg_iters, mu_iters):
    clfs = [("Coordinate Descent", NMF, cd_iters, {'solver': 'cd'}),
            ("Projected Gradient", _PGNMF, pg_iters, {'solver': 'pg'}),
            ("Multiplicative Update", NMF, mu_iters, {'solver': 'mu'}),
            ]
    return clfs

if __name__ == '__main__':
    alpha = 0.
    l1_ratio = 0.5
    n_components = 10
    tol = 1e-15

    # first benchmark on 20 newsgroup dataset: sparse, shape(11314, 39116)
    plot_name = "20 Newsgroups sparse dataset"
    cd_iters = np.arange(1, 30)
    pg_iters = np.arange(1, 6)
    mu_iters = np.arange(1, 30)
    clfs = build_clfs(cd_iters, pg_iters, mu_iters)
    X_20news = load_20news()
    run_bench(X_20news, clfs, plot_name, n_components, tol, alpha, l1_ratio)

    # second benchmark on Olivetti faces dataset: dense, shape(400, 4096)
    plot_name = "Olivetti Faces dense dataset"
    cd_iters = np.arange(1, 30)
    pg_iters = np.arange(1, 12)
    mu_iters = np.arange(1, 30)
    clfs = build_clfs(cd_iters, pg_iters, mu_iters)
    X_faces = load_faces()
    run_bench(X_faces, clfs, plot_name, n_components, tol, alpha, l1_ratio,)

    plt.show()
<|endoftext|>


<commit_before>'''
Netmiko Cisco WLC support
'''
from __future__ import print_function
from __future__ import unicode_literals
import time

from netmiko.ssh_connection import BaseSSHConnection
from netmiko.netmiko_globals import MAX_BUFFER

class CiscoWlcSSH(BaseSSHConnection):
    '''
    Netmiko Cisco WLC support
    '''
    def special_login_handler(self, delay_factor=.3):
        '''
        WLC presents with the following on login (in certain OS versions)

        login as: user

        (Cisco Controller)

        User: user

        Password:****
        '''
        i = 0
        while i <= 12:
            if self.remote_conn.recv_ready():
                output = self.remote_conn.recv(MAX_BUFFER).decode('utf-8')
                if 'login as' in output or 'User' in output:
                    self.remote_conn.sendall(self.username + '\n')
                elif 'Password' in output:
                    self.remote_conn.sendall(self.password + '\n')
                    break
                time.sleep(delay_factor)
            else:
                self.remote_conn.sendall('\n')
                time.sleep(delay_factor*3)
            i += 1

    def send_command_w_enter(self, *args, **kwargs):
        '''
        For 'show run-config' Cisco WLC adds a 'Press Enter to continue...' message
        Even though pagination is disabled

        show run-config also has excessive delays in the output which requires special
        handling.

        Arguments are the same as send_command() method
        '''

        output = self.send_command(*args, **kwargs)
        if 'Press Enter to' in output:
            new_args = list(args)
            if len(args) >= 1:
                new_args[0] = '\n'
            else:
                kwargs['command_string'] = '\n'

            if not kwargs.get('delay_factor') and not len(args) >= 2:
                kwargs['delay_factor'] = 3
            if not kwargs.get('max_loops') and not len(args) >= 3:
                kwargs['max_loops'] = 150

            # Send an 'enter'
            output = self.send_command(*new_args, **kwargs)

            # WLC has excessive delay after this appears on screen
            if '802.11b Advanced Configuration' in output:

                # Defaults to 30 seconds
                time.sleep(kwargs['delay_factor'] * 10)
                not_done = True
                i = 1
                while (not_done) and (i <= 150):
                    time.sleep(kwargs['delay_factor'])
                    i += 1
                    if self.remote_conn.recv_ready():
                        # Unicode error occurred in output (errors=ignore strips this out).
                        output += self.remote_conn.recv(MAX_BUFFER).decode('utf-8',
                                                                           errors='ignore')
                    else:
                        not_done = False

        strip_prompt = kwargs.get('strip_prompt', True)
        if strip_prompt:
            output = self.strip_prompt(output)
        return output

    def session_preparation(self):
        '''
        Prepare the session after the connection has been established

        Cisco WLC uses "config paging disable" to disable paging
        '''

        self.disable_paging(command="config paging disable\n")
        self.set_base_prompt()

    def cleanup(self):
        '''
        Reset WLC back to normal paging
        '''

        self.send_command("config paging enable\n")

<commit_msg>Improve strip prompt<commit_after>'''
Netmiko Cisco WLC support
'''
from __future__ import print_function
from __future__ import unicode_literals
import time

from netmiko.ssh_connection import BaseSSHConnection
from netmiko.netmiko_globals import MAX_BUFFER

class CiscoWlcSSH(BaseSSHConnection):
    '''
    Netmiko Cisco WLC support
    '''
    def special_login_handler(self, delay_factor=.3):
        '''
        WLC presents with the following on login (in certain OS versions)

        login as: user

        (Cisco Controller)

        User: user

        Password:****
        '''
        i = 0
        while i <= 12:
            if self.remote_conn.recv_ready():
                output = self.remote_conn.recv(MAX_BUFFER).decode('utf-8')
                if 'login as' in output or 'User' in output:
                    self.remote_conn.sendall(self.username + '\n')
                elif 'Password' in output:
                    self.remote_conn.sendall(self.password + '\n')
                    break
                time.sleep(delay_factor)
            else:
                self.remote_conn.sendall('\n')
                time.sleep(delay_factor*3)
            i += 1

    def send_command_w_enter(self, *args, **kwargs):
        '''
        For 'show run-config' Cisco WLC adds a 'Press Enter to continue...' message
        Even though pagination is disabled

        show run-config also has excessive delays in the output which requires special
        handling.

        Arguments are the same as send_command() method
        '''

        output = self.send_command(*args, **kwargs)
        if 'Press Enter to' in output:
            new_args = list(args)
            if len(args) >= 1:
                new_args[0] = '\n'
            else:
                kwargs['command_string'] = '\n'

            if not kwargs.get('delay_factor') and not len(args) >= 2:
                kwargs['delay_factor'] = 3
            if not kwargs.get('max_loops') and not len(args) >= 3:
                kwargs['max_loops'] = 150

            # Send an 'enter'
            output = self.send_command(*new_args, **kwargs)

            # WLC has excessive delay after this appears on screen
            if '802.11b Advanced Configuration' in output:

                # Defaults to 30 seconds
                time.sleep(kwargs['delay_factor'] * 10)
                not_done = True
                i = 1
                while (not_done) and (i <= 150):
                    time.sleep(kwargs['delay_factor'])
                    i += 1
                    if self.remote_conn.recv_ready():
                        # Unicode error occurred in output (errors=ignore strips this out).
                        output += self.remote_conn.recv(MAX_BUFFER).decode('utf-8',
                                                                           errors='ignore')
                    else:
                        not_done = False

        strip_prompt = kwargs.get('strip_prompt', True)
        if strip_prompt:
            # Had to strip trailing prompt twice.
            output = self.strip_prompt(output)
            output = self.strip_prompt(output)
        return output

    def session_preparation(self):
        '''
        Prepare the session after the connection has been established

        Cisco WLC uses "config paging disable" to disable paging
        '''

        self.disable_paging(command="config paging disable\n")
        self.set_base_prompt()

    def cleanup(self):
        '''
        Reset WLC back to normal paging
        '''

        self.send_command("config paging enable\n")

<|endoftext|>


<commit_before>from __future__ import absolute_import

from .docker_image import DockerImage
from .docker_image import DockerContainer
from .dockerfile import Dockerfile
from .fsl import FSL
from .miniconda import Miniconda
from .utils import SpecsParser
<commit_msg>import DockerContainer<commit_after>from __future__ import absolute_import

from neurodocker.docker.docker_image import DockerContainer, DockerImage
from neurodocker.docker.dockerfile import Dockerfile
from neurodocker.docker.fsl import FSL
from neurodocker.docker.miniconda import Miniconda
from neurodocker.docker.utils import SpecsParser
<|endoftext|>


<commit_before>from collections import OrderedDict
import sys

import numpy as np
import click

from devito import clear_cache, configuration, mode_develop, mode_benchmark, warning
from devito.tools import as_tuple, sweep
from examples.seismic.acoustic.acoustic_example import run as acoustic_run, acoustic_setup
from examples.seismic.tti.tti_example import run as tti_run, tti_setup

@click.group()
def benchmark():
    """
    Benchmarking script for seismic forward operators.

    \b
    There are three main 'execution modes':
    run: a single run with given DSE/DLE levels
    bench: complete benchmark with multiple DSE/DLE levels
    test: tests numerical correctness with different parameters

    Further, this script can generate a roofline plot from a benchmark
    """
    pass

def option_simulation(f):
    def default_list(ctx, param, value):
        return list(value if len(value) > 0 else (2, ))

    options = [
        click.option('-P', '--problem', type=click.Choice(['acoustic', 'tti']),
                     help='Problem name'),
        click.option('-d', '--shape', default=(50, 50, 50),
                     help='Number of grid points along each axis'),
        click.option('-s', '--spacing', default=(20., 20., 20.),
                     help='Spacing between grid sizes in meters'),
        click.option('-n', '--nbpml', default=10,
                     help='Number of PML layers'),
        click.option('-so', '--space-order', type=int, multiple=True,
                     callback=default_list, help='Space order of the simulation'),
        click.option('-to', '--time-order', type=int, multiple=True,
                     callback=default_list, help='Time order of the simulation'),
        click.option('-t', '--tn', default=250,
                     help='End time of the simulation in ms'),
    ]
    for option in reversed(options):
        f = option(f)
    return f

def option_performance(f):
    """Defines options for all aspects of performance tuning"""

    _preset = {
        # Fixed
        'O1': {'dse': 'basic', 'dle': 'basic'},
        'O2': {'dse': 'advanced', 'dle': 'advanced'},
        'O3': {'dse': 'aggressive', 'dle': 'advanced'},
        # Parametric
        'dse': {'dse': ['basic', 'advanced', 'aggressive'], 'dle': 'advanced'},
        'dle': {'dse': 'advanced', 'dle': ['basic', 'advanced']}
    }

    def from_preset(ctx, param, value):
        """Set all performance options according to bench-mode preset"""
        ctx.params.update(_preset[value])
        return value

    def from_value(ctx, param, value):
        """Prefer preset values and warn for competing values."""
        return ctx.params[param.name] or value

    options = [
        click.option('-bm', '--bench-mode', is_eager=True,
                     callback=from_preset, expose_value=False, default='O2',
                     type=click.Choice(['O1', 'O2', 'O3', 'dse', 'dle']),
                     help='Choose what to benchmark; ignored if execmode=run'),
        click.option('--arch', default='unknown',
                     help='Architecture on which the simulation is/was run'),
        click.option('--dse', callback=from_value,
                     type=click.Choice(['noop'] + configuration._accepted['dse']),
                     help='Devito symbolic engine (DSE) mode'),
        click.option('--dle', callback=from_value,
                     type=click.Choice(['noop'] + configuration._accepted['dle']),
                     help='Devito loop engine (DLE) mode'),
        click.option('-a', '--autotune', is_flag=True,
                     help='Switch auto tuning on/off')
    ]
    for option in reversed(options):
        f = option(f)
    return f

@benchmark.command(name='run')
@option_simulation
@option_performance
@click.option('-bs', '--block-shape', default=(0, 0, 0),
              help='Loop-blocking shape, bypass autotuning')
def cli_run(problem, **kwargs):
    """
    A single run with a specific set of performance parameters.
    """
    mode_benchmark()
    run(problem, **kwargs)

def run(problem, **kwargs):
    """
    A single run with a specific set of performance parameters.
    """
    setup = tti_setup if problem == 'tti' else acoustic_setup
    options = {}

    time_order = kwargs.pop('time_order')[0]
    space_order = kwargs.pop('space_order')[0]
    autotune = kwargs.pop('autotune')

    # Should a specific block-shape be used? Useful if one wants to skip
    # the autotuning pass as a good block-shape is already known
    block_shape = as_tuple(kwargs.pop('block_shape'))
    if all(block_shape):
        if autotune:
            warning("Skipping autotuning (using explicit block-shape `%s`)"
                    % str(block_shape))
            autotune = False
        # This is quite hacky, but it does the trick
        for d, bs in zip(['x', 'y', 'z'], block_shape):
            options['%s0_blk_size' % d] = bs

    solver = setup(space_order=space_order, time_order=time_order, **kwargs)
    solver.forward(autotune=autotune, **options)

@benchmark.command(name='test')
@option_simulation
@option_performance
def cli_test(problem, **kwargs):
    """
    Test numerical correctness with different parameters.
    """
    mode_develop()
    test(problem, **kwargs)

def test(problem, **kwargs):
    """
    Test numerical correctness with different parameters.
    """
    run = tti_run if problem == 'tti' else acoustic_run
    sweep_options = ('space_order', 'time_order', 'dse', 'dle', 'autotune')

    last_res = None
    for params in sweep(kwargs, keys=sweep_options):
        kwargs.update(params)
        _, _, _, res = run(**kwargs)

        if last_res is None:
            last_res = res
        else:
            for i in range(len(res)):
                assert np.isclose(res[i], last_res[i])

@benchmark.command(name='bench')
@option_simulation
@option_performance
@click.option('-r', '--resultsdir', default='results',
              help='Directory containing results')
@click.option('-x', '--repeats', default=3,
              help='Number of test case repetitions')
def cli_bench(problem, **kwargs):
    """
    Complete benchmark with multiple simulation and performance parameters.
    """
    mode_benchmark()
    kwargs['autotune'] = configuration['autotuning'].level
    bench(problem, **kwargs)

def bench(problem, **kwargs):
    """
    Complete benchmark with multiple simulation and performance parameters.
    """
    run = tti_run if problem == 'tti' else acoustic_run
    resultsdir = kwargs.pop('resultsdir')
    repeats = kwargs.pop('repeats')

    bench = get_ob_bench(problem, resultsdir, kwargs)
    bench.execute(get_ob_exec(run), warmups=0, repeats=repeats)
    bench.save()

    # Final clean up, just in case the benchmarker is used from external Python modules
    clear_cache()

@benchmark.command(name='plot')
@option_simulation
@option_performance
@click.option('--backend', default='core',
              type=click.Choice(configuration._accepted['backend']),
              help='Used execution backend (e.g., core, yask)')
@click.option('-r', '--resultsdir', default='results',
              help='Directory containing results')
@click.option('--max-bw', type=float,
              help='Max GB/s of the DRAM')
@click.option('--flop-ceil', type=(float, str), multiple=True,
              help='Max GFLOPS/s of the CPU. A 2-tuple (float, str)'
                   'is expected, where the float is the performance'
                   'ceil (GFLOPS/s) and the str indicates how the'
                   'ceil was obtained (ideal peak, linpack, ...)')
@click.option('--point-runtime', is_flag=True, default=True,
              help='Annotate points with runtime values')
def cli_plot(problem, **kwargs):
    """
    Plotting mode to generate plots for performance analysis.
    """
    plot(problem, **kwargs)

def plot(problem, **kwargs):
    """
    Plotting mode to generate plots for performance analysis.
    """
    backend = kwargs.pop('backend')
    resultsdir = kwargs.pop('resultsdir')
    max_bw = kwargs.pop('max_bw')
    flop_ceils = kwargs.pop('flop_ceil')
    point_runtime = kwargs.pop('point_runtime')

    arch = kwargs['arch']
    space_order = "[%s]" % ",".join(str(i) for i in kwargs['space_order'])
    time_order = kwargs['time_order']
    shape = "[%s]" % ",".join(str(i) for i in kwargs['shape'])

    RooflinePlotter = get_ob_plotter()
    bench = get_ob_bench(problem, resultsdir, kwargs)

    bench.load()
    if not bench.loaded:
        warning("Could not load any results, nothing to plot. Exiting...")
        sys.exit(0)

    gflopss = bench.lookup(params=kwargs, measure="gflopss", event='main')
    oi = bench.lookup(params=kwargs, measure="oi", event='main')
    time = bench.lookup(params=kwargs, measure="timings", event='main')

    # What plot am I?
    modes = [i for i in ['dse', 'dle', 'autotune']
             if len(set(dict(j)[i] for j in gflopss)) > 1]

    # Filename
    figname = "%s_dim%s_so%s_to%s_arch[%s]_bkend[%s].pdf" % (
        problem, shape, space_order, time_order, arch, backend
    )

    # Legend setup. Do not plot a legend if there's no variation in performance
    # options (dse, dle, autotune)
    if modes:
        legend = {'loc': 'upper left', 'fontsize': 7, 'ncol': 4}
    else:
        legend = 'drop'

    avail_colors = ['r', 'g', 'b', 'y', 'k', 'm']
    avail_markers = ['o', 'x', '^', 'v', '<', '>']

    used_colors = {}
    used_markers = {}

    # Find min and max runtimes for instances having the same OI
    min_max = {v: [0, sys.maxsize] for v in oi.values()}
    for k, v in time.items():
        i = oi[k]
        min_max[i][0] = v if min_max[i][0] == 0 else min(v, min_max[i][0])
        min_max[i][1] = v if min_max[i][1] == sys.maxsize else max(v, min_max[i][1])

    with RooflinePlotter(figname=figname, plotdir=resultsdir,
                         max_bw=max_bw, flop_ceils=flop_ceils,
                         fancycolor=True, legend=legend) as plot:
        for k, v in gflopss.items():
            so = dict(k)['space_order']

            oi_value = oi[k]
            time_value = time[k]

            run = tuple(dict(k)[i] for i in modes)
            label = ("<%s>" % ','.join(run)) if run else None

            color = used_colors[run] if run in used_colors else avail_colors.pop(0)
            used_colors.setdefault(run, color)
            marker = used_markers[so] if so in used_markers else avail_markers.pop(0)
            used_markers.setdefault(so, marker)

            oi_loc = 0.076 if len(str(so)) == 1 else 0.09
            oi_annotate = {'s': 'SO=%s' % so, 'size': 6, 'xy': (oi_value, oi_loc)}
            if time_value in min_max[oi_value] and point_runtime:
                # Only annotate min and max runtimes on each OI line, to avoid
                # polluting the plot too much
                point_annotate = {'s': "%.0fs" % time_value, 'xytext': (0.0, 5.5),
                                  'size': 6, 'rotation': 0}
            else:
                point_annotate = None
            oi_line = time_value == min_max[oi_value][0]
            if oi_line:
                perf_annotate = {'size': 6, 'xytext': (-4, 5)}

            plot.add_point(gflops=v, oi=oi_value, marker=marker, color=color,
                           oi_line=oi_line, label=label, perf_annotate=perf_annotate,
                           oi_annotate=oi_annotate, point_annotate=point_annotate)

def get_ob_bench(problem, resultsdir, parameters):
    """Return a special :class:`opescibench.Benchmark` to manage performance runs."""
    try:
        from opescibench import Benchmark
    except:
        raise ImportError('Could not import opescibench utility package.\n'
                          'Please install https://github.com/opesci/opescibench')

    class DevitoBenchmark(Benchmark):

        def param_string(self, params):
            devito_params, params = OrderedDict(), dict(params)
            devito_params['arch'] = params['arch']
            devito_params['shape'] = ",".join(str(i) for i in params['shape'])
            devito_params['nbpml'] = params['nbpml']
            devito_params['tn'] = params['tn']
            devito_params['so'] = params['space_order']
            devito_params['to'] = params['time_order']
            devito_params['dse'] = params['dse']
            devito_params['dle'] = params['dle']
            devito_params['at'] = params['autotune']
            return '_'.join(['%s[%s]' % (k, v) for k, v in devito_params.items()])

    return DevitoBenchmark(name=problem, resultsdir=resultsdir, parameters=parameters)

def get_ob_exec(func):
    """Return a special :class:`opescibench.Executor` to execute performance runs."""
    try:
        from opescibench import Executor
    except:
        raise ImportError('Could not import opescibench utility package.\n'
                          'Please install https://github.com/opesci/opescibench')

    class DevitoExecutor(Executor):

        def __init__(self, func):
            super(DevitoExecutor, self).__init__()
            self.func = func

        def run(self, *args, **kwargs):
            clear_cache()

            gflopss, oi, timings, _ = self.func(*args, **kwargs)

            for key in timings.keys():
                self.register(gflopss[key], measure="gflopss", event=key)
                self.register(oi[key], measure="oi", event=key)
                self.register(timings[key], measure="timings", event=key)

    return DevitoExecutor(func)

def get_ob_plotter():
    try:
        from opescibench import RooflinePlotter
    except:
        raise ImportError('Could not import opescibench utility package.\n'
                          'Please install https://github.com/opesci/opescibench'
                          'To plot performance results, make sure to have the'
                          'Matplotlib package installed')
    return RooflinePlotter

if __name__ == "__main__":
    benchmark()
<commit_msg>bench: Drop obsolete dle=basic<commit_after>from collections import OrderedDict
import sys

import numpy as np
import click

from devito import clear_cache, configuration, mode_develop, mode_benchmark, warning
from devito.tools import as_tuple, sweep
from examples.seismic.acoustic.acoustic_example import run as acoustic_run, acoustic_setup
from examples.seismic.tti.tti_example import run as tti_run, tti_setup

@click.group()
def benchmark():
    """
    Benchmarking script for seismic forward operators.

    \b
    There are three main 'execution modes':
    run: a single run with given DSE/DLE levels
    bench: complete benchmark with multiple DSE/DLE levels
    test: tests numerical correctness with different parameters

    Further, this script can generate a roofline plot from a benchmark
    """
    pass

def option_simulation(f):
    def default_list(ctx, param, value):
        return list(value if len(value) > 0 else (2, ))

    options = [
        click.option('-P', '--problem', type=click.Choice(['acoustic', 'tti']),
                     help='Problem name'),
        click.option('-d', '--shape', default=(50, 50, 50),
                     help='Number of grid points along each axis'),
        click.option('-s', '--spacing', default=(20., 20., 20.),
                     help='Spacing between grid sizes in meters'),
        click.option('-n', '--nbpml', default=10,
                     help='Number of PML layers'),
        click.option('-so', '--space-order', type=int, multiple=True,
                     callback=default_list, help='Space order of the simulation'),
        click.option('-to', '--time-order', type=int, multiple=True,
                     callback=default_list, help='Time order of the simulation'),
        click.option('-t', '--tn', default=250,
                     help='End time of the simulation in ms'),
    ]
    for option in reversed(options):
        f = option(f)
    return f

def option_performance(f):
    """Defines options for all aspects of performance tuning"""

    _preset = {
        # Fixed
        'O1': {'dse': 'basic', 'dle': 'advanced'},
        'O2': {'dse': 'advanced', 'dle': 'advanced'},
        'O3': {'dse': 'aggressive', 'dle': 'advanced'},
        # Parametric
        'dse': {'dse': ['basic', 'advanced', 'aggressive'], 'dle': 'advanced'},
    }

    def from_preset(ctx, param, value):
        """Set all performance options according to bench-mode preset"""
        ctx.params.update(_preset[value])
        return value

    def from_value(ctx, param, value):
        """Prefer preset values and warn for competing values."""
        return ctx.params[param.name] or value

    options = [
        click.option('-bm', '--bench-mode', is_eager=True,
                     callback=from_preset, expose_value=False, default='O2',
                     type=click.Choice(['O1', 'O2', 'O3', 'dse']),
                     help='Choose what to benchmark; ignored if execmode=run'),
        click.option('--arch', default='unknown',
                     help='Architecture on which the simulation is/was run'),
        click.option('--dse', callback=from_value,
                     type=click.Choice(['noop'] + configuration._accepted['dse']),
                     help='Devito symbolic engine (DSE) mode'),
        click.option('--dle', callback=from_value,
                     type=click.Choice(['noop'] + configuration._accepted['dle']),
                     help='Devito loop engine (DLE) mode'),
        click.option('-a', '--autotune', is_flag=True,
                     help='Switch auto tuning on/off')
    ]
    for option in reversed(options):
        f = option(f)
    return f

@benchmark.command(name='run')
@option_simulation
@option_performance
@click.option('-bs', '--block-shape', default=(0, 0, 0),
              help='Loop-blocking shape, bypass autotuning')
def cli_run(problem, **kwargs):
    """
    A single run with a specific set of performance parameters.
    """
    mode_benchmark()
    run(problem, **kwargs)

def run(problem, **kwargs):
    """
    A single run with a specific set of performance parameters.
    """
    setup = tti_setup if problem == 'tti' else acoustic_setup
    options = {}

    time_order = kwargs.pop('time_order')[0]
    space_order = kwargs.pop('space_order')[0]
    autotune = kwargs.pop('autotune')

    # Should a specific block-shape be used? Useful if one wants to skip
    # the autotuning pass as a good block-shape is already known
    block_shape = as_tuple(kwargs.pop('block_shape'))
    if all(block_shape):
        if autotune:
            warning("Skipping autotuning (using explicit block-shape `%s`)"
                    % str(block_shape))
            autotune = False
        # This is quite hacky, but it does the trick
        for d, bs in zip(['x', 'y', 'z'], block_shape):
            options['%s0_blk_size' % d] = bs

    solver = setup(space_order=space_order, time_order=time_order, **kwargs)
    solver.forward(autotune=autotune, **options)

@benchmark.command(name='test')
@option_simulation
@option_performance
def cli_test(problem, **kwargs):
    """
    Test numerical correctness with different parameters.
    """
    mode_develop()
    test(problem, **kwargs)

def test(problem, **kwargs):
    """
    Test numerical correctness with different parameters.
    """
    run = tti_run if problem == 'tti' else acoustic_run
    sweep_options = ('space_order', 'time_order', 'dse', 'dle', 'autotune')

    last_res = None
    for params in sweep(kwargs, keys=sweep_options):
        kwargs.update(params)
        _, _, _, res = run(**kwargs)

        if last_res is None:
            last_res = res
        else:
            for i in range(len(res)):
                assert np.isclose(res[i], last_res[i])

@benchmark.command(name='bench')
@option_simulation
@option_performance
@click.option('-r', '--resultsdir', default='results',
              help='Directory containing results')
@click.option('-x', '--repeats', default=3,
              help='Number of test case repetitions')
def cli_bench(problem, **kwargs):
    """
    Complete benchmark with multiple simulation and performance parameters.
    """
    mode_benchmark()
    kwargs['autotune'] = configuration['autotuning'].level
    bench(problem, **kwargs)

def bench(problem, **kwargs):
    """
    Complete benchmark with multiple simulation and performance parameters.
    """
    run = tti_run if problem == 'tti' else acoustic_run
    resultsdir = kwargs.pop('resultsdir')
    repeats = kwargs.pop('repeats')

    bench = get_ob_bench(problem, resultsdir, kwargs)
    bench.execute(get_ob_exec(run), warmups=0, repeats=repeats)
    bench.save()

    # Final clean up, just in case the benchmarker is used from external Python modules
    clear_cache()

@benchmark.command(name='plot')
@option_simulation
@option_performance
@click.option('--backend', default='core',
              type=click.Choice(configuration._accepted['backend']),
              help='Used execution backend (e.g., core, yask)')
@click.option('-r', '--resultsdir', default='results',
              help='Directory containing results')
@click.option('--max-bw', type=float,
              help='Max GB/s of the DRAM')
@click.option('--flop-ceil', type=(float, str), multiple=True,
              help='Max GFLOPS/s of the CPU. A 2-tuple (float, str)'
                   'is expected, where the float is the performance'
                   'ceil (GFLOPS/s) and the str indicates how the'
                   'ceil was obtained (ideal peak, linpack, ...)')
@click.option('--point-runtime', is_flag=True, default=True,
              help='Annotate points with runtime values')
def cli_plot(problem, **kwargs):
    """
    Plotting mode to generate plots for performance analysis.
    """
    plot(problem, **kwargs)

def plot(problem, **kwargs):
    """
    Plotting mode to generate plots for performance analysis.
    """
    backend = kwargs.pop('backend')
    resultsdir = kwargs.pop('resultsdir')
    max_bw = kwargs.pop('max_bw')
    flop_ceils = kwargs.pop('flop_ceil')
    point_runtime = kwargs.pop('point_runtime')

    arch = kwargs['arch']
    space_order = "[%s]" % ",".join(str(i) for i in kwargs['space_order'])
    time_order = kwargs['time_order']
    shape = "[%s]" % ",".join(str(i) for i in kwargs['shape'])

    RooflinePlotter = get_ob_plotter()
    bench = get_ob_bench(problem, resultsdir, kwargs)

    bench.load()
    if not bench.loaded:
        warning("Could not load any results, nothing to plot. Exiting...")
        sys.exit(0)

    gflopss = bench.lookup(params=kwargs, measure="gflopss", event='main')
    oi = bench.lookup(params=kwargs, measure="oi", event='main')
    time = bench.lookup(params=kwargs, measure="timings", event='main')

    # What plot am I?
    modes = [i for i in ['dse', 'dle', 'autotune']
             if len(set(dict(j)[i] for j in gflopss)) > 1]

    # Filename
    figname = "%s_dim%s_so%s_to%s_arch[%s]_bkend[%s].pdf" % (
        problem, shape, space_order, time_order, arch, backend
    )

    # Legend setup. Do not plot a legend if there's no variation in performance
    # options (dse, dle, autotune)
    if modes:
        legend = {'loc': 'upper left', 'fontsize': 7, 'ncol': 4}
    else:
        legend = 'drop'

    avail_colors = ['r', 'g', 'b', 'y', 'k', 'm']
    avail_markers = ['o', 'x', '^', 'v', '<', '>']

    used_colors = {}
    used_markers = {}

    # Find min and max runtimes for instances having the same OI
    min_max = {v: [0, sys.maxsize] for v in oi.values()}
    for k, v in time.items():
        i = oi[k]
        min_max[i][0] = v if min_max[i][0] == 0 else min(v, min_max[i][0])
        min_max[i][1] = v if min_max[i][1] == sys.maxsize else max(v, min_max[i][1])

    with RooflinePlotter(figname=figname, plotdir=resultsdir,
                         max_bw=max_bw, flop_ceils=flop_ceils,
                         fancycolor=True, legend=legend) as plot:
        for k, v in gflopss.items():
            so = dict(k)['space_order']

            oi_value = oi[k]
            time_value = time[k]

            run = tuple(dict(k)[i] for i in modes)
            label = ("<%s>" % ','.join(run)) if run else None

            color = used_colors[run] if run in used_colors else avail_colors.pop(0)
            used_colors.setdefault(run, color)
            marker = used_markers[so] if so in used_markers else avail_markers.pop(0)
            used_markers.setdefault(so, marker)

            oi_loc = 0.076 if len(str(so)) == 1 else 0.09
            oi_annotate = {'s': 'SO=%s' % so, 'size': 6, 'xy': (oi_value, oi_loc)}
            if time_value in min_max[oi_value] and point_runtime:
                # Only annotate min and max runtimes on each OI line, to avoid
                # polluting the plot too much
                point_annotate = {'s': "%.0fs" % time_value, 'xytext': (0.0, 5.5),
                                  'size': 6, 'rotation': 0}
            else:
                point_annotate = None
            oi_line = time_value == min_max[oi_value][0]
            if oi_line:
                perf_annotate = {'size': 6, 'xytext': (-4, 5)}

            plot.add_point(gflops=v, oi=oi_value, marker=marker, color=color,
                           oi_line=oi_line, label=label, perf_annotate=perf_annotate,
                           oi_annotate=oi_annotate, point_annotate=point_annotate)

def get_ob_bench(problem, resultsdir, parameters):
    """Return a special :class:`opescibench.Benchmark` to manage performance runs."""
    try:
        from opescibench import Benchmark
    except:
        raise ImportError('Could not import opescibench utility package.\n'
                          'Please install https://github.com/opesci/opescibench')

    class DevitoBenchmark(Benchmark):

        def param_string(self, params):
            devito_params, params = OrderedDict(), dict(params)
            devito_params['arch'] = params['arch']
            devito_params['shape'] = ",".join(str(i) for i in params['shape'])
            devito_params['nbpml'] = params['nbpml']
            devito_params['tn'] = params['tn']
            devito_params['so'] = params['space_order']
            devito_params['to'] = params['time_order']
            devito_params['dse'] = params['dse']
            devito_params['dle'] = params['dle']
            devito_params['at'] = params['autotune']
            return '_'.join(['%s[%s]' % (k, v) for k, v in devito_params.items()])

    return DevitoBenchmark(name=problem, resultsdir=resultsdir, parameters=parameters)

def get_ob_exec(func):
    """Return a special :class:`opescibench.Executor` to execute performance runs."""
    try:
        from opescibench import Executor
    except:
        raise ImportError('Could not import opescibench utility package.\n'
                          'Please install https://github.com/opesci/opescibench')

    class DevitoExecutor(Executor):

        def __init__(self, func):
            super(DevitoExecutor, self).__init__()
            self.func = func

        def run(self, *args, **kwargs):
            clear_cache()

            gflopss, oi, timings, _ = self.func(*args, **kwargs)

            for key in timings.keys():
                self.register(gflopss[key], measure="gflopss", event=key)
                self.register(oi[key], measure="oi", event=key)
                self.register(timings[key], measure="timings", event=key)

    return DevitoExecutor(func)

def get_ob_plotter():
    try:
        from opescibench import RooflinePlotter
    except:
        raise ImportError('Could not import opescibench utility package.\n'
                          'Please install https://github.com/opesci/opescibench'
                          'To plot performance results, make sure to have the'
                          'Matplotlib package installed')
    return RooflinePlotter

if __name__ == "__main__":
    benchmark()
<|endoftext|>