<commit_before>#! /usr/bin/env python2
#
# This file is part of khmer, http://github.com/ged-lab/khmer/, and is
# Copyright (C) Michigan State University, 2009-2015. It is licensed under
# the three-clause BSD license; see LICENSE.
# Contact: khmer-project@idyll.org
#
# pylint: disable=invalid-name,missing-docstring
"""
Eliminate surplus reads.

Eliminate reads with median k-mer abundance higher than
DESIRED_COVERAGE.  Output sequences will be placed in 'infile.keep', with the
option to output to STDOUT.

% python scripts/normalize-by-median.py [ -C <cutoff> ] <data1> <data2> ...

Use '-h' for parameter help.
"""

import sys
import screed
import os
import khmer
import textwrap
from itertools import izip
from khmer.khmer_args import (build_counting_args, add_loadhash_args,
                              report_on_config, info)
import argparse
from khmer.kfile import (check_space, check_space_for_hashtable,
                         check_valid_file_exists)
from khmer.utils import write_record, check_is_pair
DEFAULT_DESIRED_COVERAGE = 10

# Iterate a collection in arbitrary batches
# from: http://stackoverflow.com/questions/4628290/pairs-from-single-list

def batchwise(coll, size):
    iter_coll = iter(coll)
    return izip(*[iter_coll] * size)

# Returns true if the pair of records are properly pairs

# pylint: disable=too-many-locals,too-many-branches
def normalize_by_median(input_filename, outfp, htable, paired, cutoff,
                        report_fp=None):

    desired_coverage = cutoff
    ksize = htable.ksize()

    # In paired mode we read two records at a time
    batch_size = 1
    if paired:
        batch_size = 2

    index = -1
    total = 0
    discarded = 0
    for index, batch in enumerate(batchwise(screed.open(
            input_filename, parse_description=False), batch_size)):
        if index > 0 and index % 100000 == 0:
            print >>sys.stderr, '... kept {kept} of {total} or'\
                ' {perc:2}%'.format(kept=total - discarded, total=total,
                                    perc=int(100. - discarded /
                                             float(total) * 100.))
            print >>sys.stderr, '... in file', input_filename

            if report_fp:
                print >> report_fp, total, total - discarded, \
                    1. - (discarded / float(total))
                report_fp.flush()

        total += batch_size

        # If in paired mode, check that the reads are properly interleaved

        if paired:
            if not check_is_pair(batch[0], batch[1]):
                raise IOError('Error: Improperly interleaved pairs \
                    {b0} {b1}'.format(b0=batch[0].name, b1=batch[1].name))

        # Emit the batch of reads if any read passes the filter
        # and all reads are longer than K
        passed_filter = False
        passed_length = True
        for record in batch:
            if len(record.sequence) < ksize:
                passed_length = False
                continue

            seq = record.sequence.replace('N', 'A')
            med, _, _ = htable.get_median_count(seq)

            if med < desired_coverage:
                htable.consume(seq)
                passed_filter = True

        # Emit records if any passed
        if passed_length and passed_filter:
            for record in batch:
                write_record(record, outfp)
        else:
            discarded += batch_size

    if report_fp:
        print >> report_fp, total, total - discarded, \
            1. - (discarded / float(total))
        report_fp.flush()

    return total, discarded

def handle_error(error, output_name, input_name, fail_save, htable):
    print >> sys.stderr, '** ERROR:', error
    print >> sys.stderr, '** Failed on {name}: '.format(name=input_name)
    if fail_save:
        tablename = os.path.basename(input_name) + '.ct.failed'
        print >> sys.stderr, \
            '** ...dumping k-mer counting table to {tn}'.format(tn=tablename)
        htable.save(tablename)
    try:
        os.remove(output_name)
    except:  # pylint: disable=bare-except
        print >> sys.stderr, '** ERROR: problem removing corrupt filtered file'

def normalize_by_median_and_check(input_filename, htable, single_output_file,
                                  fail_save, paired, cutoff, force,
                                  corrupt_files, report_fp=None):
    total = 0
    discarded = 0

    total_acc = None
    discarded_acc = None

    if single_output_file:
        if single_output_file is sys.stdout:
            output_name = '/dev/stdout'
        else:
            output_name = single_output_file.name
        outfp = single_output_file

    else:
        output_name = os.path.basename(input_filename) + '.keep'
        outfp = open(output_name, 'w')

    try:
        total_acc, discarded_acc = normalize_by_median(
            input_filename, outfp, htable, paired, cutoff, report_fp=None)
    except IOError as err:
        handle_error(err, output_name, input_filename, fail_save,
                     htable)
        if not force:
            print >> sys.stderr, '** Exiting!'

            sys.exit(1)
        else:
            print >> sys.stderr, '*** Skipping error file, moving on...'
            corrupt_files.append(input_filename)
    else:
        if total_acc == 0 and discarded_acc == 0:
            print >> sys.stderr, 'SKIPPED empty file', input_filename
        else:
            total += total_acc
            discarded += discarded_acc
            print >> sys.stderr, \
                'DONE with {inp}; kept {kept} of {total} or {perc:2}%'\
                .format(inp=input_filename, kept=total - discarded,
                        total=total, perc=int(100. - discarded /
                                              float(total) * 100.))
            print >> sys.stderr, 'output in', output_name

    return total_acc, discarded_acc, corrupt_files

def get_parser():
    epilog = ("""
    Discard sequences based on whether or not their median k-mer abundance lies
    above a specified cutoff. Kept sequences will be placed in <fileN>.keep.

    Paired end reads will be considered together if :option:`-p` is set. If
    either read will be kept, then both will be kept. This should result in
    keeping (or discarding) each sequencing fragment. This helps with retention
    of repeats, especially. With :option: `-u`/:option:`--unpaired-reads`,
    unpaired reads from the specified file will be read after the paired data
    is read.

    With :option:`-s`/:option:`--savetable`, the k-mer counting table
    will be saved to the specified file after all sequences have been
    processed. With :option:`-d`, the k-mer counting table will be
    saved every d files for multifile runs; if :option:`-s` is set,
    the specified name will be used, and if not, the name `backup.ct`
    will be used.  :option:`-l`/:option:`--loadtable` will load the
    specified k-mer counting table before processing the specified
    files.  Note that these tables are are in the same format as those
    produced by :program:`load-into-counting.py` and consumed by
    :program:`abundance-dist.py`.

    :option:`-f`/:option:`--fault-tolerant` will force the program to continue
    upon encountering a formatting error in a sequence file; the k-mer counting
    table up to that point will be dumped, and processing will continue on the
    next file.

    To append reads to an output file (rather than overwriting it), send output
    to STDOUT with `--out -` and use UNIX file redirection syntax (`>>`) to
    append to the file.

    Example::

        normalize-by-median.py -k 17 tests/test-data/test-abund-read-2.fa

    Example::

""" "        normalize-by-median.py -p -k 17 tests/test-data/test-abund-read-paired.fa"  # noqa
    """

    Example::

""" "        normalize-by-median.py -p -k 17 -o - tests/test-data/paired.fq >> appended-output.fq"  # noqa
    """

    Example::

""" "        normalize-by-median.py -k 17 -f tests/test-data/test-error-reads.fq tests/test-data/test-fastq-reads.fq"  # noqa
    """

    Example::

""" "        normalize-by-median.py -k 17 -d 2 -s test.ct tests/test-data/test-abund-read-2.fa tests/test-data/test-fastq-reads")   # noqa
    parser = build_counting_args(
        descr="Do digital normalization (remove mostly redundant sequences)",
        epilog=textwrap.dedent(epilog))
    parser.add_argument('-C', '--cutoff', type=int,
                        default=DEFAULT_DESIRED_COVERAGE)
    parser.add_argument('-p', '--paired', action='store_true')
    parser.add_argument('-u', '--unpaired-reads',
                        metavar="unpaired_reads_filename", help='with paired data only,\
                        include an unpaired file')
    parser.add_argument('-s', '--savetable', metavar="filename", default='',
                        help='save the k-mer counting table to disk after all'
                        'reads are loaded.')
    parser.add_argument('-R', '--report',
                        metavar='filename', type=argparse.FileType('w'))
    parser.add_argument('-f', '--fault-tolerant', dest='force',
                        help='continue on next file if read errors are \
                         encountered', action='store_true')
    parser.add_argument('--save-on-failure', dest='fail_save',
                        action='store_false', default=True,
                        help='Save k-mer counting table when an error occurs')
    parser.add_argument('-d', '--dump-frequency', dest='dump_frequency',
                        type=int, help='dump k-mer counting table every d '
                        'files', default=-1)
    parser.add_argument('-o', '--out', metavar="filename",
                        dest='single_output_file',
                        type=argparse.FileType('w'),
                        default=None, help='only output a single file with '
                        'the specified filename; use a single dash "-" to '
                        'specify that output should go to STDOUT (the '
                        'terminal)')
    parser.add_argument('input_filenames', metavar='input_sequence_filename',
                        help='Input FAST[AQ] sequence filename.', nargs='+')
    parser.add_argument('--report-total-kmers', '-t', action='store_true',
                        help="Prints the total number of k-mers"
                        " post-normalization to stderr")
    parser.add_argument('--force', default=False, action='store_true',
                        help='Overwrite output file if it exists')
    add_loadhash_args(parser)
    return parser

def main():  # pylint: disable=too-many-branches,too-many-statements
    info('normalize-by-median.py', ['diginorm'])
    args = get_parser().parse_args()

    report_on_config(args)

    report_fp = args.report

    # check for similar filenames
    filenames = []
    for pathfilename in args.input_filenames:
        filename = pathfilename.split('/')[-1]
        if (filename in filenames):
            print >>sys.stderr, "WARNING: At least two input files are named \
%s . (The script normalize-by-median.py can not handle this, only one .keep \
file for one of the input files will be generated.)" % filename
        else:
            filenames.append(filename)

    # check for others
    check_valid_file_exists(args.input_filenames)
    check_space(args.input_filenames, args.force)
    if args.savetable:
        check_space_for_hashtable(
            args.n_tables * args.min_tablesize, args.force)

    # list to save error files along with throwing exceptions
    corrupt_files = []

    if args.loadtable:
        print 'loading k-mer counting table from', args.loadtable
        htable = khmer.load_counting_hash(args.loadtable)
    else:
        print >> sys.stderr, 'making k-mer counting table'
        htable = khmer.new_counting_hash(args.ksize, args.min_tablesize,
                                         args.n_tables)

    input_filename = None

    for index, input_filename in enumerate(args.input_filenames):
        total_acc, discarded_acc, corrupt_files = \
            normalize_by_median_and_check(
                input_filename, htable, args.single_output_file,
                args.fail_save, args.paired, args.cutoff, args.force,
                corrupt_files, report_fp)

        if (args.dump_frequency > 0 and
                index > 0 and index % args.dump_frequency == 0):
            print 'Backup: Saving k-mer counting file through', input_filename
            if args.savetable:
                hashname = args.savetable
                print '...saving to', hashname
            else:
                hashname = 'backup.ct'
                print 'Nothing given for savetable, saving to', hashname
            htable.save(hashname)

    if args.paired and args.unpaired_reads:
        args.paired = False
        output_name = args.unpaired_reads
        if not args.single_output_file:
            output_name = os.path.basename(args.unpaired_reads) + '.keep'
        outfp = open(output_name, 'w')
        total_acc, discarded_acc, corrupt_files = \
            normalize_by_median_and_check(
                args.unpaired_reads, htable, args.single_output_file,
                args.fail_save, args.paired, args.cutoff, args.force,
                corrupt_files, report_fp)

    if args.report_total_kmers:
        print >> sys.stderr, 'Total number of unique k-mers: {0}'.format(
            htable.n_unique_kmers())

    if args.savetable:
        print 'Saving k-mer counting table through', input_filename
        print '...saving to', args.savetable
        htable.save(args.savetable)

    fp_rate = \
        khmer.calc_expected_collisions(htable, args.force, max_false_pos=.8)
    # for max_false_pos see Zhang et al., http://arxiv.org/abs/1309.2975

    print >> sys.stderr, \
        'fp rate estimated to be {fpr:1.3f}'.format(fpr=fp_rate)

    if args.force and len(corrupt_files) > 0:
        print >> sys.stderr, "** WARNING: Finished with errors!"
        print >> sys.stderr, "** IOErrors occurred in the following files:"
        print >> sys.stderr, "\t", " ".join(corrupt_files)

if __name__ == '__main__':
    main()

# vim: set ft=python ts=4 sts=4 sw=4 et tw=79:
<commit_msg>initial CM implementation<commit_after>#! /usr/bin/env python2
#
# This file is part of khmer, http://github.com/ged-lab/khmer/, and is
# Copyright (C) Michigan State University, 2009-2015. It is licensed under
# the three-clause BSD license; see LICENSE.
# Contact: khmer-project@idyll.org
#
# pylint: disable=invalid-name,missing-docstring
"""
Eliminate surplus reads.

Eliminate reads with median k-mer abundance higher than
DESIRED_COVERAGE.  Output sequences will be placed in 'infile.keep', with the
option to output to STDOUT.

% python scripts/normalize-by-median.py [ -C <cutoff> ] <data1> <data2> ...

Use '-h' for parameter help.
"""

import sys
import screed
import os
import khmer
import textwrap
from itertools import izip
from contextlib import contextmanager

from khmer.khmer_args import (build_counting_args, add_loadhash_args,
                              report_on_config, info)
import argparse
from khmer.kfile import (check_space, check_space_for_hashtable,
                         check_valid_file_exists)
from khmer.utils import write_record, check_is_pair
DEFAULT_DESIRED_COVERAGE = 10

# Iterate a collection in arbitrary batches
# from: http://stackoverflow.com/questions/4628290/pairs-from-single-list

def batchwise(coll, size):
    iter_coll = iter(coll)
    return izip(*[iter_coll] * size)

# Returns true if the pair of records are properly pairs

# pylint: disable=too-many-locals,too-many-branches
def normalize_by_median(input_filename, outfp, htable, paired, cutoff,
                        report_fp=None):

    desired_coverage = cutoff
    ksize = htable.ksize()

    # In paired mode we read two records at a time
    batch_size = 1
    if paired:
        batch_size = 2

    index = -1
    total = 0
    discarded = 0
    for index, batch in enumerate(batchwise(screed.open(
            input_filename, parse_description=False), batch_size)):
        if index > 0 and index % 100000 == 0:
            print >>sys.stderr, '... kept {kept} of {total} or'\
                ' {perc:2}%'.format(kept=total - discarded, total=total,
                                    perc=int(100. - discarded /
                                             float(total) * 100.))
            print >>sys.stderr, '... in file', input_filename

            if report_fp:
                print >> report_fp, total, total - discarded, \
                    1. - (discarded / float(total))
                report_fp.flush()

        total += batch_size

        # If in paired mode, check that the reads are properly interleaved

        if paired:
            if not check_is_pair(batch[0], batch[1]):
                raise IOError('Error: Improperly interleaved pairs \
                    {b0} {b1}'.format(b0=batch[0].name, b1=batch[1].name))

        # Emit the batch of reads if any read passes the filter
        # and all reads are longer than K
        passed_filter = False
        passed_length = True
        for record in batch:
            if len(record.sequence) < ksize:
                passed_length = False
                continue

            seq = record.sequence.replace('N', 'A')
            med, _, _ = htable.get_median_count(seq)

            if med < desired_coverage:
                htable.consume(seq)
                passed_filter = True

        # Emit records if any passed
        if passed_length and passed_filter:
            for record in batch:
                write_record(record, outfp)
        else:
            discarded += batch_size

    if report_fp:
        print >> report_fp, total, total - discarded, \
            1. - (discarded / float(total))
        report_fp.flush()

    return total, discarded

def handle_error(error, output_name, input_name, fail_save, htable):
    print >> sys.stderr, '** ERROR:', error
    print >> sys.stderr, '** Failed on {name}: '.format(name=input_name)
    if fail_save:
        tablename = os.path.basename(input_name) + '.ct.failed'
        print >> sys.stderr, \
            '** ...dumping k-mer counting table to {tn}'.format(tn=tablename)
        htable.save(tablename)
    try:
        os.remove(output_name)
    except:  # pylint: disable=bare-except
        print >> sys.stderr, '** ERROR: problem removing corrupt filtered file'

@contextmanager
def FailSafe(ifile, ofile, save_on_fail, ht, corrupted, total, dicarded, jedi):
    try:
        yield
    except IOError as err:
        handle_error(err, ofile, ifile, save_on_fail,
                     ht)
        if not jedi:
            print >> sys.stderr, '** Exiting!'

            sys.exit(1)
        else:
            print >> sys.stderr, '*** Skipping error file, moving on...'
            corrupted.append(ifile)

def normalize_by_median_and_check(input_filename, htable, single_output_file,
                                  fail_save, paired, cutoff, force,
                                  corrupt_files, report_fp=None):
    total = 0
    discarded = 0

    total_acc = None
    discarded_acc = None

    if single_output_file:
        if single_output_file is sys.stdout:
            output_name = '/dev/stdout'
        else:
            output_name = single_output_file.name
        outfp = single_output_file

    else:
        output_name = os.path.basename(input_filename) + '.keep'
        outfp = open(output_name, 'w')

    with FailSafe(input_filename, outfp, fail_save, htable, corrupt_files,
                  total, discarded, force):

        total_acc, discarded_acc = normalize_by_median(
            input_filename, outfp, htable, paired, cutoff, report_fp=None)

        if total_acc == 0 and discarded_acc == 0:
            print >> sys.stderr, 'SKIPPED empty file', input_filename
        elif total_acc is not None and discarded_acc is not None:
            # will be none if the context manager caught an error
            total += total_acc
            discarded += discarded_acc
            print >> sys.stderr, \
                'DONE with {inp}; kept {kept} of {total} or {perc:2}%'\
                .format(inp=input_filename, kept=total - discarded,
                        total=total, perc=int(100. - discarded /
                                              float(total) * 100.))
            print >> sys.stderr, 'output in', outfp

    return total_acc, discarded_acc, corrupt_files

def get_parser():
    epilog = ("""
    Discard sequences based on whether or not their median k-mer abundance lies
    above a specified cutoff. Kept sequences will be placed in <fileN>.keep.

    Paired end reads will be considered together if :option:`-p` is set. If
    either read will be kept, then both will be kept. This should result in
    keeping (or discarding) each sequencing fragment. This helps with retention
    of repeats, especially. With :option: `-u`/:option:`--unpaired-reads`,
    unpaired reads from the specified file will be read after the paired data
    is read.

    With :option:`-s`/:option:`--savetable`, the k-mer counting table
    will be saved to the specified file after all sequences have been
    processed. With :option:`-d`, the k-mer counting table will be
    saved every d files for multifile runs; if :option:`-s` is set,
    the specified name will be used, and if not, the name `backup.ct`
    will be used.  :option:`-l`/:option:`--loadtable` will load the
    specified k-mer counting table before processing the specified
    files.  Note that these tables are are in the same format as those
    produced by :program:`load-into-counting.py` and consumed by
    :program:`abundance-dist.py`.

    :option:`-f`/:option:`--fault-tolerant` will force the program to continue
    upon encountering a formatting error in a sequence file; the k-mer counting
    table up to that point will be dumped, and processing will continue on the
    next file.

    To append reads to an output file (rather than overwriting it), send output
    to STDOUT with `--out -` and use UNIX file redirection syntax (`>>`) to
    append to the file.

    Example::

        normalize-by-median.py -k 17 tests/test-data/test-abund-read-2.fa

    Example::

""" "        normalize-by-median.py -p -k 17 tests/test-data/test-abund-read-paired.fa"  # noqa
    """

    Example::

""" "        normalize-by-median.py -p -k 17 -o - tests/test-data/paired.fq >> appended-output.fq"  # noqa
    """

    Example::

""" "        normalize-by-median.py -k 17 -f tests/test-data/test-error-reads.fq tests/test-data/test-fastq-reads.fq"  # noqa
    """

    Example::

""" "        normalize-by-median.py -k 17 -d 2 -s test.ct tests/test-data/test-abund-read-2.fa tests/test-data/test-fastq-reads")   # noqa
    parser = build_counting_args(
        descr="Do digital normalization (remove mostly redundant sequences)",
        epilog=textwrap.dedent(epilog))
    parser.add_argument('-C', '--cutoff', type=int,
                        default=DEFAULT_DESIRED_COVERAGE)
    parser.add_argument('-p', '--paired', action='store_true')
    parser.add_argument('-u', '--unpaired-reads',
                        metavar="unpaired_reads_filename", help='with paired data only,\
                        include an unpaired file')
    parser.add_argument('-s', '--savetable', metavar="filename", default='',
                        help='save the k-mer counting table to disk after all'
                        'reads are loaded.')
    parser.add_argument('-R', '--report',
                        metavar='filename', type=argparse.FileType('w'))
    parser.add_argument('-f', '--fault-tolerant', dest='force',
                        help='continue on next file if read errors are \
                         encountered', action='store_true')
    parser.add_argument('--save-on-failure', dest='fail_save',
                        action='store_false', default=True,
                        help='Save k-mer counting table when an error occurs')
    parser.add_argument('-d', '--dump-frequency', dest='dump_frequency',
                        type=int, help='dump k-mer counting table every d '
                        'files', default=-1)
    parser.add_argument('-o', '--out', metavar="filename",
                        dest='single_output_file',
                        type=argparse.FileType('w'),
                        default=None, help='only output a single file with '
                        'the specified filename; use a single dash "-" to '
                        'specify that output should go to STDOUT (the '
                        'terminal)')
    parser.add_argument('input_filenames', metavar='input_sequence_filename',
                        help='Input FAST[AQ] sequence filename.', nargs='+')
    parser.add_argument('--report-total-kmers', '-t', action='store_true',
                        help="Prints the total number of k-mers"
                        " post-normalization to stderr")
    parser.add_argument('--force', default=False, action='store_true',
                        help='Overwrite output file if it exists')
    add_loadhash_args(parser)
    return parser

def main():  # pylint: disable=too-many-branches,too-many-statements
    info('normalize-by-median.py', ['diginorm'])
    args = get_parser().parse_args()

    report_on_config(args)

    report_fp = args.report

    # check for similar filenames
    filenames = []
    for pathfilename in args.input_filenames:
        filename = pathfilename.split('/')[-1]
        if (filename in filenames):
            print >>sys.stderr, "WARNING: At least two input files are named \
%s . (The script normalize-by-median.py can not handle this, only one .keep \
file for one of the input files will be generated.)" % filename
        else:
            filenames.append(filename)

    # check for others
    check_valid_file_exists(args.input_filenames)
    check_space(args.input_filenames, args.force)
    if args.savetable:
        check_space_for_hashtable(
            args.n_tables * args.min_tablesize, args.force)

    # list to save error files along with throwing exceptions
    corrupt_files = []

    if args.loadtable:
        print 'loading k-mer counting table from', args.loadtable
        htable = khmer.load_counting_hash(args.loadtable)
    else:
        print >> sys.stderr, 'making k-mer counting table'
        htable = khmer.new_counting_hash(args.ksize, args.min_tablesize,
                                         args.n_tables)

    input_filename = None

    for index, input_filename in enumerate(args.input_filenames):
        total_acc, discarded_acc, corrupt_files = \
            normalize_by_median_and_check(
                input_filename, htable, args.single_output_file,
                args.fail_save, args.paired, args.cutoff, args.force,
                corrupt_files, report_fp)

        if (args.dump_frequency > 0 and
                index > 0 and index % args.dump_frequency == 0):
            print 'Backup: Saving k-mer counting file through', input_filename
            if args.savetable:
                hashname = args.savetable
                print '...saving to', hashname
            else:
                hashname = 'backup.ct'
                print 'Nothing given for savetable, saving to', hashname
            htable.save(hashname)

    if args.paired and args.unpaired_reads:
        args.paired = False
        output_name = args.unpaired_reads
        if not args.single_output_file:
            output_name = os.path.basename(args.unpaired_reads) + '.keep'
        outfp = open(output_name, 'w')
        total_acc, discarded_acc, corrupt_files = \
            normalize_by_median_and_check(
                args.unpaired_reads, htable, args.single_output_file,
                args.fail_save, args.paired, args.cutoff, args.force,
                corrupt_files, report_fp)

    if args.report_total_kmers:
        print >> sys.stderr, 'Total number of unique k-mers: {0}'.format(
            htable.n_unique_kmers())

    if args.savetable:
        print 'Saving k-mer counting table through', input_filename
        print '...saving to', args.savetable
        htable.save(args.savetable)

    fp_rate = \
        khmer.calc_expected_collisions(htable, args.force, max_false_pos=.8)
    # for max_false_pos see Zhang et al., http://arxiv.org/abs/1309.2975

    print >> sys.stderr, \
        'fp rate estimated to be {fpr:1.3f}'.format(fpr=fp_rate)

    if args.force and len(corrupt_files) > 0:
        print >> sys.stderr, "** WARNING: Finished with errors!"
        print >> sys.stderr, "** IOErrors occurred in the following files:"
        print >> sys.stderr, "\t", " ".join(corrupt_files)

if __name__ == '__main__':
    main()

# vim: set ft=python ts=4 sts=4 sw=4 et tw=79:
<|endoftext|>


<commit_before># coding=utf-8

import json
import re

from nemubot import context
from nemubot.exception import IMException
from nemubot.hooks import hook
from nemubot.tools.xmlparser.node import ModuleState

nemubotversion = 3.4

from nemubot.module.more import Response
from nemubot.module.networking.page import headers

PASSWD_FILE = None
# You can get one with: curl -b "sessionid=YOURSESSIONID" 'https://accounts.cri.epita.net/api/users/?limit=10000' > users.json
APIEXTRACT_FILE = None

def load(context):
    global PASSWD_FILE
    if not context.config or "passwd" not in context.config:
        print("No passwd file given")
    else:
        PASSWD_FILE = context.config["passwd"]
        print("passwd file loaded:", PASSWD_FILE)

    global APIEXTRACT_FILE
    if not context.config or "apiextract" not in context.config:
        print("No passwd file given")
    else:
        APIEXTRACT_FILE = context.config["apiextract"]
        print("JSON users file loaded:", APIEXTRACT_FILE)

    if PASSWD_FILE is None and APIEXTRACT_FILE is None:
        return None

    if not context.data.hasNode("aliases"):
        context.data.addChild(ModuleState("aliases"))
    context.data.getNode("aliases").setIndex("from", "alias")

    if not context.data.hasNode("pics"):
        context.data.addChild(ModuleState("pics"))
    context.data.getNode("pics").setIndex("login", "pict")

    import nemubot.hooks
    context.add_hook(nemubot.hooks.Command(cmd_whois, "whois", keywords={"lookup": "Perform a lookup of the begining of the login instead of an exact search."}),
                     "in","Command")

class Login:

    def __init__(self, line=None, login=None, uidNumber=None, firstname=None, lastname=None, promo=None, **kwargs):
        if line is not None:
            s = line.split(":")
            self.login = s[0]
            self.uid = s[2]
            self.gid = s[3]
            self.cn = s[4]
            self.home = s[5]
        else:
            self.login = login
            self.uid = uidNumber
            self.promo = promo
            self.cn = firstname + " " + lastname
            try:
                self.gid = "epita" + str(int(promo))
            except:
                self.gid = promo

    def get_promo(self):
        if hasattr(self, "promo"):
            return self.promo
        if hasattr(self, "home"):
            try:
                return self.home.split("/")[2].replace("_", " ")
            except:
                return self.gid

    def get_photo(self):
        if self.login in context.data.getNode("pics").index:
            return context.data.getNode("pics").index[self.login]["url"]
        for url in [ "https://photos.cri.epita.fr/%s", "https://intra-bocal.epitech.eu/trombi/%s.jpg" ]:
            url = url % self.login
            try:
                _, status, _, _ = headers(url)
                if status == 200:
                    return url
            except:
                logger.exception("On URL %s", url)
        return None

def login_lookup(login, search=False):
    if login in context.data.getNode("aliases").index:
        login = context.data.getNode("aliases").index[login]["to"]

    if APIEXTRACT_FILE:
        with open(APIEXTRACT_FILE, encoding="utf-8") as f:
            api = json.load(f)
            for l in api["results"]:
                if (not search and l["login"] == login) or (search and (("login" in l and l["login"].find(login) != -1) or ("cn" in l and l["cn"].find(login) != -1) or ("uid" in l and str(l["uid"]) == login))):
                    yield Login(**l)

    login_ = login + (":" if not search else "")
    lsize = len(login_)

    if PASSWD_FILE:
        with open(PASSWD_FILE, encoding="iso-8859-15") as f:
            for l in f.readlines():
                if l[:lsize] == login_:
                    yield Login(l.strip())

def cmd_whois(msg):
    if len(msg.args) < 1:
        raise IMException("Provide a name")

    def format_response(t):
        srch, l = t
        if type(l) is Login:
            pic = l.get_photo()
            return "%s is %s (%s %s): %s%s" % (srch, l.cn.capitalize(), l.login, l.uid, l.get_promo(), " and looks like %s" % pic if pic is not None else "")
        else:
            return l % srch

    res = Response(channel=msg.channel, count=" (%d more logins)", line_treat=format_response)
    for srch in msg.args:
        found = False
        for l in login_lookup(srch, "lookup" in msg.kwargs):
            found = True
            res.append_message((srch, l))
        if not found:
            res.append_message((srch, "Unknown %s :("))
    return res

@hook.command("nicks")
def cmd_nicks(msg):
    if len(msg.args) < 1:
        raise IMException("Provide a login")
    nick = login_lookup(msg.args[0])
    if nick is None:
        nick = msg.args[0]
    else:
        nick = nick.login

    nicks = []
    for alias in context.data.getNode("aliases").getChilds():
        if alias["to"] == nick:
            nicks.append(alias["from"])
    if len(nicks) >= 1:
        return Response("%s is also known as %s." % (nick, ", ".join(nicks)), channel=msg.channel)
    else:
        return Response("%s has no known alias." % nick, channel=msg.channel)

@hook.ask()
def parseask(msg):
    res = re.match(r"^(\S+)\s*('s|suis|est|is|was|were)\s+([a-zA-Z0-9_-]{3,8})$", msg.message, re.I)
    if res is not None:
        nick = res.group(1)
        login = res.group(3)
        if nick == "my" or nick == "I" or nick == "i" or nick == "je" or nick == "mon" or nick == "ma":
            nick = msg.frm
        if nick in context.data.getNode("aliases").index:
            context.data.getNode("aliases").index[nick]["to"] = login
        else:
            ms = ModuleState("alias")
            ms.setAttribute("from", nick)
            ms.setAttribute("to", login)
            context.data.getNode("aliases").addChild(ms)
        context.save()
        return Response("ok, c'est noté, %s est %s"
                        % (nick, login),
                        channel=msg.channel,
                        nick=msg.frm)
<commit_msg>whois: don't use custom picture anymore<commit_after># coding=utf-8

import json
import re

from nemubot import context
from nemubot.exception import IMException
from nemubot.hooks import hook
from nemubot.tools.xmlparser.node import ModuleState

nemubotversion = 3.4

from nemubot.module.more import Response
from nemubot.module.networking.page import headers

PASSWD_FILE = None
# You can get one with: curl -b "sessionid=YOURSESSIONID" 'https://accounts.cri.epita.net/api/users/?limit=10000' > users.json
APIEXTRACT_FILE = None

def load(context):
    global PASSWD_FILE
    if not context.config or "passwd" not in context.config:
        print("No passwd file given")
    else:
        PASSWD_FILE = context.config["passwd"]
        print("passwd file loaded:", PASSWD_FILE)

    global APIEXTRACT_FILE
    if not context.config or "apiextract" not in context.config:
        print("No passwd file given")
    else:
        APIEXTRACT_FILE = context.config["apiextract"]
        print("JSON users file loaded:", APIEXTRACT_FILE)

    if PASSWD_FILE is None and APIEXTRACT_FILE is None:
        return None

    if not context.data.hasNode("aliases"):
        context.data.addChild(ModuleState("aliases"))
    context.data.getNode("aliases").setIndex("from", "alias")

    import nemubot.hooks
    context.add_hook(nemubot.hooks.Command(cmd_whois, "whois", keywords={"lookup": "Perform a lookup of the begining of the login instead of an exact search."}),
                     "in","Command")

class Login:

    def __init__(self, line=None, login=None, uidNumber=None, firstname=None, lastname=None, promo=None, **kwargs):
        if line is not None:
            s = line.split(":")
            self.login = s[0]
            self.uid = s[2]
            self.gid = s[3]
            self.cn = s[4]
            self.home = s[5]
        else:
            self.login = login
            self.uid = uidNumber
            self.promo = promo
            self.cn = firstname + " " + lastname
            try:
                self.gid = "epita" + str(int(promo))
            except:
                self.gid = promo

    def get_promo(self):
        if hasattr(self, "promo"):
            return self.promo
        if hasattr(self, "home"):
            try:
                return self.home.split("/")[2].replace("_", " ")
            except:
                return self.gid

    def get_photo(self):
        for url in [ "https://photos.cri.epita.fr/%s", "https://intra-bocal.epitech.eu/trombi/%s.jpg" ]:
            url = url % self.login
            try:
                _, status, _, _ = headers(url)
                if status == 200:
                    return url
            except:
                logger.exception("On URL %s", url)
        return None

def login_lookup(login, search=False):
    if login in context.data.getNode("aliases").index:
        login = context.data.getNode("aliases").index[login]["to"]

    if APIEXTRACT_FILE:
        with open(APIEXTRACT_FILE, encoding="utf-8") as f:
            api = json.load(f)
            for l in api["results"]:
                if (not search and l["login"] == login) or (search and (("login" in l and l["login"].find(login) != -1) or ("cn" in l and l["cn"].find(login) != -1) or ("uid" in l and str(l["uid"]) == login))):
                    yield Login(**l)

    login_ = login + (":" if not search else "")
    lsize = len(login_)

    if PASSWD_FILE:
        with open(PASSWD_FILE, encoding="iso-8859-15") as f:
            for l in f.readlines():
                if l[:lsize] == login_:
                    yield Login(l.strip())

def cmd_whois(msg):
    if len(msg.args) < 1:
        raise IMException("Provide a name")

    def format_response(t):
        srch, l = t
        if type(l) is Login:
            pic = l.get_photo()
            return "%s is %s (%s %s): %s%s" % (srch, l.cn.capitalize(), l.login, l.uid, l.get_promo(), " and looks like %s" % pic if pic is not None else "")
        else:
            return l % srch

    res = Response(channel=msg.channel, count=" (%d more logins)", line_treat=format_response)
    for srch in msg.args:
        found = False
        for l in login_lookup(srch, "lookup" in msg.kwargs):
            found = True
            res.append_message((srch, l))
        if not found:
            res.append_message((srch, "Unknown %s :("))
    return res

@hook.command("nicks")
def cmd_nicks(msg):
    if len(msg.args) < 1:
        raise IMException("Provide a login")
    nick = login_lookup(msg.args[0])
    if nick is None:
        nick = msg.args[0]
    else:
        nick = nick.login

    nicks = []
    for alias in context.data.getNode("aliases").getChilds():
        if alias["to"] == nick:
            nicks.append(alias["from"])
    if len(nicks) >= 1:
        return Response("%s is also known as %s." % (nick, ", ".join(nicks)), channel=msg.channel)
    else:
        return Response("%s has no known alias." % nick, channel=msg.channel)

@hook.ask()
def parseask(msg):
    res = re.match(r"^(\S+)\s*('s|suis|est|is|was|were)\s+([a-zA-Z0-9_-]{3,8})$", msg.message, re.I)
    if res is not None:
        nick = res.group(1)
        login = res.group(3)
        if nick == "my" or nick == "I" or nick == "i" or nick == "je" or nick == "mon" or nick == "ma":
            nick = msg.frm
        if nick in context.data.getNode("aliases").index:
            context.data.getNode("aliases").index[nick]["to"] = login
        else:
            ms = ModuleState("alias")
            ms.setAttribute("from", nick)
            ms.setAttribute("to", login)
            context.data.getNode("aliases").addChild(ms)
        context.save()
        return Response("ok, c'est noté, %s est %s"
                        % (nick, login),
                        channel=msg.channel,
                        nick=msg.frm)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

"""
Deploys the website to a target directory supplied as an argument.
"""

import json
import os
import sys

def iterate_over(dict_or_list, result):
    """
    Iterates recursively over nested lists and dictionaries
    keeping track of all "path" values with the key "includePath"
    within nested dictionaries.
    """
    if isinstance(dict_or_list, list):
        for child in dict_or_list:
            iterate_over(child, result)
    elif isinstance(dict_or_list, dict):
        for key, value in dict_or_list.items():
            if key == "includePath":
                for child in value:
                    result.add(child["path"])
            else:
                iterate_over(value, result)

def main(arguments):
    """Main function of this program."""

    workspace = os.path.realpath(os.path.join(__file__, os.pardir, os.pardir))
    print("Workspace root: '{}'".format(workspace))

    with open(os.path.join(workspace, ".vscode", ".cmaketools.json")) as f:
        data = json.loads(f.read())

    result = set()

    iterate_over(data, result)

    result = [x.replace(workspace, "${workspaceRoot}") for x in result]

    print(json.dumps(result, indent=0))

if __name__ == '__main__':
    sys.exit(main(sys.argv[1:]))
<commit_msg>Fix incorrect global comment in include path script<commit_after>#!/usr/bin/env python3

"""
Parses .vscode/.cmaketools.json to obtain a list of include paths.
These can then be subsequently pasted into .vscode/c_cpp_properties.json
to make intellisense work. This is script exists purely for convenience
and only needs to be used when the include paths change (e.g. when a new
dependency is added).
"""

import json
import os
import sys

def iterate_over(dict_or_list, result):
    """
    Iterates recursively over nested lists and dictionaries
    keeping track of all "path" values with the key "includePath"
    within nested dictionaries.
    """
    if isinstance(dict_or_list, list):
        for child in dict_or_list:
            iterate_over(child, result)
    elif isinstance(dict_or_list, dict):
        for key, value in dict_or_list.items():
            if key == "includePath":
                for child in value:
                    result.add(child["path"])
            else:
                iterate_over(value, result)

def main(arguments):
    """Main function of this program."""

    workspace = os.path.realpath(os.path.join(__file__, os.pardir, os.pardir))
    print("Workspace root: '{}'".format(workspace))

    with open(os.path.join(workspace, ".vscode", ".cmaketools.json")) as f:
        data = json.loads(f.read())

    result = set()

    iterate_over(data, result)

    result = [x.replace(workspace, "${workspaceRoot}") for x in result]

    print(json.dumps(result, indent=0))

if __name__ == '__main__':
    sys.exit(main(sys.argv[1:]))
<|endoftext|>


<commit_before>import sys
import numpy as np
import os

import matplotlib.pyplot as plt

output_dir = sys.argv[1]

def filename(dataset, workflow, size, ext):
    return os.path.join(output_dir, "%s_%s_%s.%s" % (dataset, workflow, size, ext))

def get_overhead_percent(filename):
    with open(filename) as f:
        op_time = 0.0
        for line in f:
            line = line.replace("\n", "")
            if "Operation" in line:
                continue
            if "Full timer" in line:
                total_time = float(line.split(", ")[1])
                return op_time / total_time
            op_time += float(line.split(", ")[1])

def get_table_sizes(filename):
    with open(filename) as f:
        lines = []
        for line in f:
            line = line.replace("\n", "").split(", ")
            table = line[0]
            count = int(line[1])
            lines.append((table, count))
        return lines

modelsizes = {}
dbsizes = {}
overhead = {}
tablesizes = {}
for dataset in ("imdb", "housing", "animal"):
    for workflow in ("simple", "full", "exploratory"):
        dbsizes[(dataset, workflow)] = []
        modelsizes[(dataset, workflow)] = []
        overhead[(dataset, workflow)] = []
        for size in (1, 10000, 20000, 40000, 70000, 100000, 200000, 400000, 700000, 1000000):
            fname = filename(dataset, workflow, size, "dbsize")
            if os.path.isfile(fname):
                with open(fname) as f:
                    dbsize = f.read().split("\t")[0]
                    orig_dbsize = dbsize
                    if "M" in dbsize:
                        dbsize = float(dbsize.replace("M", ""))
                    elif "K" in dbsize:
                        dbsize = float(dbsize.replace("K", "")) / 1024.0
                    dbsizes[(dataset, workflow)].append((orig_dbsize, dbsize))
            fname = filename(dataset, workflow, size, "modelsize")
            if os.path.isfile(fname):
                with open(fname) as f:
                    modelsize = f.readlines()[-1].split("\t")[0]
                    orig_modelsize = modelsize
                    if "M" in modelsize:
                        modelsize = float(modelsize.replace("M", ""))
                    elif "K" in modelsize:
                        modelsize = float(modelsize.replace("K", "")) / 1024.0
                    modelsizes[(dataset, workflow)].append((orig_modelsize, modelsize))
            fname = filename(dataset, workflow, size, "csv")
            if os.path.isfile(fname):
                ohead = get_overhead_percent(fname)
                overhead[(dataset, workflow)].append((size, ohead))
            fname = filename(dataset, workflow, size, "tablesizes")
            if os.path.isfile(fname):
                if size == 1:
                    tablesizes[(dataset, workflow)] = get_table_sizes(fname)

print "Database Sizes"
for (k, v) in dbsizes.iteritems():
    dbsize = max([x[1] for x in v])
    dbsize = [x[0] for x in v if x[1] == dbsize][0]
    print ", ".join((k[0], k[1], dbsize))

print "Model Sizes"
for (k, v) in modelsizes.iteritems():
    dbsize = max([x[1] for x in v])
    dbsize = [x[0] for x in v if x[1] == dbsize][0]
    print ", ".join((k[0], k[1], dbsize))

for (k, v) in overhead.iteritems():
    plt.scatter([x[0] for x in v], [y[1] for y in v])
    plt.title("Overhead percent for %s %s" % k)
    plt.xlabel("Dataset size (num rows)")
    plt.ylabel("ModelDB Overhead time (%)")
    plt.xlim([0, 1000000])
    plt.ylim([0, 0.7])
    plt.show()

for (k, v) in tablesizes.iteritems():
    plt.figure(figsize=(18, 10))
    objects = [x[0] for x in v]
    y_pos = np.arange(len(objects))
    performance = [x[1] for x in v]
    plt.barh(y_pos, performance, align='center', alpha=0.5)
    plt.yticks(y_pos, objects)
    plt.xlabel('Size')
    plt.title('Table sizes for %s %s' % k)
    plt.show()
<commit_msg>Fix plot title and axes<commit_after>import sys
import numpy as np
import os

import matplotlib.pyplot as plt

output_dir = sys.argv[1]

def filename(dataset, workflow, size, ext):
    return os.path.join(output_dir, "%s_%s_%s.%s" % (dataset, workflow, size, ext))

def get_overhead_percent(filename):
    with open(filename) as f:
        op_time = 0.0
        for line in f:
            line = line.replace("\n", "")
            if "Operation" in line:
                continue
            if "Full timer" in line:
                total_time = float(line.split(", ")[1])
                return op_time / total_time
            op_time += float(line.split(", ")[1])

def get_table_sizes(filename):
    with open(filename) as f:
        lines = []
        for line in f:
            line = line.replace("\n", "").split(", ")
            table = line[0]
            count = int(line[1])
            lines.append((table, count))
        return lines

modelsizes = {}
dbsizes = {}
overhead = {}
tablesizes = {}
for dataset in ("imdb", "housing", "animal"):
    for workflow in ("simple", "full", "exploratory"):
        dbsizes[(dataset, workflow)] = []
        modelsizes[(dataset, workflow)] = []
        overhead[(dataset, workflow)] = []
        for size in (1, 10000, 20000, 40000, 70000, 100000, 200000, 400000, 700000, 1000000):
            fname = filename(dataset, workflow, size, "dbsize")
            if os.path.isfile(fname):
                with open(fname) as f:
                    dbsize = f.read().split("\t")[0]
                    orig_dbsize = dbsize
                    if "M" in dbsize:
                        dbsize = float(dbsize.replace("M", ""))
                    elif "K" in dbsize:
                        dbsize = float(dbsize.replace("K", "")) / 1024.0
                    dbsizes[(dataset, workflow)].append((orig_dbsize, dbsize))
            fname = filename(dataset, workflow, size, "modelsize")
            if os.path.isfile(fname):
                with open(fname) as f:
                    modelsize = f.readlines()[-1].split("\t")[0]
                    orig_modelsize = modelsize
                    if "M" in modelsize:
                        modelsize = float(modelsize.replace("M", ""))
                    elif "K" in modelsize:
                        modelsize = float(modelsize.replace("K", "")) / 1024.0
                    modelsizes[(dataset, workflow)].append((orig_modelsize, modelsize))
            fname = filename(dataset, workflow, size, "csv")
            if os.path.isfile(fname):
                ohead = get_overhead_percent(fname)
                overhead[(dataset, workflow)].append((size, ohead))
            fname = filename(dataset, workflow, size, "tablesizes")
            if os.path.isfile(fname):
                if size == 1:
                    tablesizes[(dataset, workflow)] = get_table_sizes(fname)

print "Database Sizes"
for (k, v) in dbsizes.iteritems():
    dbsize = max([x[1] for x in v])
    dbsize = [x[0] for x in v if x[1] == dbsize][0]
    print ", ".join((k[0], k[1], dbsize))

print "Model Sizes"
for (k, v) in modelsizes.iteritems():
    dbsize = max([x[1] for x in v])
    dbsize = [x[0] for x in v if x[1] == dbsize][0]
    print ", ".join((k[0], k[1], dbsize))

for (k, v) in overhead.iteritems():
    plt.scatter([x[0] for x in v], [y[1]*100 for y in v])
    plt.title("Time Overhead (as percentage of overall time) for %s %s" % k)
    plt.xlabel("Dataset size (num rows)")
    plt.ylabel("ModelDB Overhead time (%)")
    plt.xlim([0, 1000000])
    plt.ylim([0, 70])
    plt.show()

for (k, v) in tablesizes.iteritems():
    plt.figure(figsize=(18, 10))
    objects = [x[0] for x in v]
    y_pos = np.arange(len(objects))
    performance = [x[1] for x in v]
    plt.barh(y_pos, performance, align='center', alpha=0.5)
    plt.yticks(y_pos, objects)
    plt.xlabel('Size')
    plt.title('Table sizes for %s %s' % k)
    plt.show()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from __future__ import absolute_import

import uuid
import argparse
import logging

from moneta.cluster import MonetaCluster
from moneta.server import MonetaServer

logger = logging.getLogger('moneta')

def run():
    parser = argparse.ArgumentParser()
    parser.add_argument('--listen', nargs='?', default='127.0.0.1:32000', help='Listen host:port')
    parser.add_argument('--zookeeper', nargs='?', default='127.0.0.1:2181', help='Zookeeper hosts (comma-separated list of host:port items)')
    parser.add_argument('--nodename', nargs='?', default=uuid.uuid1().hex, help='Node name')
    parser.add_argument('--pools', nargs='?', type=lambda s: s.split(','), default="default", help='Comma separated list of pools')
    parser.add_argument('--logfile', nargs='?', default=None, help='Log file')
    parser.add_argument('--loglevel', nargs='?', default="info", help='Log level', choices = ['debug', 'info', 'warning', 'error', 'critical', 'fatal'])
    args = parser.parse_args()

    if args.logfile:
        logging.basicConfig(filename = args.logfile, format =  '%(asctime)s [%(name)s] %(levelname)s: %(message)s')
    else:
        logging.basicConfig(format =  '%(asctime)s [%(name)s] %(levelname)s: %(message)s')

    loglevel = {
        'debug': logging.DEBUG,
        'info': logging.INFO,
        'warning': logging.WARNING,
        'error': logging.ERROR,
        'critical': logging.CRITICAL,
        'fatal': logging.FATAL
    }[args.loglevel]

    logger.setLevel(loglevel)

    try:
        logger.debug('Starting')
        cluster = MonetaCluster(args.nodename, args.listen, args.zookeeper, pools = args.pools)
        server = MonetaServer(cluster, args.listen)
        logger.info('Started')

        server.run_forever()

    except BaseException:
        cluster.stop()
        logger.exception('An exception occurred. Terminating moneta.')
        exit(1)
<commit_msg>Better handling of SIGTERM and SIGINT<commit_after># -*- coding: utf-8 -*-

from __future__ import absolute_import

import uuid
import argparse
import logging
import sys
import signal

from moneta.cluster import MonetaCluster
from moneta.server import MonetaServer

logger = logging.getLogger('moneta')

def run():
    parser = argparse.ArgumentParser()
    parser.add_argument('--listen', nargs='?', default='127.0.0.1:32000', help='Listen host:port')
    parser.add_argument('--zookeeper', nargs='?', default='127.0.0.1:2181', help='Zookeeper hosts (comma-separated list of host:port items)')
    parser.add_argument('--nodename', nargs='?', default=uuid.uuid1().hex, help='Node name')
    parser.add_argument('--pools', nargs='?', type=lambda s: s.split(','), default="default", help='Comma separated list of pools')
    parser.add_argument('--logfile', nargs='?', default=None, help='Log file')
    parser.add_argument('--loglevel', nargs='?', default="info", help='Log level', choices = ['debug', 'info', 'warning', 'error', 'critical', 'fatal'])
    args = parser.parse_args()

    # Logging
    if args.logfile:
        logging.basicConfig(filename = args.logfile, format =  '%(asctime)s [%(name)s] %(levelname)s: %(message)s')
    else:
        logging.basicConfig(format =  '%(asctime)s [%(name)s] %(levelname)s: %(message)s')

    loglevel = {
        'debug': logging.DEBUG,
        'info': logging.INFO,
        'warning': logging.WARNING,
        'error': logging.ERROR,
        'critical': logging.CRITICAL,
        'fatal': logging.FATAL
    }[args.loglevel]

    logger.setLevel(loglevel)

    # Signals
    def handle_sigterm(_signo, _stack_frame):
        logger.debug('Received SIGTERM.')
        sys.exit(0)

    signal.signal(signal.SIGTERM, handle_sigterm)

    # Main
    try:
        logger.debug('Starting')
        cluster = MonetaCluster(args.nodename, args.listen, args.zookeeper, pools = args.pools)
        server = MonetaServer(cluster, args.listen)
        logger.info('Started')

        server.run_forever()

    except (SystemExit, KeyboardInterrupt):
        logger.info('Termination requested.')
        cluster.stop()
        sys.exit(0)

    except BaseException:
        logger.exception('An exception occurred. Terminating moneta.')
        cluster.stop()
        sys.exit(1)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging
from six import string_types

from .base import Machine

class Steps(Machine):
    """
    Implements a linear process with multiple steps

    Example::

        steps = [

            Step({
                'label': u'Level 1',
                'message': u'Initial capture of information',
            }, 1),

            Step({
                'label': u'Level 2',
                'message': u'Escalation to technical experts',
                'moderators': 'alice@acme.com',
            }, 2),

            Step({
                'label': u'Level 3',
                'message': u'Escalation to decision stakeholders',
                'participants': 'bob@acme.com',
            }, 3),

            Step({
                'label': u'Terminated',
                'message': u'Process is closed, yet conversation can continue',
            }, 4),

        ]
        machine = Steps(bot=bot, steps=steps)
        machine.start()
        ...

    """

    def on_init(self,
                steps=None,
                prefix='steps',
                **kwargs):
        """
        Handles extended initialisation parameters

        :param steps: The steps for this process
        :type steps: list of Step or of dict

        :param prefix: the main keyword for configuration of this machine
        :type prefix: str

        """
        super(Steps, self).on_init(prefix, **kwargs)

        self.steps = []
        if steps:
            for item in steps:
                if isinstance(item, dict):
                    self.steps.append(Step(item, len(self.steps)+1))
                else:
                    self.steps.append(item)

        states = ['begin',
                  'running',
                  'completed',
                  'end']

        transitions = [

            {'source': 'begin',
             'target': 'running',
             'condition': self.if_ready,
             'action': self.next_step,
            },

            {'source': 'running',
             'target': 'completed',
             'condition': self.step_has_completed,
            },

            {'source': 'completed',
             'target': 'running',
             'condition': self.if_next,
             'action': self.next_step,
            },

            {'source': 'completed',
             'target': 'end',
             'condition': self.if_end,
             'action': self.stop,
            },

        ]

        self.build(states=states,
                   transitions=transitions,
                   initial='begin')

    @property
    def current_step(self):
        """
        Gets current step

        :return: current step, or None
        """
        index = self.get('_index')
        if index is None:
            return None

        return self.steps[index]

    def next_step(self):
        """
        Moves to next step

        :return: current step, or None

        This function loads and runs the next step in the process, if any.
        If all steps have been consumed it returns None.
        """
        if not self.steps or len(self.steps) < 1:
            return None  # no steps have been set

        index = self.get('_index')

        if index is None:
            index = 0
        elif index < len(self.steps)-1:
            index += 1
        else:
            return None  # all steps have been used

        self.set('_index', index)

        step = self.steps[index]

        step.trigger(bot=self.bot)

        return step

    def step_has_completed(self, **kwargs):
        """
        Checks if the machine for this step has finished its job

        :return: False if the machine is still running, True otherwise
        """
        step = self.current_step
        if step is None:
            return True

        machine = step.machine
        if machine is None:
            return True

        return not machine.is_running()

    def if_ready(self, **kwargs):
        """
        Checks if state machine can engage on first step

        To be implemented in sub-class where complex initialization activities
        are required.
        """
        return True

    def if_next(self, **kwargs):
        """
        Checks if next step should be engaged

        :param event: a label of event submitted to the machine
        :type event: str

        This function is used by the state machine for testing the transition
        to the next available step in the process.

        Since this function is an integral part of the state machine itself, it
        should be triggered via a call of the ``step()`` member function.

        For example::

            machine.step(event='next')

        """

        if kwargs.get('event') == 'next':
            return True

        return False

    def if_end(self, **kwargs):
        """
        Checks if all steps have been used

        Since this function is an integral part of the state machine itself, it
        should be triggered via a call of the ``step()`` member function.

        For example::

            machine.step(event='tick')

        """

        index = self.get('_index')

        if index and index >= len(self.steps)-1:
            return True  # all steps have been used

        return False

class Step(object):
    """
    Represents a step in a linear process
    """

    def __init__(self, attributes, index):
        """
        Represents a step in a linear process

        :param attributes: the dict to use
        :type attributes: dict

        :param index: index of this step
        :type index: int

        """
        self.label = attributes.get('label', u'Step {}'.format(index))

        self.message = attributes.get('message',
                                      u'(no description is available)')

        self.content = attributes.get('content', None)

        self.file = attributes.get('file', None)

        self.moderators = attributes.get('moderators', [])
        if isinstance(self.moderators, string_types):
            self.moderators = [self.moderators]

        self.participants = attributes.get('participants', [])
        if isinstance(self.participants, string_types):
            self.participants = [self.participants]

        self.machine = attributes.get('machine', None)

    def say(self, bot):
        """
        Reports on this step

        :param bot: the bit to use
        :type bot: ShellBot

        This function posts to the chat space some information on this step.

        Example::

            step.say(bot)

        """
        bot.say(u"Current state: {} - {}".format(self.label,
                                                 self.message))

    def trigger(self, bot):
        """
        Triggers a step

        :param bot: the bit to use
        :type bot: ShellBot

        This function does everything that is coming with a step:
        - send a message to the chat space,
        - maybe in MarkDown or HTML,
        - maybe with some attachment,
        - add participants and/or moderators to the space,
        - start a state machine

        Example::

            step.trigger(bot)

        """
        bot.say(u"New state: {} - {}".format(self.label,
                                             self.message),
                content=self.content,
                file=self.file)

        bot.add_moderators(self.moderators)
        bot.add_participants(self.participants)

        if self.machine:
            self._step_process = self.machine.start()
<commit_msg>comment transitions in the Steps state machine<commit_after># -*- coding: utf-8 -*-

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging
from six import string_types

from .base import Machine

class Steps(Machine):
    """
    Implements a linear process with multiple steps

    Example::

        steps = [

            Step({
                'label': u'Level 1',
                'message': u'Initial capture of information',
            }, 1),

            Step({
                'label': u'Level 2',
                'message': u'Escalation to technical experts',
                'moderators': 'alice@acme.com',
            }, 2),

            Step({
                'label': u'Level 3',
                'message': u'Escalation to decision stakeholders',
                'participants': 'bob@acme.com',
            }, 3),

            Step({
                'label': u'Terminated',
                'message': u'Process is closed, yet conversation can continue',
            }, 4),

        ]
        machine = Steps(bot=bot, steps=steps)
        machine.start()
        ...

    """

    def on_init(self,
                steps=None,
                prefix='steps',
                **kwargs):
        """
        Handles extended initialisation parameters

        :param steps: The steps for this process
        :type steps: list of Step or of dict

        :param prefix: the main keyword for configuration of this machine
        :type prefix: str

        """
        super(Steps, self).on_init(prefix, **kwargs)

        self.steps = []
        if steps:
            for item in steps:
                if isinstance(item, dict):
                    self.steps.append(Step(item, len(self.steps)+1))
                else:
                    self.steps.append(item)

        states = ['begin',
                  'running',
                  'completed',
                  'end']

        transitions = [

            {  # engage on first step as soon as we are ready
             'source': 'begin',
             'target': 'running',
             'condition': self.if_ready,
             'action': self.next_step,
            },

            {  # wait for the underlying machine of this step to complete
             'source': 'running',
             'target': 'completed',
             'condition': self.step_has_completed,
            },

            {  # start next step if there is one and if we are ready for it
             'source': 'completed',
             'target': 'running',
             'condition': self.if_next,
             'action': self.next_step,
            },

            {  # end this state machine
             'source': 'completed',
             'target': 'end',
             'condition': self.if_end,
             'action': self.stop,
            },

        ]

        self.build(states=states,
                   transitions=transitions,
                   initial='begin')

    @property
    def current_step(self):
        """
        Gets current step

        :return: current step, or None
        """
        index = self.get('_index')
        if index is None:
            return None

        return self.steps[index]

    def next_step(self):
        """
        Moves to next step

        :return: current step, or None

        This function loads and runs the next step in the process, if any.
        If all steps have been consumed it returns None.
        """
        if not self.steps or len(self.steps) < 1:
            return None  # no steps have been set

        index = self.get('_index')

        if index is None:
            index = 0
        elif index < len(self.steps)-1:
            index += 1
        else:
            return None  # all steps have been used

        self.set('_index', index)

        step = self.steps[index]

        step.trigger(bot=self.bot)

        return step

    def step_has_completed(self, **kwargs):
        """
        Checks if the machine for this step has finished its job

        :return: False if the machine is still running, True otherwise
        """
        step = self.current_step
        if step is None:
            return True

        machine = step.machine
        if machine is None:
            return True

        return not machine.is_running()

    def if_ready(self, **kwargs):
        """
        Checks if state machine can engage on first step

        To be implemented in sub-class where complex initialization activities
        are required.
        """
        return True

    def if_next(self, **kwargs):
        """
        Checks if next step should be engaged

        :param event: a label of event submitted to the machine
        :type event: str

        This function is used by the state machine for testing the transition
        to the next available step in the process.

        Since this function is an integral part of the state machine itself, it
        should be triggered via a call of the ``step()`` member function.

        For example::

            machine.step(event='next')

        """

        if kwargs.get('event') == 'next':
            return True

        return False

    def if_end(self, **kwargs):
        """
        Checks if all steps have been used

        Since this function is an integral part of the state machine itself, it
        should be triggered via a call of the ``step()`` member function.

        For example::

            machine.step(event='tick')

        """

        index = self.get('_index')

        if index and index >= len(self.steps)-1:
            return True  # all steps have been used

        return False

class Step(object):
    """
    Represents a step in a linear process
    """

    def __init__(self, attributes, index):
        """
        Represents a step in a linear process

        :param attributes: the dict to use
        :type attributes: dict

        :param index: index of this step
        :type index: int

        """
        self.label = attributes.get('label', u'Step {}'.format(index))

        self.message = attributes.get('message',
                                      u'(no description is available)')

        self.content = attributes.get('content', None)

        self.file = attributes.get('file', None)

        self.moderators = attributes.get('moderators', [])
        if isinstance(self.moderators, string_types):
            self.moderators = [self.moderators]

        self.participants = attributes.get('participants', [])
        if isinstance(self.participants, string_types):
            self.participants = [self.participants]

        self.machine = attributes.get('machine', None)

    def say(self, bot):
        """
        Reports on this step

        :param bot: the bit to use
        :type bot: ShellBot

        This function posts to the chat space some information on this step.

        Example::

            step.say(bot)

        """
        bot.say(u"Current state: {} - {}".format(self.label,
                                                 self.message))

    def trigger(self, bot):
        """
        Triggers a step

        :param bot: the bit to use
        :type bot: ShellBot

        This function does everything that is coming with a step:
        - send a message to the chat space,
        - maybe in MarkDown or HTML,
        - maybe with some attachment,
        - add participants and/or moderators to the space,
        - start a state machine

        Example::

            step.trigger(bot)

        """
        bot.say(u"New state: {} - {}".format(self.label,
                                             self.message),
                content=self.content,
                file=self.file)

        bot.add_moderators(self.moderators)
        bot.add_participants(self.participants)

        if self.machine:
            self._step_process = self.machine.start()
<|endoftext|>


<commit_before>import base64
import json
import stat

import anchore_engine.configuration.localconfig
from anchore_engine import utils
from anchore_engine.subsys import logger

def make_image_content_response(content_type, content_data):
    localconfig = anchore_engine.configuration.localconfig.get_config()
    all_content_types = localconfig.get("image_content_types", []) + localconfig.get("image_metadata_types", [])

    if content_type not in all_content_types:
        logger.warn("input content_type (%s) not supported (%s)", content_type, all_content_types)
        return []

    if not content_data:
        logger.warn("empty content data given to format - returning empty result")
        return []

    builder = CONTENT_RESPONSE_BUILDER_DISPATCH.get(content_type, _build_default_response)
    return builder(content_data)

def _build_os_response(content_data):
    response = []
    for package_name, package_info in content_data.items():
        el = {}
        try:
            el["package"] = package_name
            for field in ["license", "origin", "size", "type", "version"]:
                if field in package_info:
                    el[field] = package_info[field]
                else:
                    el[field] = None

                if field == "license":
                    if el[field]:
                        el["licenses"] = el[field].split(" ")
                    else:
                        el["licenses"] = []

            # Special formatting for rpms
            if package_info.get("type", "").lower() in ["rpm"]:
                v = package_info.get("version", None)
                r = package_info.get("release", None)
                if (v and r) and (v.lower() != "n/a") and r.lower() != "n/a":
                    el["version"] = "{}-{}".format(v, r)
        except:
            continue
        response.append(el)
    return response

def _build_npm_response(content_data):
    response = []
    for package in list(content_data.keys()):
        el = {}
        try:
            el["package"] = content_data[package]["name"]
            el["type"] = "NPM"
            el["location"] = package
            el["version"] = content_data[package]["versions"][0]
            el["origin"] = ",".join(content_data[package]["origins"]) or "Unknown"
            el["license"] = " ".join(content_data[package]["lics"]) or "Unknown"
            el["licenses"] = content_data[package]["lics"] or ["Unknown"]
        except:
            continue
        response.append(el)
    return response

def _build_gem_response(content_data):
    response = []
    for package in list(content_data.keys()):
        el = {}
        try:
            el["package"] = content_data[package]["name"]
            el["type"] = "GEM"
            el["location"] = package
            el["version"] = content_data[package]["versions"][0]
            el["origin"] = ",".join(content_data[package]["origins"]) or "Unknown"
            el["license"] = " ".join(content_data[package]["lics"]) or "Unknown"
            el["licenses"] = content_data[package]["lics"] or ["Unknown"]
        except:
            continue
        response.append(el)
    return response

def _build_python_response(content_data):
    response = []
    for package in list(content_data.keys()):
        el = {}
        try:
            el["package"] = content_data[package]["name"]
            el["type"] = "PYTHON"
            el["location"] = content_data[package]["location"]
            el["version"] = content_data[package]["version"]
            el["origin"] = content_data[package]["origin"] or "Unknown"
            el["license"] = content_data[package]["license"] or "Unknown"
            el["licenses"] = content_data[package]["license"].split(" ") or ["Unknown"]
        except:
            continue
        response.append(el)
    return response

def _build_java_response(content_data):
    response = []
    for package in list(content_data.keys()):
        el = {}
        try:
            el["package"] = content_data[package]["name"]
            el["type"] = content_data[package]["type"].upper()
            el["location"] = content_data[package]["location"]
            el["specification-version"] = content_data[package]["specification-version"]
            el["implementation-version"] = content_data[package]["implementation-version"]
            el["maven-version"] = content_data[package]["maven-version"]
            el["origin"] = content_data[package]["origin"] or "Unknown"
        except:
            continue
        response.append(el)
    return response

def _build_files_response(content_data):
    response = []
    elmap = {
        "linkdst": "linkdest",
        "size": "size",
        "mode": "mode",
        "sha256": "sha256",
        "type": "type",
        "uid": "uid",
        "gid": "gid",
    }
    for filename in list(content_data.keys()):
        el = {}
        try:
            el["filename"] = filename
            for elkey in list(elmap.keys()):
                try:
                    el[elmap[elkey]] = content_data[filename][elkey]
                except:
                    el[elmap[elkey]] = None

            # special formatting
            el["mode"] = format(stat.S_IMODE(el["mode"]), "05o")
            if el["sha256"] == "DIRECTORY_OR_OTHER":
                el["sha256"] = None
        except:
            continue
        response.append(el)
    return response

def _safe_base64_encode(data_provider):
    try:
        return utils.ensure_str(
            base64.encodebytes(utils.ensure_bytes(data_provider()))
        )
    except Exception as err:
        logger.warn("could not base64 encode content - exception: %s", err)
    return ""

def _build_docker_history_response(content_data):
    return _safe_base64_encode(lambda: json.dumps(content_data))

def _build_dockerfile_response(content_data):
    return _safe_base64_encode(lambda: content_data)

def _build_manifest_response(content_data):
   return _safe_base64_encode(lambda: content_data)

def _build_default_response(content_data):
    response = []
    try:
        for package in list(content_data.keys()):
            el = {}
            try:
                el["package"] = content_data[package]["name"]
                el["type"] = content_data[package]["type"].upper()
                el["location"] = content_data[package].get("location", None) or "Unknown"
                el["version"] = content_data[package].get("version", None) or "Unknown"
                el["origin"] = content_data[package].get("origin", None) or "Unknown"
                el["license"] = content_data[package].get("license", None) or "Unknown"
                el["licenses"] = content_data[package].get("license", "Unknown").split(" ")
            except Exception as err:
                continue
            response.append(el)
        if not response:
            raise Exception("empty return list after generic element parse")
    except Exception as err:
        logger.debug("couldn't parse any generic package elements, returning raw content_data - exception: %s", err)
        response = content_data

    return response

CONTENT_RESPONSE_BUILDER_DISPATCH = {
    "os": _build_os_response,
    "npm": _build_npm_response,
    "gem": _build_gem_response,
    "python": _build_python_response,
    "java": _build_java_response,
    "files": _build_files_response,
    "docker_history": _build_docker_history_response,
    "dockerfile": _build_dockerfile_response,
    "manifest": _build_manifest_response,
}
<commit_msg>Adds release to version string on package output for any os package that has a valid release component. Fixes #504<commit_after>import base64
import json
import stat

import anchore_engine.configuration.localconfig
from anchore_engine import utils
from anchore_engine.subsys import logger
from anchore_engine.common import os_package_types

def make_image_content_response(content_type, content_data):
    localconfig = anchore_engine.configuration.localconfig.get_config()
    all_content_types = localconfig.get("image_content_types", []) + localconfig.get("image_metadata_types", [])

    if content_type not in all_content_types:
        logger.warn("input content_type (%s) not supported (%s)", content_type, all_content_types)
        return []

    if not content_data:
        logger.warn("empty content data given to format - returning empty result")
        return []

    builder = CONTENT_RESPONSE_BUILDER_DISPATCH.get(content_type, _build_default_response)
    return builder(content_data)

def _build_os_response(content_data):
    response = []
    for package_name, package_info in content_data.items():
        el = {}
        try:
            el["package"] = package_name
            for field in ["license", "origin", "size", "type", "version"]:
                if field in package_info:
                    el[field] = package_info[field]
                else:
                    el[field] = None

                if field == "license":
                    if el[field]:
                        el["licenses"] = el[field].split(" ")
                    else:
                        el["licenses"] = []

            # Special formatting for os packages. Ensure that if there is a release field it is added to the version string
            if package_info.get("type", "").lower() in os_package_types:
                v = package_info.get("version", None)
                r = package_info.get("release", None)
                if (v and r) and (v.lower() != "n/a") and r.lower() != "n/a":
                    el["version"] = "{}-{}".format(v, r)
        except:
            continue
        response.append(el)
    return response

def _build_npm_response(content_data):
    response = []
    for package in list(content_data.keys()):
        el = {}
        try:
            el["package"] = content_data[package]["name"]
            el["type"] = "NPM"
            el["location"] = package
            el["version"] = content_data[package]["versions"][0]
            el["origin"] = ",".join(content_data[package]["origins"]) or "Unknown"
            el["license"] = " ".join(content_data[package]["lics"]) or "Unknown"
            el["licenses"] = content_data[package]["lics"] or ["Unknown"]
        except:
            continue
        response.append(el)
    return response

def _build_gem_response(content_data):
    response = []
    for package in list(content_data.keys()):
        el = {}
        try:
            el["package"] = content_data[package]["name"]
            el["type"] = "GEM"
            el["location"] = package
            el["version"] = content_data[package]["versions"][0]
            el["origin"] = ",".join(content_data[package]["origins"]) or "Unknown"
            el["license"] = " ".join(content_data[package]["lics"]) or "Unknown"
            el["licenses"] = content_data[package]["lics"] or ["Unknown"]
        except:
            continue
        response.append(el)
    return response

def _build_python_response(content_data):
    response = []
    for package in list(content_data.keys()):
        el = {}
        try:
            el["package"] = content_data[package]["name"]
            el["type"] = "PYTHON"
            el["location"] = content_data[package]["location"]
            el["version"] = content_data[package]["version"]
            el["origin"] = content_data[package]["origin"] or "Unknown"
            el["license"] = content_data[package]["license"] or "Unknown"
            el["licenses"] = content_data[package]["license"].split(" ") or ["Unknown"]
        except:
            continue
        response.append(el)
    return response

def _build_java_response(content_data):
    response = []
    for package in list(content_data.keys()):
        el = {}
        try:
            el["package"] = content_data[package]["name"]
            el["type"] = content_data[package]["type"].upper()
            el["location"] = content_data[package]["location"]
            el["specification-version"] = content_data[package]["specification-version"]
            el["implementation-version"] = content_data[package]["implementation-version"]
            el["maven-version"] = content_data[package]["maven-version"]
            el["origin"] = content_data[package]["origin"] or "Unknown"
        except:
            continue
        response.append(el)
    return response

def _build_files_response(content_data):
    response = []
    elmap = {
        "linkdst": "linkdest",
        "size": "size",
        "mode": "mode",
        "sha256": "sha256",
        "type": "type",
        "uid": "uid",
        "gid": "gid",
    }
    for filename in list(content_data.keys()):
        el = {}
        try:
            el["filename"] = filename
            for elkey in list(elmap.keys()):
                try:
                    el[elmap[elkey]] = content_data[filename][elkey]
                except:
                    el[elmap[elkey]] = None

            # special formatting
            el["mode"] = format(stat.S_IMODE(el["mode"]), "05o")
            if el["sha256"] == "DIRECTORY_OR_OTHER":
                el["sha256"] = None
        except:
            continue
        response.append(el)
    return response

def _safe_base64_encode(data_provider):
    try:
        return utils.ensure_str(
            base64.encodebytes(utils.ensure_bytes(data_provider()))
        )
    except Exception as err:
        logger.warn("could not base64 encode content - exception: %s", err)
    return ""

def _build_docker_history_response(content_data):
    return _safe_base64_encode(lambda: json.dumps(content_data))

def _build_dockerfile_response(content_data):
    return _safe_base64_encode(lambda: content_data)

def _build_manifest_response(content_data):
   return _safe_base64_encode(lambda: content_data)

def _build_default_response(content_data):
    response = []
    try:
        for package in list(content_data.keys()):
            el = {}
            try:
                el["package"] = content_data[package]["name"]
                el["type"] = content_data[package]["type"].upper()
                el["location"] = content_data[package].get("location", None) or "Unknown"
                el["version"] = content_data[package].get("version", None) or "Unknown"
                el["origin"] = content_data[package].get("origin", None) or "Unknown"
                el["license"] = content_data[package].get("license", None) or "Unknown"
                el["licenses"] = content_data[package].get("license", "Unknown").split(" ")
            except Exception as err:
                continue
            response.append(el)
        if not response:
            raise Exception("empty return list after generic element parse")
    except Exception as err:
        logger.debug("couldn't parse any generic package elements, returning raw content_data - exception: %s", err)
        response = content_data

    return response

CONTENT_RESPONSE_BUILDER_DISPATCH = {
    "os": _build_os_response,
    "npm": _build_npm_response,
    "gem": _build_gem_response,
    "python": _build_python_response,
    "java": _build_java_response,
    "files": _build_files_response,
    "docker_history": _build_docker_history_response,
    "dockerfile": _build_dockerfile_response,
    "manifest": _build_manifest_response,
}
<|endoftext|>