<commit_before># Copyright 2013, 2014, 2015 Reahl Software Services (Pty) Ltd. All rights reserved.
#
#    This file is part of Reahl.
#
#    Reahl is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation; version 3 of the License.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""Classes that aid in dealing with Eggs and setting them up."""

from __future__ import print_function, unicode_literals, absolute_import, division
import os
import os.path
import logging

from pkg_resources import Requirement
from pkg_resources import iter_entry_points
from pkg_resources import require
from pkg_resources import resource_isdir
from pkg_resources import resource_listdir
from pkg_resources import working_set

from reahl.component.decorators import memoized

class EntryPointKeyEncodedAttachmentName(object):

    order_delimeter = '-'

    def __init__(self, encoded_string=None, path=None, order=None):
        self.encoded_string = encoded_string
        self._order = order
        self._path = path

    @property
    def is_encoded(self):
        return self.encoded_string is not None

    @property
    def order(self):
        if self.is_encoded:
            return int(self.encoded_string.split(self.order_delimeter)[0])
        return self._order

    @property
    def encoded_path(self):
        if self.is_encoded:
            return '-'.join(self.encoded_string.split(self.order_delimeter)[1:])
        return self._path.replace('/', '+')

    @property
    def decoded_path(self):
        if self.is_encoded:
            return self.encoded_path.replace('+', '/')
        return self._path

    def as_encoded_key(self):
        if self.is_encoded:
            return self.encoded_string
        return '%s%s%s' % (self.order, self.order_delimeter, self.encoded_path)

class Attachment(object):
    def __init__(self, filename, label):
        self.filename = filename
        self.label = label

class ReahlEgg(object):
    interface_cache = {}

    def __init__(self, distribution):
        self.distribution = distribution

    @property
    def name(self):
        return self.distribution.key

    @property
    def version(self):
        return self.distribution.version

    @property
    def configuration_spec(self):
        entry_point_dict = self.distribution.get_entry_map().get('reahl.configspec', {})
        entry_point = entry_point_dict.get('config', None)
        return entry_point.load() if entry_point else None

    def read_config(self, config):
        if self.configuration_spec:
            config.read(self.configuration_spec)

    def validate_config(self, config):
        if self.configuration_spec:
            config.validate_required(self.configuration_spec)

    def list_config(self, config):
        if self.configuration_spec:
            return config.list_required(self.configuration_spec)
        return []

    def get_persisted_classes_in_order(self, orm_control):
        return self.get_ordered_classes_exported_on('reahl.persistlist')

    def find_attachments(self, label):
        entry_point = 'reahl.attachments.%s' % label
        entry_point_dict = self.distribution.get_entry_map().get(entry_point, {})
        names = [EntryPointKeyEncodedAttachmentName(encoded_string=name) for name in entry_point_dict.keys()]
        ordered_names = [name.decoded_path for name in sorted(names, key=lambda name: name.order)]
        return [Attachment(i, label) for i in ordered_names]

    @property
    def migrations_in_order(self):
        return self.get_ordered_classes_exported_on('reahl.migratelist')

    def compute_migrations(self, current_schema_version):
        return [cls for cls in self.migrations_in_order
                if cls.is_applicable(current_schema_version, self.version)]

    def get_ordered_classes_exported_on(self, entry_point):
        entry_point_dict = self.distribution.get_entry_map().get(entry_point, {})
        found_eps = set()
        for ep in entry_point_dict.values():
            if ep in found_eps:
                raise AssertionError('%s is listed twice' % ep)
            found_eps.add(ep)
        return [entry.load() for order, entry in sorted([(int(order), e) for order, e in entry_point_dict.items()])]

    def get_classes_exported_on(self, entry_point):
        entry_point_dict = self.distribution.get_entry_map().get(entry_point, {})
        return [entry.load() for name, entry in entry_point_dict.items()]

    @property
    def translation_pot_filename(self):
        entry_point_dict = self.distribution.get_entry_map().get('reahl.translations')
        translations_package_name = entry_point_dict[self.name].module_name
        translations_file_path = translations_package_name.replace('.', '/')
        return self.distribution.get_resource_filename(self.distribution, '%s/%s' % (translations_file_path, self.name))

    @property
    def scheduled_jobs(self):
        return self.get_classes_exported_on('reahl.scheduled_jobs')

    def do_daily_maintenance(self):
        for job in self.scheduled_jobs:
            job()

    def get_roles_to_add(self):
        "returns a list of role classes"
        return []

    @property
    def locale_dirname(self):
        return self.distribution.get_resource_filename(working_set, 'i18n')

    @classmethod
    def get_egg_internal_path_for(cls, translations_entry_point):
        module = translations_entry_point.load()
        dir_or_egg_name = translations_entry_point.dist.location.split(os.sep)[-1]
        paths = [p for p in module.__path__ if p.find('%s%s' % (dir_or_egg_name, os.path.sep)) > 0]
        paths = [p for p in paths if not p.startswith(os.path.join(translations_entry_point.dist.location, '.'))]
        unique_paths = [p.split('%s/' % dir_or_egg_name)[-1] for p in paths]
        unique_paths = set([p for p in unique_paths if '.egg' not in p])
        assert len(unique_paths) <=1, \
            'Only one translations package per component is allowed, found %s for %s' % (paths, translations_entry_point.dist)
        assert len(unique_paths) >0, \
            'No translations found for %s, did you specify a translations package and forget to add locales in there?' % translations_entry_point.dist
        return unique_paths.pop()

    @memoized
    @classmethod
    def get_languages_supported_by_all(cls, root_egg):
        egg_interfaces = cls.get_all_relevant_interfaces(root_egg)
        default_languages = ['en_gb']
        if not egg_interfaces:
            return default_languages

        domains_in_use = [e.name for e in egg_interfaces]

        languages_for_eggs = {}
        for translation_entry_point in iter_entry_points('reahl.translations'):
            requirement = translation_entry_point.dist.as_requirement()
            egg_internal_path = cls.get_egg_internal_path_for(translation_entry_point)

            if resource_isdir(requirement, egg_internal_path):
                languages = [d for d in resource_listdir(requirement, egg_internal_path)
                             if (resource_isdir(requirement, '%s/%s' % (egg_internal_path, d)) and not d.startswith('__'))]
            else:
                logging.error('Translations of %s not found in %s' % (requirement, egg_internal_path))
                languages = []

            for language in languages:
                language_path = '%s/%s/LC_MESSAGES' % (egg_internal_path, language)
                domains = [d[:-3] for d in resource_listdir(requirement, language_path) if d.endswith('.mo')]
                for domain in domains:
                    if domain in domains_in_use:
                        languages = languages_for_eggs.setdefault(domain, set())
                        languages.add(language)

        if not languages_for_eggs.values():
            return default_languages
        languages = (list(languages_for_eggs.values()))[0].intersection(*languages_for_eggs.values())
        languages.update(default_languages)
        return list(languages)

    @classmethod
    def do_daily_maintenance_for_egg(cls, root_egg):
        eggs_in_order = ReahlEgg.get_all_relevant_interfaces(root_egg)
        for egg in eggs_in_order:
            if isinstance(egg, ReahlEgg):
                egg.do_daily_maintenance()

    @classmethod
    def clear_cache(cls):
        cls.interface_cache.clear()

    @classmethod
    def get_all_relevant_interfaces(cls, main_egg):
        # (We cache the result of this computation, since it is
        #  quite expensive and is called often in the tests.)
        interfaces = cls.interface_cache.get(main_egg, None)
        if interfaces:
            return interfaces
        interfaces = cls.compute_all_relevant_interfaces(main_egg)
        cls.interface_cache[main_egg] = interfaces
        return interfaces

    @classmethod
    def topological_sort(cls, distributions):
    # Algorithm from: http://www.logarithmic.net/pfh-files/blog/01208083168/sort.py
    # See also: http://en.wikipedia.org/wiki/Topological_sorting

        graph = {}
        for dist in distributions:
            dependencies = [working_set.find(i) for i in dist.requires()]
            my_requirements =  dist.requires()
            #we want the subset of stuff in the basket we actually depend on, not just the basket itself
            basket_requirements = [i for i in my_requirements
                                   if i.extras]
            for basket in basket_requirements:
                dependencies.extend([working_set.find(Requirement.parse(i)) for i in basket.extras])

            graph[dist] = dependencies

        count = { }
        for node in graph:
            count[node] = 0
        for node in graph:
            for successor in graph[node]:
                count[successor] += 1

        ready = [ node for node in graph if count[node] == 0 ]

        result = []
        while ready:
            node = ready.pop(-1)
            result.append(node)

            for successor in graph[node]:
                count[successor] -= 1
                if count[successor] == 0:
                    ready.append(successor)

        assert set(distributions) == set(result)
        return result

    @classmethod
    def get_eggs_for(cls, main_egg):
        distributions = require(main_egg)
        return list(set(distributions)) # To get rid of duplicates

    @classmethod
    def compute_ordered_dependent_distributions(cls, main_egg):
        return cls.topological_sort(cls.get_eggs_for(main_egg))

    @classmethod
    def compute_all_relevant_interfaces(cls, main_egg):
        interfaces = []

        for i in cls.compute_ordered_dependent_distributions(main_egg):
            entry_map = i.get_entry_map('reahl.eggs')
            if entry_map:
                classes = list(entry_map.values())
                assert len(classes) == 1, 'Only one eggdeb class per egg allowed'
                interfaces.append(classes[0].load()(i))

        return interfaces

<commit_msg>implemented a fancier topological sort that detects and reports circular dependencies<commit_after># Copyright 2013, 2014, 2015 Reahl Software Services (Pty) Ltd. All rights reserved.
#
#    This file is part of Reahl.
#
#    Reahl is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation; version 3 of the License.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""Classes that aid in dealing with Eggs and setting them up."""

from __future__ import print_function, unicode_literals, absolute_import, division
import os
import os.path
import logging

from pkg_resources import Requirement
from pkg_resources import iter_entry_points
from pkg_resources import require
from pkg_resources import resource_isdir
from pkg_resources import resource_listdir
from pkg_resources import working_set

from reahl.component.decorators import memoized

class EntryPointKeyEncodedAttachmentName(object):

    order_delimeter = '-'

    def __init__(self, encoded_string=None, path=None, order=None):
        self.encoded_string = encoded_string
        self._order = order
        self._path = path

    @property
    def is_encoded(self):
        return self.encoded_string is not None

    @property
    def order(self):
        if self.is_encoded:
            return int(self.encoded_string.split(self.order_delimeter)[0])
        return self._order

    @property
    def encoded_path(self):
        if self.is_encoded:
            return '-'.join(self.encoded_string.split(self.order_delimeter)[1:])
        return self._path.replace('/', '+')

    @property
    def decoded_path(self):
        if self.is_encoded:
            return self.encoded_path.replace('+', '/')
        return self._path

    def as_encoded_key(self):
        if self.is_encoded:
            return self.encoded_string
        return '%s%s%s' % (self.order, self.order_delimeter, self.encoded_path)

class Attachment(object):
    def __init__(self, filename, label):
        self.filename = filename
        self.label = label

class NoDependencyPathFound(Exception):
    def __init__(self, from_vertex, to_vertex):
        self.from_vertex = from_vertex
        self.to_vertex = to_vertex

    def str(self):
        return 'No dependency path found from %s to %s' % (self.from_vertex, to_vertex)

class CircularDependencyDetected(Exception):
    def __init__(self, cycle):
        self.cycle = cycle

    def __str__(self):
        return ' -> '.join([str(i) for i in self.cycle])

class DepthFirstSearch(object):
    def __init__(self, graph):
        self.graph = graph
        self.discovered = {}
        self.entered = {}
        self.exited = {}
        self.count = 0
        self.topological_order = []
        self.parents = {}

    def path(self, from_vertex, to_vertex):
        path = []
        i = to_vertex
        path.append(i)
        while i in self.parents and i is not from_vertex:
            i = self.parents[i]
            path.append(i)
        if i is not from_vertex:
            raise NoDependencyPathFound(from_vertex, to_vertex)
        path.reverse()
        return path

    def search(self, from_vertex):
        self.discovered[from_vertex] = True
        self.entered[from_vertex] = self.count
        self.count += 1
        for i in self.graph[from_vertex]:
            if i not in self.discovered:
                self.parents[i] = from_vertex
                self.search(i)
            elif self.entered[i] < self.entered[from_vertex] and i not in self.exited:
                raise CircularDependencyDetected(self.path(i, from_vertex)+[i])
            elif i not in self.parents:
                self.parents[i] = from_vertex

        self.exited[from_vertex] = self.count
        self.count += 1
        self.topological_order.append(from_vertex)

    def topological_sort(self):
        for i in self.graph.keys():
            if i not in self.discovered:
                self.search(i)
        return reversed(self.topological_order)

class ReahlEgg(object):
    interface_cache = {}

    def __init__(self, distribution):
        self.distribution = distribution

    @property
    def name(self):
        return self.distribution.key

    @property
    def version(self):
        return self.distribution.version

    @property
    def configuration_spec(self):
        entry_point_dict = self.distribution.get_entry_map().get('reahl.configspec', {})
        entry_point = entry_point_dict.get('config', None)
        return entry_point.load() if entry_point else None

    def read_config(self, config):
        if self.configuration_spec:
            config.read(self.configuration_spec)

    def validate_config(self, config):
        if self.configuration_spec:
            config.validate_required(self.configuration_spec)

    def list_config(self, config):
        if self.configuration_spec:
            return config.list_required(self.configuration_spec)
        return []

    def get_persisted_classes_in_order(self, orm_control):
        return self.get_ordered_classes_exported_on('reahl.persistlist')

    def find_attachments(self, label):
        entry_point = 'reahl.attachments.%s' % label
        entry_point_dict = self.distribution.get_entry_map().get(entry_point, {})
        names = [EntryPointKeyEncodedAttachmentName(encoded_string=name) for name in entry_point_dict.keys()]
        ordered_names = [name.decoded_path for name in sorted(names, key=lambda name: name.order)]
        return [Attachment(i, label) for i in ordered_names]

    @property
    def migrations_in_order(self):
        return self.get_ordered_classes_exported_on('reahl.migratelist')

    def compute_migrations(self, current_schema_version):
        return [cls for cls in self.migrations_in_order
                if cls.is_applicable(current_schema_version, self.version)]

    def get_ordered_classes_exported_on(self, entry_point):
        entry_point_dict = self.distribution.get_entry_map().get(entry_point, {})
        found_eps = set()
        for ep in entry_point_dict.values():
            if ep in found_eps:
                raise AssertionError('%s is listed twice' % ep)
            found_eps.add(ep)
        return [entry.load() for order, entry in sorted([(int(order), e) for order, e in entry_point_dict.items()])]

    def get_classes_exported_on(self, entry_point):
        entry_point_dict = self.distribution.get_entry_map().get(entry_point, {})
        return [entry.load() for name, entry in entry_point_dict.items()]

    @property
    def translation_pot_filename(self):
        entry_point_dict = self.distribution.get_entry_map().get('reahl.translations')
        translations_package_name = entry_point_dict[self.name].module_name
        translations_file_path = translations_package_name.replace('.', '/')
        return self.distribution.get_resource_filename(self.distribution, '%s/%s' % (translations_file_path, self.name))

    @property
    def scheduled_jobs(self):
        return self.get_classes_exported_on('reahl.scheduled_jobs')

    def do_daily_maintenance(self):
        for job in self.scheduled_jobs:
            job()

    def get_roles_to_add(self):
        "returns a list of role classes"
        return []

    @property
    def locale_dirname(self):
        return self.distribution.get_resource_filename(working_set, 'i18n')

    @classmethod
    def get_egg_internal_path_for(cls, translations_entry_point):
        module = translations_entry_point.load()
        dir_or_egg_name = translations_entry_point.dist.location.split(os.sep)[-1]
        paths = [p for p in module.__path__ if p.find('%s%s' % (dir_or_egg_name, os.path.sep)) > 0]
        paths = [p for p in paths if not p.startswith(os.path.join(translations_entry_point.dist.location, '.'))]
        unique_paths = [p.split('%s/' % dir_or_egg_name)[-1] for p in paths]
        unique_paths = set([p for p in unique_paths if '.egg' not in p])
        assert len(unique_paths) <=1, \
            'Only one translations package per component is allowed, found %s for %s' % (paths, translations_entry_point.dist)
        assert len(unique_paths) >0, \
            'No translations found for %s, did you specify a translations package and forget to add locales in there?' % translations_entry_point.dist
        return unique_paths.pop()

    @memoized
    @classmethod
    def get_languages_supported_by_all(cls, root_egg):
        egg_interfaces = cls.get_all_relevant_interfaces(root_egg)
        default_languages = ['en_gb']
        if not egg_interfaces:
            return default_languages

        domains_in_use = [e.name for e in egg_interfaces]

        languages_for_eggs = {}
        for translation_entry_point in iter_entry_points('reahl.translations'):
            requirement = translation_entry_point.dist.as_requirement()
            egg_internal_path = cls.get_egg_internal_path_for(translation_entry_point)

            if resource_isdir(requirement, egg_internal_path):
                languages = [d for d in resource_listdir(requirement, egg_internal_path)
                             if (resource_isdir(requirement, '%s/%s' % (egg_internal_path, d)) and not d.startswith('__'))]
            else:
                logging.error('Translations of %s not found in %s' % (requirement, egg_internal_path))
                languages = []

            for language in languages:
                language_path = '%s/%s/LC_MESSAGES' % (egg_internal_path, language)
                domains = [d[:-3] for d in resource_listdir(requirement, language_path) if d.endswith('.mo')]
                for domain in domains:
                    if domain in domains_in_use:
                        languages = languages_for_eggs.setdefault(domain, set())
                        languages.add(language)

        if not languages_for_eggs.values():
            return default_languages
        languages = (list(languages_for_eggs.values()))[0].intersection(*languages_for_eggs.values())
        languages.update(default_languages)
        return list(languages)

    @classmethod
    def do_daily_maintenance_for_egg(cls, root_egg):
        eggs_in_order = ReahlEgg.get_all_relevant_interfaces(root_egg)
        for egg in eggs_in_order:
            if isinstance(egg, ReahlEgg):
                egg.do_daily_maintenance()

    @classmethod
    def clear_cache(cls):
        cls.interface_cache.clear()

    @classmethod
    def get_all_relevant_interfaces(cls, main_egg):
        # (We cache the result of this computation, since it is
        #  quite expensive and is called often in the tests.)
        interfaces = cls.interface_cache.get(main_egg, None)
        if interfaces:
            return interfaces
        interfaces = cls.compute_all_relevant_interfaces(main_egg)
        cls.interface_cache[main_egg] = interfaces
        return interfaces

    @classmethod
    def topological_sort(cls, distributions):
        graph = {}
        for dist in distributions:
            dependencies = [working_set.find(i) for i in dist.requires()]
            my_requirements =  dist.requires()
            #we want the subset of stuff in the basket we actually depend on, not just the basket itself
            basket_requirements = [i for i in my_requirements
                                   if i.extras]
            for basket in basket_requirements:
                dependencies.extend([working_set.find(Requirement.parse(i)) for i in basket.extras])

            graph[dist] = dependencies

        return DepthFirstSearch(graph).topological_sort()

    @classmethod
    def get_eggs_for(cls, main_egg):
        distributions = require(main_egg)
        return list(set(distributions)) # To get rid of duplicates

    @classmethod
    def compute_ordered_dependent_distributions(cls, main_egg):
        return cls.topological_sort(cls.get_eggs_for(main_egg))

    @classmethod
    def compute_all_relevant_interfaces(cls, main_egg):
        interfaces = []

        for i in cls.compute_ordered_dependent_distributions(main_egg):
            entry_map = i.get_entry_map('reahl.eggs')
            if entry_map:
                classes = list(entry_map.values())
                assert len(classes) == 1, 'Only one eggdeb class per egg allowed'
                interfaces.append(classes[0].load()(i))

        return interfaces

<|endoftext|>


<commit_before># ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# There's no good way in cosmos to create a tag
# based on output or to conditionally run a process
# This is a wrapper to run the umi utilities.
#
# Run a program (SeqPrep) for a directory in batches
# This is a helper script for Martin Aryee's
# scripts to demultiplex Illumina sequencing
# reads with sample specific and molecule
# specific tags.
# https://github.com/aryeelab/umi/wiki
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
__author__ = 'Allison MacLeay'

import sys
import os
import argparse
import time

# -----------------------------------------
# Get bash command for specified
# file name prefix
# -----------------------------------------
def get_cmd(group, prog, params):
    path = '.'
    outdir = ''
    cmd = ''
    job_group = ''
    a1 = ''
    a2 = ''
    if 'path' in params.keys():
        path = params['path']
    if 'out' in params.keys():
        outdir = params['out']
        if (outdir[0] == '/'):
            outdir = outdir[1:]
        else:
            outdir = os.path.abspath(outdir)[1:]
        if (outdir[-1] != '/'):
            outdir = outdir + '/'
    if 'job_group' in params.keys():
        job_group = params['job_group']
    if 'a1' in params.keys():
        a1 = params['a1']
    if 'a1' in params.keys():
        a2 = params['a2']
    fastq_r1 = os.path.join(path, group + '.r1.fastq.gz')
    fastq_r2 = os.path.join(path ,group + '.r2.fastq.gz')
    out1 = '' + group + '_trim.r1.fastq.gz'
    out2 = '' + group + '_trim.r2.fastq.gz'
    cmd = (cmd + """ \
    {prog} -f {fastq_r1} \
-r {fastq_r2} \
-1 {outdir}{out1} \
-2 {outdir}{out2} \
-A {adapter1} \
-B {adapter2} -z \
        """ ).format(
        prog=prog,
        fastq_r1=fastq_r1,
        fastq_r2=fastq_r2,
        outdir='/' + outdir,
        out1=out1,
        out2=out2,
        adapter1=a1,
        adapter2=a2,
    )
    return cmd

# -----------------------------------------
# Return all unique file prefixes
# -----------------------------------------
def get_names(dir):
    files = next(os.walk(dir))[2]
    files = map(lambda x: x.split('.')[0], files)
    dfiles = set(files)
    return dfiles

# -----------------------------------------
# Delay completion of script until all
# files are written
#-----------------------------------------
def check_done(group_id,ct):
    start = time.time()
    timeout = (24 * 60 * 60)  # 24 hours
    done = 0
    pr_ct = 0

    while done==0:
        pr_ct+=1
        if (time.time()-start)>timeout:
            print 'Job timed out'
            done=1
        else:
            time.sleep(10)
            if pr_ct % 6 == 0:
                print 'checking for job completion after waiting %d seconds' % (time.time()-start)
            if are_jobs_done(group_id,ct):
                print 'No running jobs found in group ' + group_id
                done=1
    return

def wait_while_job_running(jobname,maxtm):
    done = is_job_done(jobname)
    tmst = time.time()
    duration = 0
    while done == False and (duration < maxtm):
        print "Waiting for job " + jobname + " to complete \n" + str(duration) + " seconds have elapsed."
        time.sleep(10)
        duration = time.time() - tmst
        done = is_job_done(jobname)
    return

# -----------------------------------------
# Zip it
#-----------------------------------------
def gzip_if_not(folder,bsub_off,out,err,group_id):
    zipping = False
    zipped = {}
    for file in next(os.walk(folder))[2]:
        parts = file.split('.')
        if parts[-1] != 'gz':
             if parts[0] not in zipped.keys():
                 zipped[parts[0]]=1
                 zipping = True
                 gzip(os.path.join(folder,parts[0] + '*'),'gz'+group_id+parts[0],bsub_off,out,err)
    return zipping

def gzip(filename,jobname,bsub_off,out,err):
    cmd = 'gzip ' + filename
    if not bsub_off:
       cmd = bsub_cmd(out,err) + ' -J ' + jobname + ' ' + cmd
    os.system(cmd)
    return
# -----------------------------------------
# LSF utilities
#-----------------------------------------
def bsub_cmd(out,err):
    txt = 'bsub -q medium -u am282 -o ' + out + ' -e ' + err + ' '
    return txt

def is_job_done(jobname):
    done = True
    ct=0
    jobs = os.popen("bjobs -J " + jobname).read().split('\n')
    for line in jobs:
        ct+=1
        if ct == 1:
            continue
        status = get_job_status(line)
        if status == 'RUN':
            done = False
        if status == 'PEND':
            done = False
    return done

def are_jobs_done(group,lsf_ct):
    jobs=os.popen("bjobs -g " + group).read().split('\n')
    ct=0
    run_ct=0
    group_status=True
    for line in jobs:
        ct+=1
        if ct==1:
            continue
        status=get_job_status(line)
        if status== 'RUN':
            group_status=False
            run_ct+=1
        if status== 'PEND':
            group_status=False
            run_ct+=1
    if( (ct-1) != lsf_ct):
        print 'Different number of jobs passed in ('+str(lsf_ct)+') and recovered (' + str(ct-1) + ')'
    print "" + str(run_ct) + " jobs are currently running or pending in group " + group
    return group_status

def get_job_status(line):
    status=''
    status_arr=line.strip().split()
    if len(status_arr)>2:
        status=status_arr[2]
    else:
        status=''
    return status

def get_group_id(group):
    i = os.popen("bjgroup | grep "+ group).read()
    i = i.split('\n')
    num=''
    max=0
    for line in i:
        group_arr=line.split()
        if len(group_arr)>1:
            group_name=group_arr[0]
            if are_jobs_done(group_name,0):
                # clean old jobs
                os.system("bgdel " + group_name)
            num=line.strip().split('_')[0].split('/')[-1]
        if str(num)=='':
            num=0
        if type(num) is str:
                if not num.isdigit():
                    num=0
        if int(num) > max:
            max = int(num)
    print 'last id was ' + str(max)
    return group + '/' + str(max+1) + "_trim"

#-----------------------------------------
#	MAIN
# run a program (SeqPrep) for all files in a directory
# that have the same prefix
#-----------------------------------------
if __name__ == '__main__':
    parser = argparse.ArgumentParser(
            description="Run command for removing adapter sequecnes in batches of similarly prefixed names.")
    parser.add_argument('--dir', default='.', help='directory containing output of umi demultiplex')
    parser.add_argument('--script', default='./SeqPrep', help='SeqPrep absolute path.  default is SeqPrep in current directory')
    parser.add_argument('--a1', required=True, help='Adapter 1')
    parser.add_argument('--a2', required=True, help='Adapter 2')
    parser.add_argument('--out', default='tagout', help='directory to deposit output files')
    parser.add_argument('--log', default='batch_log', help='directory to deposit bsub log files')
    parser.add_argument('--bsub_off', action='store_true', help='turn bsub off to test on systems without lsf')
    #parser.add_argument('--undet',  action='store_false', help='include reads less than parameter set my min reads.  Default will skip files named undetermined')
    args = parser.parse_args()

    p = {}
    lsf_group=''
    lsf_group_cmd=''
    if hasattr(args, 'dir'):
        p['path'] = args.dir
    if hasattr(args, 'out'):
        p['out'] = args.out
        os.system('mkdir -p ' + args.out)
    if hasattr(args, 'log'):
        os.system('mkdir -p ' + args.log)
        os.system('ls ' + p['path'] + ' >> ' + args.log + '/ls_inputdir.txt')
    if hasattr(args, 'a1'):
        p['a1']=args.a1
    if hasattr(args, 'a2'):
        p['a2']=args.a2
    lsf_out = ''
    lsf_err = ''
    if not args.bsub_off:
        lsf_out = os.path.join(args.log, 'lsf_out.log')
        lsf_err = os.path.join(args.log, 'lsf_err.log')
    f = get_names(args.dir)
    if len(f) < 1:
        print "Error: No file prefixes were found in " + args.dir + "."
    count_lsf = 0
    group_id = 'groupID'
    if not args.bsub_off:
        lsf_group = get_group_id("/demux")
        group_id = lsf_group.split("/demux/")[1]
        lsf_group_cmd=' -g ' + lsf_group
    needed_zip = gzip_if_not(args.dir,args.bsub_off,lsf_out,lsf_err,group_id)
    for tag in f:
        if (tag.find('undetermined') > -1 ):
            # skip undeterminded for now
            cmd = 'echo skipping undetermined files'
        elif (args.bsub_off):
            cmd = get_cmd(tag, args.script, p)
        else:
            if needed_zip:
                wait_while_job_running('gz'+group_id+tag,60*10) # ten minutes max
            cmd = bsub_cmd(lsf_out,lsf_err) + lsf_group_cmd + ' ' + get_cmd(tag, args.script, p)
            # Keep track of lsf job for listener
            count_lsf = count_lsf + 1

        print 'batch process running command:\n' + cmd
        os.system(cmd)

    if (count_lsf > 0):
        if lsf_group != '':
            check_done(lsf_group, count_lsf)
    print 'batch_process done'

<commit_msg>Adding an option for max length of output reads<commit_after># ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# There's no good way in cosmos to create a tag
# based on output or to conditionally run a process
# This is a wrapper to run the umi utilities.
#
# Run a program (SeqPrep) for a directory in batches
# This is a helper script for Martin Aryee's
# scripts to demultiplex Illumina sequencing
# reads with sample specific and molecule
# specific tags.
# https://github.com/aryeelab/umi/wiki
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
__author__ = 'Allison MacLeay'

import sys
import os
import argparse
import time

# -----------------------------------------
# Get bash command for specified
# file name prefix
# -----------------------------------------
def get_cmd(group, prog, params):
    path = '.'
    outdir = ''
    cmd = ''
    job_group = ''
    a1 = ''
    a2 = ''
    olen = ''
    if 'path' in params.keys():
        path = params['path']
    if 'out' in params.keys():
        outdir = params['out']
        if (outdir[0] == '/'):
            outdir = outdir[1:]
        else:
            outdir = os.path.abspath(outdir)[1:]
        if (outdir[-1] != '/'):
            outdir = outdir + '/'
    if 'job_group' in params.keys():
        job_group = params['job_group']
    if 'a1' in params.keys():
        a1 = params['a1']
    if 'a1' in params.keys():
        a2 = params['a2']
    if 'olen' in params.keys():
        olen = '-k ' + str(params['olen'])
    fastq_r1 = os.path.join(path, group + '.r1.fastq.gz')
    fastq_r2 = os.path.join(path ,group + '.r2.fastq.gz')
    out1 = '' + group + '_trim.r1.fastq.gz'
    out2 = '' + group + '_trim.r2.fastq.gz'
    cmd = (cmd + """ \
    {prog} -f {fastq_r1} \
-r {fastq_r2} \
-1 {outdir}{out1} \
-2 {outdir}{out2} \
-A {adapter1} \
-B {adapter2} \
-z {olen} \
        """ ).format(
        prog=prog,
        fastq_r1=fastq_r1,
        fastq_r2=fastq_r2,
        outdir='/' + outdir,
        out1=out1,
        out2=out2,
        adapter1=a1,
        adapter2=a2,
        olen=olen,
    )
    return cmd

# -----------------------------------------
# Return all unique file prefixes
# -----------------------------------------
def get_names(dir):
    files = next(os.walk(dir))[2]
    files = map(lambda x: x.split('.')[0], files)
    dfiles = set(files)
    return dfiles

# -----------------------------------------
# Delay completion of script until all
# files are written
#-----------------------------------------
def check_done(group_id,ct):
    start = time.time()
    timeout = (24 * 60 * 60)  # 24 hours
    done = 0
    pr_ct = 0

    while done==0:
        pr_ct+=1
        if (time.time()-start)>timeout:
            print 'Job timed out'
            done=1
        else:
            time.sleep(10)
            if pr_ct % 6 == 0:
                print 'checking for job completion after waiting %d seconds' % (time.time()-start)
            if are_jobs_done(group_id,ct):
                print 'No running jobs found in group ' + group_id
                done=1
    return

def wait_while_job_running(jobname,maxtm):
    done = is_job_done(jobname)
    tmst = time.time()
    duration = 0
    while done == False and (duration < maxtm):
        print "Waiting for job " + jobname + " to complete \n" + str(duration) + " seconds have elapsed."
        time.sleep(10)
        duration = time.time() - tmst
        done = is_job_done(jobname)
    return

# -----------------------------------------
# Zip it
#-----------------------------------------
def gzip_if_not(folder,bsub_off,out,err,group_id):
    zipping = False
    zipped = {}
    for file in next(os.walk(folder))[2]:
        parts = file.split('.')
        if parts[-1] != 'gz':
             if parts[0] not in zipped.keys():
                 zipped[parts[0]]=1
                 zipping = True
                 gzip(os.path.join(folder,parts[0] + '*'),'gz'+group_id+parts[0],bsub_off,out,err)
    return zipping

def gzip(filename,jobname,bsub_off,out,err):
    cmd = 'gzip ' + filename
    if not bsub_off:
       cmd = bsub_cmd(out,err) + ' -J ' + jobname + ' ' + cmd
    os.system(cmd)
    return
# -----------------------------------------
# LSF utilities
#-----------------------------------------
def bsub_cmd(out,err):
    txt = 'bsub -q medium -u am282 -o ' + out + ' -e ' + err + ' '
    return txt

def is_job_done(jobname):
    done = True
    ct=0
    jobs = os.popen("bjobs -J " + jobname).read().split('\n')
    for line in jobs:
        ct+=1
        if ct == 1:
            continue
        status = get_job_status(line)
        if status == 'RUN':
            done = False
        if status == 'PEND':
            done = False
    return done

def are_jobs_done(group,lsf_ct):
    jobs=os.popen("bjobs -g " + group).read().split('\n')
    ct=0
    run_ct=0
    group_status=True
    for line in jobs:
        ct+=1
        if ct==1:
            continue
        status=get_job_status(line)
        if status== 'RUN':
            group_status=False
            run_ct+=1
        if status== 'PEND':
            group_status=False
            run_ct+=1
    if( (ct-1) != lsf_ct):
        print 'Different number of jobs passed in ('+str(lsf_ct)+') and recovered (' + str(ct-1) + ')'
    print "" + str(run_ct) + " jobs are currently running or pending in group " + group
    return group_status

def get_job_status(line):
    status=''
    status_arr=line.strip().split()
    if len(status_arr)>2:
        status=status_arr[2]
    else:
        status=''
    return status

def get_group_id(group):
    i = os.popen("bjgroup | grep "+ group).read()
    i = i.split('\n')
    num=''
    max=0
    for line in i:
        group_arr=line.split()
        if len(group_arr)>1:
            group_name=group_arr[0]
            if are_jobs_done(group_name,0):
                # clean old jobs
                os.system("bgdel " + group_name)
            num=line.strip().split('_')[0].split('/')[-1]
        if str(num)=='':
            num=0
        if type(num) is str:
                if not num.isdigit():
                    num=0
        if int(num) > max:
            max = int(num)
    print 'last id was ' + str(max)
    return group + '/' + str(max+1) + "_trim"

#-----------------------------------------
#	MAIN
# run a program (SeqPrep) for all files in a directory
# that have the same prefix
#-----------------------------------------
if __name__ == '__main__':
    parser = argparse.ArgumentParser(
            description="Run command for removing adapter sequecnes in batches of similarly prefixed names.")
    parser.add_argument('--dir', default='.', help='directory containing output of umi demultiplex')
    parser.add_argument('--script', default='./SeqPrep', help='SeqPrep absolute path.  default is SeqPrep in current directory')
    parser.add_argument('--a1', required=True, help='Adapter 1')
    parser.add_argument('--a2', required=True, help='Adapter 2')
    parser.add_argument('--olen', required=False, help='max length of output reads')
    parser.add_argument('--out', default='tagout', help='directory to deposit output files')
    parser.add_argument('--log', default='batch_log', help='directory to deposit bsub log files')
    parser.add_argument('--bsub_off', action='store_true', help='turn bsub off to test on systems without lsf')
    #parser.add_argument('--undet',  action='store_false', help='include reads less than parameter set my min reads.  Default will skip files named undetermined')
    args = parser.parse_args()

    p = {}
    lsf_group=''
    lsf_group_cmd=''
    if hasattr(args, 'dir'):
        p['path'] = args.dir
    if hasattr(args, 'out'):
        p['out'] = args.out
        os.system('mkdir -p ' + args.out)
    if hasattr(args, 'log'):
        os.system('mkdir -p ' + args.log)
        os.system('ls ' + p['path'] + ' >> ' + args.log + '/ls_inputdir.txt')
    if hasattr(args, 'a1'):
        p['a1']=args.a1
    if hasattr(args, 'a2'):
        p['a2']=args.a2
    if hasattr(args, 'olen'):
        p['olen']=args.olen
    lsf_out = ''
    lsf_err = ''
    if not args.bsub_off:
        lsf_out = os.path.join(args.log, 'lsf_out.log')
        lsf_err = os.path.join(args.log, 'lsf_err.log')
    f = get_names(args.dir)
    if len(f) < 1:
        print "Error: No file prefixes were found in " + args.dir + "."
    count_lsf = 0
    group_id = 'groupID'
    if not args.bsub_off:
        lsf_group = get_group_id("/demux")
        group_id = lsf_group.split("/demux/")[1]
        lsf_group_cmd=' -g ' + lsf_group
    needed_zip = gzip_if_not(args.dir,args.bsub_off,lsf_out,lsf_err,group_id)
    for tag in f:
        if (tag.find('undetermined') > -1 ):
            # skip undeterminded for now
            cmd = 'echo skipping undetermined files'
        elif (args.bsub_off):
            cmd = get_cmd(tag, args.script, p)
        else:
            if needed_zip:
                wait_while_job_running('gz'+group_id+tag,60*10) # ten minutes max
            cmd = bsub_cmd(lsf_out,lsf_err) + lsf_group_cmd + ' ' + get_cmd(tag, args.script, p)
            # Keep track of lsf job for listener
            count_lsf = count_lsf + 1

        print 'batch process running command:\n' + cmd
        os.system(cmd)

    if (count_lsf > 0):
        if lsf_group != '':
            check_done(lsf_group, count_lsf)
    print 'batch_process done'

<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import unicode_literals

from django.core.urlresolvers import reverse_lazy
from django.http.response import HttpResponseForbidden
from django.utils.datastructures import SortedDict
from django.views import generic

from braces.views import LoginRequiredMixin
from bazaar.listings.models import Publishing
from .filters import ProductFilter
from .forms import ProductForm
from .models import Product
from ..warehouse.models import Location
from ..mixins import BazaarPrefixMixin, FilterSortableListView

class ProductListView(LoginRequiredMixin, BazaarPrefixMixin, FilterSortableListView):
    model = Product
    paginate_by = 20
    filter_class = ProductFilter
    sort_fields = ('name', 'price', 'purchase_price', 'stock')

    def get_queryset(self):
        qs = super(ProductListView, self).get_queryset()
        location_storage = Location.objects.get_or_create(type=Location.LOCATION_STORAGE)[0]
        qs = qs.extra(
            select=SortedDict([
                ("stock",
                 "SELECT SUM(quantity) "
                 "FROM warehouse_stock "
                 "WHERE warehouse_stock.product_id = goods_product.id "
                 "AND warehouse_stock.location_id = %s"),
                ("purchase_price",
                 "SELECT SUM(unit_price) "
                 "FROM warehouse_stock "
                 "WHERE warehouse_stock.product_id = goods_product.id "
                 "AND warehouse_stock.location_id = %s"),
            ]),
            select_params=(
                location_storage.id,
                location_storage.id,
            )
        )
        return qs

class ProductDetailView(LoginRequiredMixin, BazaarPrefixMixin, generic.DetailView):
    model = Product
    fields = ['photo', 'name', 'description', 'ean', 'quantity', 'price', 'cost']

    def get_context_data(self, **kwargs):
        # Call the base implementation first to get a context
        context = super(ProductDetailView, self).get_context_data(**kwargs)

        # Check if this product was ever published
        product = self.get_object()
        has_publishings = Publishing.objects.filter(listing__products=product).exists()
        context['deletable'] = not has_publishings

        return context

class ProductCreateView(LoginRequiredMixin, BazaarPrefixMixin, generic.CreateView):
    model = Product
    form_class = ProductForm

    fields = ['name', 'description', 'ean', 'photo', 'price']

    def get_success_url(self):
        return reverse_lazy("bazaar:product-detail", kwargs={'pk': self.object.id})

class ProductDeleteView(LoginRequiredMixin, BazaarPrefixMixin, generic.DeleteView):
    model = Product
    success_url = reverse_lazy('bazaar:product-list')

    def delete(self, request, *args, **kwargs):
        product = self.get_object()
        has_publishings = Publishing.objects.filter(listing__products=product).exists()
        if has_publishings:
            return HttpResponseForbidden()

        # delete all associated listings
        self.get_object().listings.all().delete()

        return super(ProductDeleteView, self).delete(request, *args, **kwargs)

class ProductUpdateView(LoginRequiredMixin, BazaarPrefixMixin, generic.UpdateView):
    model = Product
    form_class = ProductForm

    def get_success_url(self):
        return reverse_lazy("bazaar:product-detail", kwargs={'pk': self.object.id})
<commit_msg>minor fix<commit_after>from __future__ import absolute_import
from __future__ import unicode_literals

from django.core.urlresolvers import reverse_lazy
from django.http.response import HttpResponseForbidden
from django.utils.datastructures import SortedDict
from django.views import generic

from braces.views import LoginRequiredMixin
from bazaar.listings.models import Publishing
from .filters import ProductFilter
from .forms import ProductForm
from .models import Product
from ..warehouse.models import Location
from ..mixins import BazaarPrefixMixin, FilterSortableListView

class ProductListView(LoginRequiredMixin, BazaarPrefixMixin, FilterSortableListView):
    model = Product
    paginate_by = 20
    filter_class = ProductFilter
    sort_fields = ('name', 'price', 'purchase_price', 'stock')

    def get_queryset(self):
        qs = super(ProductListView, self).get_queryset()
        location_storage = Location.objects.get_or_create(type=Location.LOCATION_STORAGE)[0]
        qs = qs.extra(
            select=SortedDict([
                ("stock",
                 "SELECT SUM(quantity) "
                 "FROM warehouse_stock "
                 "WHERE warehouse_stock.product_id = goods_product.id "
                 "AND warehouse_stock.location_id = %s"),
                ("purchase_price",
                 "SELECT SUM(unit_price) "
                 "FROM warehouse_stock "
                 "WHERE warehouse_stock.product_id = goods_product.id "
                 "AND warehouse_stock.location_id = %s"),
            ]),
            select_params=(
                location_storage.id,
                location_storage.id,
            )
        )
        return qs

class ProductDetailView(LoginRequiredMixin, BazaarPrefixMixin, generic.DetailView):
    model = Product
    fields = ['photo', 'name', 'description', 'ean', 'quantity', 'price', 'cost']

    def get_context_data(self, **kwargs):
        # Call the base implementation first to get a context
        context = super(ProductDetailView, self).get_context_data(**kwargs)

        # Check if this product was ever published
        product = self.get_object()
        has_publishings = Publishing.objects.filter(listing__products=product).exists()
        context['deletable'] = not has_publishings

        return context

class ProductCreateView(LoginRequiredMixin, BazaarPrefixMixin, generic.CreateView):
    model = Product
    form_class = ProductForm

    fields = ['name', 'description', 'ean', 'photo', 'price']

    def get_success_url(self):
        return reverse_lazy("bazaar:product-detail", kwargs={'pk': self.object.id})

class ProductDeleteView(LoginRequiredMixin, BazaarPrefixMixin, generic.DeleteView):
    model = Product
    success_url = reverse_lazy('bazaar:product-list')

    def delete(self, request, *args, **kwargs):
        product = self.get_object()
        has_publishings = Publishing.objects.filter(listing__products=product).exists()
        if has_publishings:
            return HttpResponseForbidden()

        # delete all associated listings
        product.listings.all().delete()

        return super(ProductDeleteView, self).delete(request, *args, **kwargs)

class ProductUpdateView(LoginRequiredMixin, BazaarPrefixMixin, generic.UpdateView):
    model = Product
    form_class = ProductForm

    def get_success_url(self):
        return reverse_lazy("bazaar:product-detail", kwargs={'pk': self.object.id})
<|endoftext|>


<commit_before>"""HLA typing from extracted HLA alleles with OptiType

https://github.com/FRED-2/OptiType
"""
import csv
import glob
import os
import re
import sys
import shutil

import toolz as tz

from bcbio import utils
from bcbio.distributed.transaction import file_transaction, tx_tmpdir
from bcbio.hla import bwakit
from bcbio.pipeline import config_utils
from bcbio.pipeline import datadict as dd
from bcbio.provenance import do

SUPPORTED_HLAS = ["HLA-A", "HLA-B", "HLA-C"]

def run(data):
    """HLA typing with OptiType, parsing output from called genotype files.
    """
    hlas = []
    for hla_fq in tz.get_in(["hla", "fastq"], data, []):
        hla_type = re.search(r"[.-](?P<hlatype>HLA-[\w-]+).fq", hla_fq).group("hlatype")
        if hla_type in SUPPORTED_HLAS:
            if utils.file_exists(hla_fq):
                hlas.append((hla_type, hla_fq))
    if len(hlas) > 0:
        out_dir = utils.safe_makedir(os.path.join(dd.get_work_dir(data), "align",
                                                  dd.get_sample_name(data), "hla",
                                                  "OptiType-HLA-A_B_C"))
        # When running UMIs and hla typing we want to pick the original fastqs
        if len(hlas) > len(SUPPORTED_HLAS):
            hlas = [x for x in hlas if os.path.basename(x[1]).find("-cumi") == -1]
        if len(hlas) == len(SUPPORTED_HLAS):
            hla_fq = combine_hla_fqs(hlas, out_dir + "-input.fq", data)
            if utils.file_exists(hla_fq):
                out_file = glob.glob(os.path.join(out_dir, "*", "*_result.tsv"))
                if len(out_file) > 0:
                    out_file = out_file[0]
                else:
                    out_file = _call_hla(hla_fq, out_dir, data)
                out_file = _prepare_calls(out_file, os.path.dirname(out_dir), data)
                data["hla"].update({"call_file": out_file,
                                    "hlacaller": "optitype"})
    return data

def combine_hla_fqs(hlas, out_file, data):
    """OptiType performs best on a combination of all extracted HLAs.
    """
    if not utils.file_exists(out_file):
        with file_transaction(data, out_file) as tx_out_file:
            with open(tx_out_file, "w") as out_handle:
                for hla_type, hla_fq in hlas:
                    if utils.file_exists(hla_fq):
                        with open(hla_fq) as in_handle:
                            shutil.copyfileobj(in_handle, out_handle)
    return out_file

def _prepare_calls(result_file, out_dir, data):
    """Write summary file of results of HLA typing by allele.
    """
    sample = dd.get_sample_name(data)
    out_file = os.path.join(out_dir, "%s-optitype.csv" % (sample))
    if not utils.file_uptodate(out_file, result_file):
        hla_truth = bwakit.get_hla_truthset(data)
        with file_transaction(data, out_file) as tx_out_file:
            with open(tx_out_file, "w") as out_handle:
                writer = csv.writer(out_handle)
                allele_info = _parse_result_file(result_file)
                if len(allele_info) == 1:
                    writer.writerow(["sample", "locus", "alleles", "expected", "validates"])
                else:
                    writer.writerow(["sample", "local", "index", "alleles", "score"])
                for j, (alleles, score) in enumerate(allele_info):
                    for hla_locus, call_alleles in alleles:
                        truth_alleles = tz.get_in([sample, hla_locus], hla_truth, [])
                        if len(allele_info) == 1:
                            writer.writerow([sample, hla_locus,
                                             ";".join(call_alleles), ";".join(truth_alleles),
                                             bwakit.matches_truth(call_alleles, truth_alleles, data)])
                        else:
                            writer.writerow([sample, hla_locus, j, ";".join(call_alleles), score])
    return out_file

def _parse_result_file(result_file):
    with open(result_file) as in_handle:
        header = in_handle.readline().rstrip("\r\n").split("\t")
        all_locus_is = []
        for hla_locus in SUPPORTED_HLAS:
            hla_locus = hla_locus.replace("HLA-", "")
            all_locus_is.append((hla_locus, [i for i, h in enumerate(header) if h.startswith(hla_locus)]))
        score_i = [i for i, h in enumerate(header) if h == "Objective"][0]
        hits = []
        for line in in_handle:
            hit = line.rstrip("\r\n").split("\t")
            assert len(hit) == len(header), result_file
            cur_hlas = []
            for hla_locus, locus_is in all_locus_is:
                cur_hlas.append((hla_locus, ["HLA-%s" % hit[i] for i in locus_is]))
            hits.append((cur_hlas, hit[score_i]))
        return hits

def _call_hla(hla_fq, out_dir, data):
    """Run OptiType HLA calling for a specific fastq input.
    """
    bin_dir = os.path.dirname(os.path.realpath(sys.executable))
    out_dir = utils.safe_makedir(out_dir)
    with tx_tmpdir(data, os.path.dirname(out_dir)) as tx_out_dir:
        config_file = os.path.join(tx_out_dir, "config.ini")
        with open(config_file, "w") as out_handle:
            razers3 = os.path.join(bin_dir, "razers3")
            if not os.path.exists(razers3):
                raise ValueError("Could not find razers3 executable at %s" % (razers3))
            out_handle.write(CONFIG_TMPL.format(razers3=razers3, cores=dd.get_cores(data)))
        resources = config_utils.get_resources("optitype", data["config"])
        if resources.get("options"):
            opts = " ".join([str(x) for x in resources["options"]])
        else:
            opts = ""
        cmd = ("OptiTypePipeline.py -v --dna {opts} -o {tx_out_dir} "
                "-i {hla_fq} -c {config_file}")
        do.run(cmd.format(**locals()), "HLA typing with OptiType")
        for outf in os.listdir(tx_out_dir):
            shutil.move(os.path.join(tx_out_dir, outf), os.path.join(out_dir, outf))
    out_file = glob.glob(os.path.join(out_dir, "*", "*_result.tsv"))
    assert len(out_file) == 1, "Expected one result file for OptiType, found %s" % out_file
    return out_file[0]

CONFIG_TMPL = """
[mapping]
razers3={razers3}
threads={cores}
[ilp]
solver=cbc
threads={cores}
[behavior]
deletebam=true
unpaired_weight=0
use_discordant=false
"""
<commit_msg>Add `--enumerate` flag to OptiType.<commit_after>"""HLA typing from extracted HLA alleles with OptiType

https://github.com/FRED-2/OptiType
"""
import csv
import glob
import os
import re
import sys
import shutil

import toolz as tz

from bcbio import utils
from bcbio.distributed.transaction import file_transaction, tx_tmpdir
from bcbio.hla import bwakit
from bcbio.pipeline import config_utils
from bcbio.pipeline import datadict as dd
from bcbio.provenance import do

SUPPORTED_HLAS = ["HLA-A", "HLA-B", "HLA-C"]

def run(data):
    """HLA typing with OptiType, parsing output from called genotype files.
    """
    hlas = []
    for hla_fq in tz.get_in(["hla", "fastq"], data, []):
        hla_type = re.search(r"[.-](?P<hlatype>HLA-[\w-]+).fq", hla_fq).group("hlatype")
        if hla_type in SUPPORTED_HLAS:
            if utils.file_exists(hla_fq):
                hlas.append((hla_type, hla_fq))
    if len(hlas) > 0:
        out_dir = utils.safe_makedir(os.path.join(dd.get_work_dir(data), "align",
                                                  dd.get_sample_name(data), "hla",
                                                  "OptiType-HLA-A_B_C"))
        # When running UMIs and hla typing we want to pick the original fastqs
        if len(hlas) > len(SUPPORTED_HLAS):
            hlas = [x for x in hlas if os.path.basename(x[1]).find("-cumi") == -1]
        if len(hlas) == len(SUPPORTED_HLAS):
            hla_fq = combine_hla_fqs(hlas, out_dir + "-input.fq", data)
            if utils.file_exists(hla_fq):
                out_file = glob.glob(os.path.join(out_dir, "*", "*_result.tsv"))
                if len(out_file) > 0:
                    out_file = out_file[0]
                else:
                    out_file = _call_hla(hla_fq, out_dir, data)
                out_file = _prepare_calls(out_file, os.path.dirname(out_dir), data)
                data["hla"].update({"call_file": out_file,
                                    "hlacaller": "optitype"})
    return data

def combine_hla_fqs(hlas, out_file, data):
    """OptiType performs best on a combination of all extracted HLAs.
    """
    if not utils.file_exists(out_file):
        with file_transaction(data, out_file) as tx_out_file:
            with open(tx_out_file, "w") as out_handle:
                for hla_type, hla_fq in hlas:
                    if utils.file_exists(hla_fq):
                        with open(hla_fq) as in_handle:
                            shutil.copyfileobj(in_handle, out_handle)
    return out_file

def _prepare_calls(result_file, out_dir, data):
    """Write summary file of results of HLA typing by allele.
    """
    sample = dd.get_sample_name(data)
    out_file = os.path.join(out_dir, "%s-optitype.csv" % (sample))
    if not utils.file_uptodate(out_file, result_file):
        hla_truth = bwakit.get_hla_truthset(data)
        with file_transaction(data, out_file) as tx_out_file:
            with open(tx_out_file, "w") as out_handle:
                writer = csv.writer(out_handle)
                allele_info = _parse_result_file(result_file)
                if len(allele_info) == 1:
                    writer.writerow(["sample", "locus", "alleles", "expected", "validates"])
                else:
                    writer.writerow(["sample", "local", "index", "alleles", "score"])
                for j, (alleles, score) in enumerate(allele_info):
                    for hla_locus, call_alleles in alleles:
                        truth_alleles = tz.get_in([sample, hla_locus], hla_truth, [])
                        if len(allele_info) == 1:
                            writer.writerow([sample, hla_locus,
                                             ";".join(call_alleles), ";".join(truth_alleles),
                                             bwakit.matches_truth(call_alleles, truth_alleles, data)])
                        else:
                            writer.writerow([sample, hla_locus, j, ";".join(call_alleles), score])
    return out_file

def _parse_result_file(result_file):
    with open(result_file) as in_handle:
        header = in_handle.readline().rstrip("\r\n").split("\t")
        all_locus_is = []
        for hla_locus in SUPPORTED_HLAS:
            hla_locus = hla_locus.replace("HLA-", "")
            all_locus_is.append((hla_locus, [i for i, h in enumerate(header) if h.startswith(hla_locus)]))
        score_i = [i for i, h in enumerate(header) if h == "Objective"][0]
        hits = []
        for line in in_handle:
            hit = line.rstrip("\r\n").split("\t")
            assert len(hit) == len(header), result_file
            cur_hlas = []
            for hla_locus, locus_is in all_locus_is:
                cur_hlas.append((hla_locus, ["HLA-%s" % hit[i] for i in locus_is]))
            hits.append((cur_hlas, hit[score_i]))
        return hits

def _call_hla(hla_fq, out_dir, data):
    """Run OptiType HLA calling for a specific fastq input.
    """
    bin_dir = os.path.dirname(os.path.realpath(sys.executable))
    out_dir = utils.safe_makedir(out_dir)
    with tx_tmpdir(data, os.path.dirname(out_dir)) as tx_out_dir:
        config_file = os.path.join(tx_out_dir, "config.ini")
        with open(config_file, "w") as out_handle:
            razers3 = os.path.join(bin_dir, "razers3")
            if not os.path.exists(razers3):
                raise ValueError("Could not find razers3 executable at %s" % (razers3))
            out_handle.write(CONFIG_TMPL.format(razers3=razers3, cores=dd.get_cores(data)))
        resources = config_utils.get_resources("optitype", data["config"])
        if resources.get("options"):
            opts = " ".join([str(x) for x in resources["options"]])
        else:
            opts = ""
        cmd = ("OptiTypePipeline.py -v --dna {opts} -o {tx_out_dir} --enumerate 10 "
                "-i {hla_fq} -c {config_file}")
        do.run(cmd.format(**locals()), "HLA typing with OptiType")
        for outf in os.listdir(tx_out_dir):
            shutil.move(os.path.join(tx_out_dir, outf), os.path.join(out_dir, outf))
    out_file = glob.glob(os.path.join(out_dir, "*", "*_result.tsv"))
    assert len(out_file) == 1, "Expected one result file for OptiType, found %s" % out_file
    return out_file[0]

CONFIG_TMPL = """
[mapping]
razers3={razers3}
threads={cores}
[ilp]
solver=cbc
threads={cores}
[behavior]
deletebam=true
unpaired_weight=0
use_discordant=false
"""
<|endoftext|>


<commit_before>"""Utility functionality for logging.
"""
import multiprocessing
import os
import socket
import sys
import time

import logbook
import logbook.queues

from bcbio import utils

LOG_NAME = "bcbio-nextgen"

def get_log_dir(config):
    d = config.get("log_dir", "log")
    return d

logger = logbook.Logger(LOG_NAME)
logger_cl = logbook.Logger(LOG_NAME + "-commands")
logger_stdout = logbook.Logger(LOG_NAME + "-stdout")
mpq = multiprocessing.Queue(-1)

def _is_cl(record, _):
    return record.channel == LOG_NAME + "-commands"

def _is_stdout(record, _):
    return record.channel == LOG_NAME + "-stdout"

def _not_cl(record, handler):
    return not _is_cl(record, handler) and not _is_stdout(record, handler)

class CloseableNestedSetup(logbook.NestedSetup):
    def close(self):
        for obj in self.objects:
            if hasattr(obj, "close"):
                obj.close()

class IOSafeMultiProcessingSubscriber(logbook.queues.MultiProcessingSubscriber):
    """Recover from zeromq interrupted system call IOErrors that stop logging.
    """
    def recv(self, timeout=None):
        try:
            return super(IOSafeMultiProcessingSubscriber, self).recv(timeout)
        except IOError as e:
            if "Interrupted system call" in str(e):
                return None
            else:
                raise

def _create_log_handler(config, add_hostname=False, direct_hostname=False):
    logbook.set_datetime_format("utc")
    handlers = [logbook.NullHandler()]
    format_str = "".join(["[{record.time:%Y-%m-%dT%H:%MZ}] " if config.get("include_time", True) else "",
                          "{record.extra[source]}: " if add_hostname else "",
                          "%s: " % (socket.gethostname)() if direct_hostname else "",
                          "{record.message}"])

    log_dir = get_log_dir(config)
    if log_dir:
        if not os.path.exists(log_dir):
            utils.safe_makedir(log_dir)
            # Wait to propagate, Otherwise see logging errors on distributed filesystems.
            time.sleep(5)
        handlers.append(logbook.FileHandler(os.path.join(log_dir, "%s.log" % LOG_NAME),
                                            format_string=format_str, level="INFO",
                                            filter=_not_cl))
        handlers.append(logbook.FileHandler(os.path.join(log_dir, "%s-debug.log" % LOG_NAME),
                                            format_string=format_str, level="DEBUG", bubble=True,
                                            filter=_not_cl))
        handlers.append(logbook.FileHandler(os.path.join(log_dir, "%s-commands.log" % LOG_NAME),
                                            format_string=format_str, level="DEBUG",
                                            filter=_is_cl))
    handlers.append(logbook.StreamHandler(sys.stdout, format_string="{record.message}",
                                          level="DEBUG", filter=_is_stdout))

    email = config.get("email", config.get("resources", {}).get("log", {}).get("email"))
    if email:
        email_str = u'''Subject: [bcbio-nextgen] {record.extra[run]} \n\n {record.message}'''
        handlers.append(logbook.MailHandler(email, [email],
                                            format_string=email_str,
                                            level='INFO', bubble=True))

    handlers.append(logbook.StreamHandler(sys.stderr, format_string=format_str, bubble=True,
                                          filter=_not_cl))
    return CloseableNestedSetup(handlers)

def create_base_logger(config=None, parallel=None):
    """Setup base logging configuration, also handling remote logging.

    Correctly sets up for local, multiprocessing and distributed runs.
    Creates subscribers for non-local runs that will be references from
    local logging.
    """
    if parallel is None: parallel = {}
    parallel_type = parallel.get("type", "local")
    cores = parallel.get("cores", 1)
    if parallel_type == "ipython":
        from bcbio.log import logbook_zmqpush
        ips = [ip for ip in socket.gethostbyname_ex(socket.gethostname())[2]
               if not ip.startswith("127.0.0")]
        if not ips:
            sys.stderr.write("Cannot resolve a local IP address that isn't 127.0.0. "
                             "Your machines might not have a local IP address "
                             "assigned or are not able to resolve it.\n")
            sys.exit(1)
        uri = "tcp://%s" % ips[0]
        subscriber = logbook_zmqpush.ZeroMQPullSubscriber()
        mport = subscriber.socket.bind_to_random_port(uri)
        wport_uri = "%s:%s" % (uri, mport)
        parallel["log_queue"] = wport_uri
        subscriber.dispatch_in_background(_create_log_handler(config, True))
    elif cores > 1:
        subscriber = IOSafeMultiProcessingSubscriber(mpq)
        subscriber.dispatch_in_background(_create_log_handler(config))
    else:
        # Do not need to setup anything for local logging
        pass
    return parallel

def setup_local_logging(config=None, parallel=None):
    """Setup logging for a local context, directing messages to appropriate base loggers.

    Handles local, multiprocessing and distributed setup, connecting
    to handlers created by the base logger.
    """
    if config is None: config = {}
    if parallel is None: parallel = {}
    parallel_type = parallel.get("type", "local")
    cores = parallel.get("cores", 1)
    wrapper = parallel.get("wrapper", None)
    if parallel_type == "ipython":
        from bcbio.log import logbook_zmqpush
        handler = logbook_zmqpush.ZeroMQPushHandler(parallel["log_queue"])
    elif cores > 1:
        handler = logbook.queues.MultiProcessingHandler(mpq)
    else:
        handler = _create_log_handler(config, direct_hostname=wrapper is not None)
    handler.push_thread()
    return handler

def setup_script_logging():
    """
    Use this logger for standalone scripts, or script-like subcommands,
    such as bcbio_prepare_samples and bcbio_nextgen.py -w template.
    """
    handlers = [logbook.NullHandler()]
    format_str = ("[{record.time:%Y-%m-%dT%H:%MZ}] "
                  "{record.level_name}: {record.message}")

    handler = logbook.StreamHandler(sys.stderr, format_string=format_str,
                                    level="DEBUG")
    handler.push_thread()
    return handler
<commit_msg>Logging: improve resolution of IP addresses<commit_after>"""Utility functionality for logging.
"""
import multiprocessing
import os
import socket
import sys
import time

import logbook
import logbook.queues

from bcbio import utils

LOG_NAME = "bcbio-nextgen"

def get_log_dir(config):
    d = config.get("log_dir", "log")
    return d

logger = logbook.Logger(LOG_NAME)
logger_cl = logbook.Logger(LOG_NAME + "-commands")
logger_stdout = logbook.Logger(LOG_NAME + "-stdout")
mpq = multiprocessing.Queue(-1)

def _is_cl(record, _):
    return record.channel == LOG_NAME + "-commands"

def _is_stdout(record, _):
    return record.channel == LOG_NAME + "-stdout"

def _not_cl(record, handler):
    return not _is_cl(record, handler) and not _is_stdout(record, handler)

class CloseableNestedSetup(logbook.NestedSetup):
    def close(self):
        for obj in self.objects:
            if hasattr(obj, "close"):
                obj.close()

class IOSafeMultiProcessingSubscriber(logbook.queues.MultiProcessingSubscriber):
    """Recover from zeromq interrupted system call IOErrors that stop logging.
    """
    def recv(self, timeout=None):
        try:
            return super(IOSafeMultiProcessingSubscriber, self).recv(timeout)
        except IOError as e:
            if "Interrupted system call" in str(e):
                return None
            else:
                raise

def _create_log_handler(config, add_hostname=False, direct_hostname=False):
    logbook.set_datetime_format("utc")
    handlers = [logbook.NullHandler()]
    format_str = "".join(["[{record.time:%Y-%m-%dT%H:%MZ}] " if config.get("include_time", True) else "",
                          "{record.extra[source]}: " if add_hostname else "",
                          "%s: " % (socket.gethostname)() if direct_hostname else "",
                          "{record.message}"])

    log_dir = get_log_dir(config)
    if log_dir:
        if not os.path.exists(log_dir):
            utils.safe_makedir(log_dir)
            # Wait to propagate, Otherwise see logging errors on distributed filesystems.
            time.sleep(5)
        handlers.append(logbook.FileHandler(os.path.join(log_dir, "%s.log" % LOG_NAME),
                                            format_string=format_str, level="INFO",
                                            filter=_not_cl))
        handlers.append(logbook.FileHandler(os.path.join(log_dir, "%s-debug.log" % LOG_NAME),
                                            format_string=format_str, level="DEBUG", bubble=True,
                                            filter=_not_cl))
        handlers.append(logbook.FileHandler(os.path.join(log_dir, "%s-commands.log" % LOG_NAME),
                                            format_string=format_str, level="DEBUG",
                                            filter=_is_cl))
    handlers.append(logbook.StreamHandler(sys.stdout, format_string="{record.message}",
                                          level="DEBUG", filter=_is_stdout))

    email = config.get("email", config.get("resources", {}).get("log", {}).get("email"))
    if email:
        email_str = u'''Subject: [bcbio-nextgen] {record.extra[run]} \n\n {record.message}'''
        handlers.append(logbook.MailHandler(email, [email],
                                            format_string=email_str,
                                            level='INFO', bubble=True))

    handlers.append(logbook.StreamHandler(sys.stderr, format_string=format_str, bubble=True,
                                          filter=_not_cl))
    return CloseableNestedSetup(handlers)

def create_base_logger(config=None, parallel=None):
    """Setup base logging configuration, also handling remote logging.

    Correctly sets up for local, multiprocessing and distributed runs.
    Creates subscribers for non-local runs that will be references from
    local logging.

    Retrieves IP address using tips from http://stackoverflow.com/a/1267524/252589
    """
    if parallel is None: parallel = {}
    parallel_type = parallel.get("type", "local")
    cores = parallel.get("cores", 1)
    if parallel_type == "ipython":
        from bcbio.log import logbook_zmqpush
        ips = [ip for ip in socket.gethostbyname_ex(socket.gethostname())[2]
               if not ip.startswith("127.")]
        ips += [(s.connect(('8.8.8.8', 53)), s.getsockname()[0], s.close())[1] for s in
                [socket.socket(socket.AF_INET, socket.SOCK_DGRAM)]]
        if not ips:
            sys.stderr.write("Cannot resolve a local IP address that isn't 127.x.x.x "
                             "Your machines might not have a local IP address "
                             "assigned or are not able to resolve it.\n")
            sys.exit(1)
        uri = "tcp://%s" % ips[0]
        subscriber = logbook_zmqpush.ZeroMQPullSubscriber()
        mport = subscriber.socket.bind_to_random_port(uri)
        wport_uri = "%s:%s" % (uri, mport)
        parallel["log_queue"] = wport_uri
        subscriber.dispatch_in_background(_create_log_handler(config, True))
    elif cores > 1:
        subscriber = IOSafeMultiProcessingSubscriber(mpq)
        subscriber.dispatch_in_background(_create_log_handler(config))
    else:
        # Do not need to setup anything for local logging
        pass
    return parallel

def setup_local_logging(config=None, parallel=None):
    """Setup logging for a local context, directing messages to appropriate base loggers.

    Handles local, multiprocessing and distributed setup, connecting
    to handlers created by the base logger.
    """
    if config is None: config = {}
    if parallel is None: parallel = {}
    parallel_type = parallel.get("type", "local")
    cores = parallel.get("cores", 1)
    wrapper = parallel.get("wrapper", None)
    if parallel_type == "ipython":
        from bcbio.log import logbook_zmqpush
        handler = logbook_zmqpush.ZeroMQPushHandler(parallel["log_queue"])
    elif cores > 1:
        handler = logbook.queues.MultiProcessingHandler(mpq)
    else:
        handler = _create_log_handler(config, direct_hostname=wrapper is not None)
    handler.push_thread()
    return handler

def setup_script_logging():
    """
    Use this logger for standalone scripts, or script-like subcommands,
    such as bcbio_prepare_samples and bcbio_nextgen.py -w template.
    """
    handlers = [logbook.NullHandler()]
    format_str = ("[{record.time:%Y-%m-%dT%H:%MZ}] "
                  "{record.level_name}: {record.message}")

    handler = logbook.StreamHandler(sys.stderr, format_string=format_str,
                                    level="DEBUG")
    handler.push_thread()
    return handler
<|endoftext|>


<commit_before>#!/usr/bin/env python

import socket
import sys
import requests
import string
from scapy.all import *

r = requests.get("http://4.ipquail.com/ip")

if r.status_code == 200:
    ipv4 = r.content.translate(None, string.whitespace)
else:
    ipv4 = 'err'

#print ipv4

message = "This is a test message from\n%s" % ipv4

server	= '192.168.203.22'
fakesrc = '1.1.1.1'

mypacket = IP(dst=server, src=fakesrc)/UDP(sport=RandShort(),dport=10000)/message
l2packet = Ether()/mypacket
sendp(l2packet)
<commit_msg>spoof packet twice, once from 1.1.1.1 once from RandIP()<commit_after>#!/usr/bin/env python

import socket
import sys
import requests
import string
from scapy.all import *

r = requests.get("http://4.ipquail.com/ip")

if r.status_code == 200:
    ipv4 = r.content.translate(None, string.whitespace)
else:
    ipv4 = 'err'

message = "This is a test message from\n%s" % ipv4

server	= '192.168.202.33'
fakesrc = '1.1.1.1'

mypacket = IP(dst=server, src=fakesrc)/UDP(sport=RandShort(),dport=10000)/message
l2packet = Ether()/mypacket
sendp(l2packet)

mypacket = IP(dst=server, src=RandIP())/UDP(sport=RandShort(),dport=10000)/message
l2packet = Ether()/mypacket
sendp(l2packet)

<|endoftext|>


<commit_before>from typing import Optional, Iterable, Dict, List, TypeVar

import attr
from cached_property import cached_property

from bead.meta import InputSpec
from bead.tech.timestamp import time_from_timestamp
from .freshness import Freshness

def none_to_empty_dict(v):
    """attr converter"""
    if v is None:
        return {}
    return v

@attr.s(auto_attribs=True)
class Dummy:
    """
    A bead.Bead look-alike when looking only at the metadata.

    Also has metadata for coloring (freshness).
    """
    # these are considered immutable once the object is created
    name: str = attr.ib(kw_only=True, default="UNKNOWN")
    content_id: str = attr.ib(kw_only=True)
    kind: str = attr.ib(kw_only=True)
    timestamp_str: str = attr.ib(kw_only=True)
    inputs: List[InputSpec] = attr.ib(kw_only=True, factory=list, converter=list)

    # these can be modified after the object is created
    input_map: Dict[str, str] = attr.ib(kw_only=True, factory=dict, converter=none_to_empty_dict)
    freshness: Freshness = attr.ib(kw_only=True, default=Freshness.SUPERSEDED)
    box_name: Optional[str] = attr.ib(kw_only=True, default=None)

    @cached_property
    def timestamp(self):
        return time_from_timestamp(self.timestamp_str)

    @cached_property
    def ref(self) -> 'Ref':
        return Ref.from_bead(self)

    @classmethod
    def from_bead(cls, bead):
        return cls(
            name=bead.name,
            content_id=bead.content_id,
            kind=bead.kind,
            timestamp_str=bead.timestamp_str,
            inputs=bead.inputs,
            input_map=bead.input_map,
            box_name=bead.box_name)

    @classmethod
    def phantom_from_input(cls, bead: 'Dummy', inputspec: InputSpec):
        """
        Create phantom beads from inputs.

        The returned bead is referenced as input from another bead,
        but we do not have the referenced bead.
        """
        phantom = (
            cls(
                name=bead.get_input_bead_name(inputspec.name),
                content_id=inputspec.content_id,
                kind=inputspec.kind,
                timestamp_str=inputspec.timestamp_str))
        phantom.freshness = Freshness.PHANTOM
        return phantom

    def set_freshness(self, freshness):
        # phantom beads do not change freshness
        if self.freshness != Freshness.PHANTOM:
            self.freshness = freshness

    @property
    def is_not_phantom(self):
        return self.freshness != Freshness.PHANTOM

    def get_input_bead_name(self, input_nick):
        '''
        Returns the bead name on which update works.
        '''
        return self.input_map.get(input_nick, input_nick)

    def set_input_bead_name(self, input_nick, bead_name):
        '''
        Sets the bead name to be used for updates in the future.
        '''
        self.input_map[input_nick] = bead_name

    def __repr__(self):
        cls = self.__class__.__name__
        kind = self.kind[:8]
        content_id = self.content_id[:8]
        inputs = repr(self.inputs)
        return f"{cls}:{self.name}:{kind}:{content_id}:{self.freshness}:{inputs}:{self.input_map}"

Bead = TypeVar('Bead')

@attr.s(frozen=True, slots=True, auto_attribs=True)
class Ref:
    """
    Unique reference for Dummy-es.

    NOTE: Using multiple boxes can make this reference non-unique, as it is possible to
    have beads with same name and content, but with different input-maps.
    This could happen e.g. for sets of beads that are "branched" - released - and
    possibly need separate future maintenance.

    This potential non-unique-ness can be mitigated by having `update` search for updates
    in the same box, the workspace was developed from.
    """
    name: str
    content_id: str

    @classmethod
    def from_bead(cls, bead) -> 'Ref':
        return cls(bead.name, bead.content_id)

    @classmethod
    def from_bead_input(cls, bead, input) -> 'Ref':
        src_bead_name = bead.get_input_bead_name(input.name)
        return cls(src_bead_name, input.content_id)

    @classmethod
    def index_for(cls, beads: Iterable[Bead]) -> Dict['Ref', Bead]:
        bead_by_ref = {}
        for bead in beads:
            bead_by_ref[cls.from_bead(bead)] = bead
        return bead_by_ref
<commit_msg>type hint<commit_after>from typing import Optional, Iterable, Dict, List, TypeVar

import attr
from cached_property import cached_property

from bead.meta import InputSpec
from bead.tech.timestamp import time_from_timestamp
from .freshness import Freshness

def none_to_empty_dict(v):
    """attr converter"""
    if v is None:
        return {}
    return v

@attr.s(auto_attribs=True)
class Dummy:
    """
    A bead.Bead look-alike when looking only at the metadata.

    Also has metadata for coloring (freshness).
    """
    # these are considered immutable once the object is created
    name: str = attr.ib(kw_only=True, default="UNKNOWN")
    content_id: str = attr.ib(kw_only=True)
    kind: str = attr.ib(kw_only=True)
    timestamp_str: str = attr.ib(kw_only=True)
    inputs: List[InputSpec] = attr.ib(kw_only=True, factory=list, converter=list)

    # these can be modified after the object is created
    input_map: Dict[str, str] = attr.ib(kw_only=True, factory=dict, converter=none_to_empty_dict)
    freshness: Freshness = attr.ib(kw_only=True, default=Freshness.SUPERSEDED)
    box_name: Optional[str] = attr.ib(kw_only=True, default=None)

    @cached_property
    def timestamp(self):
        return time_from_timestamp(self.timestamp_str)

    @cached_property
    def ref(self) -> 'Ref':
        return Ref.from_bead(self)

    @classmethod
    def from_bead(cls, bead):
        return cls(
            name=bead.name,
            content_id=bead.content_id,
            kind=bead.kind,
            timestamp_str=bead.timestamp_str,
            inputs=bead.inputs,
            input_map=bead.input_map,
            box_name=bead.box_name)

    @classmethod
    def phantom_from_input(cls, bead: 'Dummy', inputspec: InputSpec):
        """
        Create phantom beads from inputs.

        The returned bead is referenced as input from another bead,
        but we do not have the referenced bead.
        """
        phantom = (
            cls(
                name=bead.get_input_bead_name(inputspec.name),
                content_id=inputspec.content_id,
                kind=inputspec.kind,
                timestamp_str=inputspec.timestamp_str))
        phantom.freshness = Freshness.PHANTOM
        return phantom

    def set_freshness(self, freshness):
        # phantom beads do not change freshness
        if self.freshness != Freshness.PHANTOM:
            self.freshness = freshness

    @property
    def is_not_phantom(self):
        return self.freshness != Freshness.PHANTOM

    def get_input_bead_name(self, input_nick):
        '''
        Returns the bead name on which update works.
        '''
        return self.input_map.get(input_nick, input_nick)

    def set_input_bead_name(self, input_nick, bead_name):
        '''
        Sets the bead name to be used for updates in the future.
        '''
        self.input_map[input_nick] = bead_name

    def __repr__(self):
        cls = self.__class__.__name__
        kind = self.kind[:8]
        content_id = self.content_id[:8]
        inputs = repr(self.inputs)
        return f"{cls}:{self.name}:{kind}:{content_id}:{self.freshness}:{inputs}:{self.input_map}"

Bead = TypeVar('Bead')

@attr.s(frozen=True, slots=True, auto_attribs=True)
class Ref:
    """
    Unique reference for Dummy-es.

    NOTE: Using multiple boxes can make this reference non-unique, as it is possible to
    have beads with same name and content, but with different input-maps.
    This could happen e.g. for sets of beads that are "branched" - released - and
    possibly need separate future maintenance.

    This potential non-unique-ness can be mitigated by having `update` search for updates
    in the same box, the workspace was developed from.
    """
    name: str
    content_id: str

    @classmethod
    def from_bead(cls, bead) -> 'Ref':
        return cls(bead.name, bead.content_id)

    @classmethod
    def from_bead_input(cls, bead, input: InputSpec) -> 'Ref':
        src_bead_name = bead.get_input_bead_name(input.name)
        return cls(src_bead_name, input.content_id)

    @classmethod
    def index_for(cls, beads: Iterable[Bead]) -> Dict['Ref', Bead]:
        bead_by_ref = {}
        for bead in beads:
            bead_by_ref[cls.from_bead(bead)] = bead
        return bead_by_ref
<|endoftext|>


<commit_before>import pygame
import numpy as np

# Add the data points to an array (create the wave)
def create_note(bits,sampling_rate,volume,freq,duration):
    """Creates a single note as a sine wave.
    Arguments:
    bits -- How many bits are used in the values, e.g., 16
    sampling_rate -- How many samples per second, e.g., 44100
    volume -- Volume, from 0 to 1 (max)
    freq -- The frequency of the note
    duration -- Duration of the note in seconds
    Returns:
    snd_array -- A list of sine wave values based on the current note
    """
    # Calculate the volume based on the used bit value (signed int)
    if volume<0: volume = 0
    if volume>1: volume = 1
    volume = pow(2,bits-1)-1
    # An empty list
    note = []
    # Fill the list with sine wave values
    for x in range(0,int(sampling_rate*duration)):
        value = volume * np.sin(2 * np.pi * freq * x / sampling_rate)
        note.append(value)
    return note

def create_melody(bits,sampling_rate,volumes,freqs,durations):
    """Creates and returns a melody consisting of one or more notes.
    Arguments:
    bits -- How many bits are used in the values, e.g., 16
    sampling_rate -- How many samples per second, e.g., 44100
    volumes -- A list of volumes for individual notes
    freqs -- A list of frequencies for individual notes
    durations -- Note durations in a list
    Returns:
    melody -- A list of wave values based on the current melody
    """
    melody = []
    # Go through the list
    for volume,freq,duration in zip(volumes,freqs,durations):
        # Create an individual note
        note = create_note(bits,sampling_rate,volume,freq,duration)
        # Add a note to the melody list
        melody.extend(note)
    return melody

def calculate_freq(note):
    """Calculates the frequency of a give note based on it's name and frequency of A4=440Hz.
    """
    notes = ['C4','C#4','D4','D#4','E4','F4','F#4','G4','G#4','A4','A#4','B4']
    baseind = notes.index('A4')
    noteind = notes.index(note)
    n = baseind - noteind
    a = 1.059463094359
    f0 = 440
    freq = f0 * pow(a,n)
    return freq

def main():
    # Set some variables
    sampling_rate = 8000
    bits = 16
    # In this case: mono
    channels = 1
    # Volumes from 0 to 1
    volumes = [0.5,0.5]
    # The notes to be played
    notes = ['C4','D4']
    # Note frequencies
    freqs = []
    for note in notes:
        freqs.append(calculate_freq(note))
    # Note durations in seconds
    durations = [1.0,2.0]
    # The duration this program is alive, right now the same as note duration
    wait_duration = sum(durations)

    # Initialize pygame mixer
    pygame.mixer.pre_init(sampling_rate, -bits, channels)
    pygame.init()

    # Create the wave
    melody = create_melody(bits,sampling_rate,volumes,freqs,durations)
    # Create a numpy array of the list, needed later.
    # Note: We don't create a numpy array earlier, because when
    # appending values to it, a new array is always created.
    # That is not efficient.
    melody = np.array(melody).astype(np.int16)
    # Get the sound based on the array
    sound = pygame.sndarray.make_sound(melody)
    # Play and loop
    sound.play()
    # Stop after <duration>
    pygame.time.delay(int(wait_duration*1000))
    # Stop playing
    sound.stop()

if __name__=="__main__":
    main()
<commit_msg>Fixed calculation of n in note freq calc function.<commit_after>import pygame
import numpy as np

# Add the data points to an array (create the wave)
def create_note(bits,sampling_rate,volume,freq,duration):
    """Creates a single note as a sine wave.
    Arguments:
    bits -- How many bits are used in the values, e.g., 16
    sampling_rate -- How many samples per second, e.g., 44100
    volume -- Volume, from 0 to 1 (max)
    freq -- The frequency of the note
    duration -- Duration of the note in seconds
    Returns:
    snd_array -- A list of sine wave values based on the current note
    """
    # Calculate the volume based on the used bit value (signed int)
    if volume<0: volume = 0
    if volume>1: volume = 1
    volume = pow(2,bits-1)-1
    # An empty list
    note = []
    # Fill the list with sine wave values
    for x in range(0,int(sampling_rate*duration)):
        value = volume * np.sin(2 * np.pi * freq * x / sampling_rate)
        note.append(value)
    return note

def create_melody(bits,sampling_rate,volumes,freqs,durations):
    """Creates and returns a melody consisting of one or more notes.
    Arguments:
    bits -- How many bits are used in the values, e.g., 16
    sampling_rate -- How many samples per second, e.g., 44100
    volumes -- A list of volumes for individual notes
    freqs -- A list of frequencies for individual notes
    durations -- Note durations in a list
    Returns:
    melody -- A list of wave values based on the current melody
    """
    melody = []
    # Go through the list
    for volume,freq,duration in zip(volumes,freqs,durations):
        # Create an individual note
        note = create_note(bits,sampling_rate,volume,freq,duration)
        # Add a note to the melody list
        melody.extend(note)
    return melody

def calculate_freq(note):
    """Calculates the frequency of a give note based on it's name and frequency of A4=440Hz.
    """
    notes = ['C4','C#4','D4','D#4','E4','F4','F#4','G4','G#4','A4','A#4','B4']
    baseind = notes.index('A4')
    noteind = notes.index(note)
    n = noteind - baseind
    a = 1.059463094359
    f0 = 440
    freq = f0 * pow(a,n)
    return freq

def main():
    # Set some variables
    sampling_rate = 8000
    bits = 16
    # In this case: mono
    channels = 1
    # Volumes from 0 to 1
    volumes = [0.5,0.5]
    # The notes to be played
    notes = ['C4','D4']
    # Note frequencies
    freqs = []
    for note in notes:
        freqs.append(calculate_freq(note))
    # Note durations in seconds
    durations = [1.0,2.0]
    # The duration this program is alive, right now the same as note duration
    wait_duration = sum(durations)

    # Initialize pygame mixer
    pygame.mixer.pre_init(sampling_rate, -bits, channels)
    pygame.init()

    # Create the wave
    melody = create_melody(bits,sampling_rate,volumes,freqs,durations)
    # Create a numpy array of the list, needed later.
    # Note: We don't create a numpy array earlier, because when
    # appending values to it, a new array is always created.
    # That is not efficient.
    melody = np.array(melody).astype(np.int16)
    # Get the sound based on the array
    sound = pygame.sndarray.make_sound(melody)
    # Play and loop
    sound.play()
    # Stop after <duration>
    pygame.time.delay(int(wait_duration*1000))
    # Stop playing
    sound.stop()

if __name__=="__main__":
    main()
<|endoftext|>