<commit_before>'''
Define high level widgets
'''
<commit_msg>Update widget core with Pazz's tree classes<commit_after># Copyright (C) 2013  Patrick Totzke <patricktotzke@gmail.com>
# This file is released under the LGPLv2

import urwid
from urwid import WidgetWrap, ListBox
from urwid import signals
from decoration import DecoratedTree, CollapseMixin
from nested import NestedTree
from lru_cache import lru_cache

# The following are used to check dynamically if a tree offers sub-APIs

def implementsDecorateAPI(tree):
    """determines if given tree offers line decoration"""
    return isinstance(tree, (DecoratedTree, NestedTree))

def implementsCollapseAPI(tree):
    """determines if given tree can collapse positions"""
    res = False
    if isinstance(tree, (CollapseMixin, NestedTree)):
        res = True
    return res

class FocusText(urwid.WidgetWrap):
    '''
    Selectable text for tree names
    '''
    def __init__(self, text):
        self.text = text
        urwid.WidgetWrap.__init__(
                self,
                self.__widget(self.text))

    def __widget(self, text):
        return urwid.AttrMap(
                urwid.Text(text),
                'body',
                'focus')

    def selectable(self):
        return True

    def keypress(self, size, key):
        return key

class TreeListWalker(urwid.ListWalker):
    """
    ListWalker to walk through a class:`Tree`.

    This translates a :class:`Tree` into a :class:`urwid.ListWalker` that is
    digestible by :class:`urwid.ListBox`.
    It uses :meth:`Tree.[next|prev]_position <Tree.next_position>` to determine
    the next/previous position in depth first order.
    """
    def __init__(self, tree, focus=None):
        """
        :param tree: the tree to be displayed
        :type tree: Tree
        :param focus: position of node to be focussed initially.
            This has to be a valid position in the Tree.
            It defaults to the value of `Tree.root`.
        """
        self._tree = tree
        self._focus = focus or tree.root
        self.root = tree.root

    @lru_cache()
    def __getitem__(self, pos):
        """gets (possibly decorated) line widget at given position"""
        if implementsDecorateAPI(self._tree):
            entry = self._tree.get_decorated(pos)
        else:
            entry = self._tree[pos]
        return entry

    def clear_cache(self):
        """removes all cached lines"""
        self.__getitem__.cache_clear()

    def _get(self, pos):
        """looks up widget for given position; handling invalid arguments"""
        res = None, None
        if pos is not None:
            try:
                res = self[pos], pos
            except (IndexError, KeyError):
                pass
        return res

    # List Walker API.
    def get_focus(self):
        return self._get(self._focus)

    def set_focus(self, pos):
        self._focus = pos

    def get_next(self, pos):
        return self._get(self._tree.next_position(pos))

    def get_prev(self, pos):
        return self._get(self._tree.prev_position(pos))

    def positions(self, reverse=False):
        """returns a generator that walks the tree's positions"""
        return self._tree.positions(reverse)
    # end of List Walker API

class TreeBox(WidgetWrap):
    """
    A widget that displays a given :class:`Tree`.
    This is essentially a :class:`ListBox` with the ability to move the focus
    based on directions in the Tree and to collapse/expand subtrees if
    possible.

    TreeBox interprets `left/right` as well as `page up/`page down` to move the
    focus to parent/first child and next/previous sibling respectively. All
    other keys are passed to the underlying ListBox.
    """

    def __init__(self, tree, focus=None):
        """
        :param tree: tree of widgets to be displayed.
        :type tree: Tree
        :param focus: initially focussed position
        """
        self._tree = tree
        self._walker = TreeListWalker(tree)
        self._outer_list = ListBox(self._walker)
        if focus is not None:
            self._outer_list.set_focus(focus)
        self.__super.__init__(self._outer_list)

    # Widget API
    def get_focus(self):
        return self._outer_list.get_focus()

    def set_focus(self, pos):
        return self._outer_list.set_focus(pos)

    def refresh(self):
        self._walker.clear_cache()
        signals.emit_signal(self._walker, "modified")

    def keypress(self, size, key):
        key = self._outer_list.keypress(size, key)
        if key in ['left', 'right', '[', ']', '-', '+', 'C', 'E', ]:
            if key == 'left':
                self.focus_parent()
            elif key == 'right':
                self.focus_first_child()
            elif key == '[':
                self.focus_prev_sibling()
            elif key == ']':
                self.focus_next_sibling()
            elif key == '-':
                self.collapse_focussed()
            elif key == '+':
                self.expand_focussed()
            elif key == 'C':
                self.collapse_all()
            elif key == 'E':
                self.expand_all()
            # This is a hack around ListBox misbehaving:
            # it seems impossible to set the focus without calling keypress as
            # otherwise the change becomes visible only after the next render()
            return self._outer_list.keypress(size, None)
        else:
            return self._outer_list.keypress(size, key)

    # Collapse operations
    def collapse_focussed(self):
        """
        Collapse currently focussed position; works only if the underlying
        tree allows it.
        """
        if implementsCollapseAPI(self._tree):
            w, focuspos = self.get_focus()
            self._tree.collapse(focuspos)
            self._walker.clear_cache()
            self.refresh()

    def expand_focussed(self):
        """
        Expand currently focussed position; works only if the underlying
        tree allows it.
        """
        if implementsCollapseAPI(self._tree):
            w, focuspos = self.get_focus()
            self._tree.expand(focuspos)
            self._walker.clear_cache()
            self.refresh()

    def collapse_all(self):
        """
        Collapse all positions; works only if the underlying tree allows it.
        """
        if implementsCollapseAPI(self._tree):
            self._tree.collapse_all()
            self.set_focus(self._tree.root)
            self._walker.clear_cache()
            self.refresh()

    def expand_all(self):
        """
        Expand all positions; works only if the underlying tree allows it.
        """
        if implementsCollapseAPI(self._tree):
            self._tree.expand_all()
            self._walker.clear_cache()
            self.refresh()

    # Tree based focus movement
    def focus_parent(self):
        """move focus to parent node of currently focussed one"""
        w, focuspos = self.get_focus()
        parent = self._tree.parent_position(focuspos)
        if parent is not None:
            self.set_focus(parent)

    def focus_first_child(self):
        """move focus to first child of currently focussed one"""
        w, focuspos = self.get_focus()
        child = self._tree.first_child_position(focuspos)
        if child is not None:
            self.set_focus(child)

    def focus_last_child(self):
        """move focus to last child of currently focussed one"""
        w, focuspos = self.get_focus()
        child = self._tree.last_child_position(focuspos)
        if child is not None:
            self.set_focus(child)

    def focus_next_sibling(self):
        """move focus to next sibling of currently focussed one"""
        w, focuspos = self.get_focus()
        sib = self._tree.next_sibling_position(focuspos)
        if sib is not None:
            self.set_focus(sib)

    def focus_prev_sibling(self):
        """move focus to previous sibling of currently focussed one"""
        w, focuspos = self.get_focus()
        sib = self._tree.prev_sibling_position(focuspos)
        if sib is not None:
            self.set_focus(sib)

    def focus_next(self):
        """move focus to next position (DFO)"""
        w, focuspos = self.get_focus()
        next = self._tree.next_position(focuspos)
        if next is not None:
            self.set_focus(next)

    def focus_prev(self):
        """move focus to previous position (DFO)"""
        w, focuspos = self.get_focus()
        prev = self._tree.prev_position(focuspos)
        if prev is not None:
            self.set_focus(prev)
<|endoftext|>


<commit_before>"""
This config file extends the test environment configuration
so that we can run the lettuce acceptance tests.
"""

# We intentionally define lots of variables that aren't used, and
# want to import all variables from base settings files
# pylint: disable=W0401, W0614

from .test import *

# You need to start the server in debug mode,
# otherwise the browser will not render the pages correctly
DEBUG = True

# Disable warnings for acceptance tests, to make the logs readable
import logging
logging.disable(logging.ERROR)

MODULESTORE_OPTIONS = {
    'default_class': 'xmodule.raw_module.RawDescriptor',
    'host': 'localhost',
    'db': 'test_xmodule',
    'collection': 'acceptance_modulestore',
    'fs_root': TEST_ROOT / "data",
    'render_template': 'mitxmako.shortcuts.render_to_string'
}

MODULESTORE = {
    'default': {
        'ENGINE': 'xmodule.modulestore.mongo.DraftMongoModuleStore',
        'OPTIONS': MODULESTORE_OPTIONS
    },
    'direct': {
        'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore',
        'OPTIONS': MODULESTORE_OPTIONS
    },
    'draft': {
        'ENGINE': 'xmodule.modulestore.mongo.DraftMongoModuleStore',
        'OPTIONS': MODULESTORE_OPTIONS
    }
}
# Set this up so that rake lms[acceptance] and running the
# harvest command both use the same (test) database
# which they can flush without messing up your dev db
DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': TEST_ROOT / "db" / "test_mitx.db",
        'TEST_NAME': TEST_ROOT / "db" / "test_mitx.db",
    }
}

# Include the lettuce app for acceptance testing, including the 'harvest' django-admin command
INSTALLED_APPS += ('lettuce.django',)
LETTUCE_APPS = ('contentstore',)
LETTUCE_SERVER_PORT = 8001
LETTUCE_BROWSER = 'chrome'
<commit_msg>Specify a different xcontent mongo db for the acceptance tests<commit_after>"""
This config file extends the test environment configuration
so that we can run the lettuce acceptance tests.
"""

# We intentionally define lots of variables that aren't used, and
# want to import all variables from base settings files
# pylint: disable=W0401, W0614

from .test import *

# You need to start the server in debug mode,
# otherwise the browser will not render the pages correctly
DEBUG = True

# Disable warnings for acceptance tests, to make the logs readable
import logging
logging.disable(logging.ERROR)

MODULESTORE_OPTIONS = {
    'default_class': 'xmodule.raw_module.RawDescriptor',
    'host': 'localhost',
    'db': 'test_xmodule',
    'collection': 'acceptance_modulestore',
    'fs_root': TEST_ROOT / "data",
    'render_template': 'mitxmako.shortcuts.render_to_string'
}

MODULESTORE = {
    'default': {
        'ENGINE': 'xmodule.modulestore.mongo.DraftMongoModuleStore',
        'OPTIONS': MODULESTORE_OPTIONS
    },
    'direct': {
        'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore',
        'OPTIONS': MODULESTORE_OPTIONS
    },
    'draft': {
        'ENGINE': 'xmodule.modulestore.mongo.DraftMongoModuleStore',
        'OPTIONS': MODULESTORE_OPTIONS
    }
}

CONTENTSTORE = {
    'ENGINE': 'xmodule.contentstore.mongo.MongoContentStore',
    'OPTIONS': {
        'host': 'localhost',
        'db': 'acceptance_xcontent',
    },
    # allow for additional options that can be keyed on a name, e.g. 'trashcan'
    'ADDITIONAL_OPTIONS': {
        'trashcan': {
            'bucket': 'trash_fs'
        }
    }
}

# Set this up so that rake lms[acceptance] and running the
# harvest command both use the same (test) database
# which they can flush without messing up your dev db
DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': TEST_ROOT / "db" / "test_mitx.db",
        'TEST_NAME': TEST_ROOT / "db" / "test_mitx.db",
    }
}

# Include the lettuce app for acceptance testing, including the 'harvest' django-admin command
INSTALLED_APPS += ('lettuce.django',)
LETTUCE_APPS = ('contentstore',)
LETTUCE_SERVER_PORT = 8001
LETTUCE_BROWSER = 'chrome'
<|endoftext|>


<commit_before>from cmsplugin_rst.models import RstPluginModel, rst_help_text
from django import forms

class RstPluginForm(forms.ModelForm):

    body = forms.CharField(
                widget=forms.Textarea(attrs={
                    'rows':30,
                    'cols':80,
                    'style':'font-family:monospace'
                }),
                help_text=rst_help_text
            )

    class Meta:
        model = RstPluginModel
        fields = ["name", "header_level", "body"]<commit_msg>Tweak too high textarea for RST text.<commit_after>from cmsplugin_rst.models import RstPluginModel, rst_help_text
from django import forms

class RstPluginForm(forms.ModelForm):

    body = forms.CharField(
                widget=forms.Textarea(attrs={
                    'rows':20,
                    'cols':80,
                    'style':'font-family:monospace'
                }),
                help_text=rst_help_text
            )

    class Meta:
        model = RstPluginModel
        fields = ["name", "header_level", "body"]<|endoftext|>


<commit_before>import logging

logger = logging.getLogger(__name__)

class QueryHandler(object):

    def build_query(self, case_id, query=None, variant_ids=None, category='snv'):
        """Build a mongo query

        These are the different query options:
            {
                'genetic_models': list,
                'thousand_genomes_frequency': float,
                'exac_frequency': float,
                'cadd_score': float,
                'cadd_inclusive": boolean,
                'genetic_models': list(str),
                'hgnc_symbols': list,
                'region_annotations': list,
                'functional_annotations': list,
                'clinsig': list,
                'clinsig_confident_always_returned': boolean,
                'variant_type': str(('research', 'clinical')),
                'chrom': str,
                'start': int,
                'end': int,
                'gene_panels': list(str),
            }

        Arguments:
            case_id(str)
            query(dict): A dictionary with querys for the database
            variant_ids(list(str)): A list of md5 variant ids

        Returns:
            mongo_query : A dictionary in the mongo query format

        """
        query = query or {}
        mongo_query = {}
        logger.info("Building a mongo query for %s" % case_id)
        mongo_query['case_id'] = case_id
        logger.debug("Querying category %s" % category)
        mongo_query['category'] = category

        # We need to check if there is any query specified in the input query
        mongo_query['variant_type'] = query.get('variant_type', 'clinical')
        logger.debug("Set variant type to %s", mongo_query['variant_type'])

        # Requests to filter based on gene panels, hgnc_symbols or
        # coordinate ranges must always be honored. They are always added to
        # query as top level, implicit '$and'.

        if query.get('hgnc_symbols') and query.get('gene_panels'):
            gene_query = [
                        {'hgnc_symbols': {'$in': query['hgnc_symbols']}},
                        {'panels': {'$in': query['gene_panels']}}
                    ]
            mongo_query['$or']=gene_query
        else:
            if query.get('hgnc_symbols'):
                hgnc_symbols = query['hgnc_symbols']
                mongo_query['hgnc_symbols'] = {'$in': hgnc_symbols}
                logger.debug("Adding hgnc_symbols: %s to query" %
                             ', '.join(hgnc_symbols))

            if query.get('gene_panels'):
                gene_panels = query['gene_panels']
                mongo_query['panels'] = {'$in': gene_panels}

        if query.get('chrom'):
            chromosome = query['chrom']
            mongo_query['chromosome'] = chromosome
            #Only check coordinates if there is a chromosome
            if (query.get('start') and query.get('end')):
                mongo_query['position'] = {
                                                '$lte': int(query['end'])
                                        }

                mongo_query['end'] = {
                                        '$gte': int(query['start'])
                                    }

        mongo_query_minor = []

        # A minor, excluding filter criteria will hide variants in general,
        # but can be overridden by an including, major filter criteria
        # such as a Pathogenic ClinSig.
        # If there are no major criteria given, all minor criteria are added as a
        # top level '$and' to the query.
        # If there is only one major criteria given without any minor, it will also be
        # added as a top level '$and'.
        # Otherwise, major criteria are added as a high level '$or' and all minor criteria
        # are joined together with them as a single lower level '$and'.

        mongo_query_minor = []

        if query.get('thousand_genomes_frequency') is not None:
            thousandg = query.get('thousand_genomes_frequency')
            if thousandg == '-1':
                mongo_query['thousand_genomes_frequency'] = {
                                                                '$exists': False
                                                            }

            else:
                # Replace comma with dot
                mongo_query_minor.append(
                    {
                        '$or': [
                            {
                                'thousand_genomes_frequency':
                                    {
                                        '$lt': float(thousandg)
                                    }
                                },
                            {
                                'thousand_genomes_frequency':
                                    {
                                        '$exists': False
                                    }
                            }
                        ]
                    }
                )
            logger.debug("Adding thousand_genomes_frequency to query")

        if query.get('exac_frequency') is not None:
            exac = query['exac_frequency']
            if exac == '-1':
                mongo_query['exac_frequency'] = {'$exists': False}
            else:
                mongo_query_minor.append({
                    '$or': [
                        {'exac_frequency': {'$lt': float(exac)}},
                        {'exac_frequency': {'$exists': False}}
                    ]
                })

        if query.get('local_obs') is not None:
            mongo_query_minor.append({
                '$or': [
                    {'local_obs_old': {'$exists': False}},
                    {'local_obs_old': {'$lt': query['local_obs'] + 1}},
                ]
            })

        if query.get('cadd_score') is not None:
            cadd = query['cadd_score']
            cadd_query = {'cadd_score': {'$gt': float(cadd)}}
            logger.debug("Adding cadd_score: %s to query" % cadd)

            if query.get('cadd_inclusive') == True:
                cadd_query = {
                    '$or': [
                        cadd_query,
                        {'cadd_score': {'$exists': False}}
                        ]}
                logger.debug("Adding cadd inclusive to query")

            mongo_query_minor.append(cadd_query)

        if query.get('genetic_models'):
            models = query['genetic_models']
            mongo_query_minor.append({'genetic_models': {'$in': models}})

            logger.debug("Adding genetic_models: %s to query" %
                         ', '.join(models))

        if query.get('functional_annotations'):
            functional = query['functional_annotations']
            mongo_query_minor.append({'genes.functional_annotation': {'$in': functional}})

            logger.debug("Adding functional_annotations %s to query" %
                         ', '.join(functional))

        if query.get('region_annotations'):
            region = query['region_annotations']
            mongo_query_minor.append({'genes.region_annotation': {'$in': region}})

            logger.debug("Adding region_annotations %s to query" %
                         ', '.join(region))

        if query.get('size'):
            size = query['size']
            size_query = {'length': {'$gt': int(size)}}
            logger.debug("Adding length: %s to query" % size)

            if query.get('size_inclusive') == 'yes':
                size_query = {
                    '$or': [
                        size_query,
                        {'length': {'$exists': False}}

                    ]}
                logger.debug("Adding size inclusive to query.")

            mongo_query_minor.append(size_query)

        if query.get('svtype'):
            svtype = query['svtype']
            mongo_query_minor.append({'sub_category': {'$in': svtype}})
            logger.debug("Adding SV_type %s to query" %
                         ', '.join(svtype))

        if query.get('depth'):
            logger.debug("add depth filter")
            mongo_query_minor.append({
                'tumor.read_depth': {
                    '$gt': query.get('depth'),
                }
            })

        if query.get('alt_count'):
            logger.debug("add min alt count filter")
            mongo_query_minor.append({
                'tumor.alt_depth': {
                    '$gt': query.get('alt_count'),
                }
            })

        if query.get('control_frequency'):
            logger.debug("add minimum control frequency filter")
            mongo_query_minor.append({
                'normal.alt_freq': {
                    '$lt': float(query.get('control_frequency')),
                }
            })

        mongo_query_major = None

        # Given a request to always return confident clinical variants,
        # add the clnsig query as a major criteria, but only
        # trust clnsig entries with trusted revstat levels.

        if query.get('clinsig'):
            rank = [int(item) for item in query['clinsig']]

            if query.get('clinsig_confident_always_returned') == True:

                trusted_revision_level = ['mult', 'single', 'exp', 'guideline']

                mongo_query_major = { "clnsig":
                        {
                            '$elemMatch': { 'value':
                                            { '$in': rank },
                                            'revstat':
                                            { '$in': trusted_revision_level }
                                          }
                         }
                    }

            else:
                logger.debug("add CLINSIG filter for rank: %s" %
                             ', '.join(str(query['clinsig'])))
                if mongo_query_minor:
                    mongo_query_minor.append({'clnsig.value': {'$in': rank}})
                else:
                    # if this is the only minor critera, use implicit and.
                    mongo_query['clnsig.value'] = {'$in': rank}

        if mongo_query_minor and mongo_query_major:
            mongo_query['$or'] = [ {'$and': mongo_query_minor }, mongo_query_major ]
        elif mongo_query_minor:
            mongo_query['$and'] = mongo_query_minor
        elif mongo_query_major:
            mongo_query['clnsig'] = mongo_query_major['clnsig']

        if variant_ids:
            mongo_query['variant_id'] = {'$in': variant_ids}

            logger.debug("Adding variant_ids %s to query" % ', '.join(variant_ids))

        logger.debug("mongo query: %s", mongo_query)

        return mongo_query
<commit_msg>use None filter instead of $exists, fix #652<commit_after>import logging

logger = logging.getLogger(__name__)

class QueryHandler(object):

    def build_query(self, case_id, query=None, variant_ids=None, category='snv'):
        """Build a mongo query

        These are the different query options:
            {
                'genetic_models': list,
                'thousand_genomes_frequency': float,
                'exac_frequency': float,
                'cadd_score': float,
                'cadd_inclusive": boolean,
                'genetic_models': list(str),
                'hgnc_symbols': list,
                'region_annotations': list,
                'functional_annotations': list,
                'clinsig': list,
                'clinsig_confident_always_returned': boolean,
                'variant_type': str(('research', 'clinical')),
                'chrom': str,
                'start': int,
                'end': int,
                'gene_panels': list(str),
            }

        Arguments:
            case_id(str)
            query(dict): A dictionary with querys for the database
            variant_ids(list(str)): A list of md5 variant ids

        Returns:
            mongo_query : A dictionary in the mongo query format

        """
        query = query or {}
        mongo_query = {}
        logger.info("Building a mongo query for %s" % case_id)
        mongo_query['case_id'] = case_id
        logger.debug("Querying category %s" % category)
        mongo_query['category'] = category

        # We need to check if there is any query specified in the input query
        mongo_query['variant_type'] = query.get('variant_type', 'clinical')
        logger.debug("Set variant type to %s", mongo_query['variant_type'])

        # Requests to filter based on gene panels, hgnc_symbols or
        # coordinate ranges must always be honored. They are always added to
        # query as top level, implicit '$and'.

        if query.get('hgnc_symbols') and query.get('gene_panels'):
            gene_query = [
                        {'hgnc_symbols': {'$in': query['hgnc_symbols']}},
                        {'panels': {'$in': query['gene_panels']}}
                    ]
            mongo_query['$or']=gene_query
        else:
            if query.get('hgnc_symbols'):
                hgnc_symbols = query['hgnc_symbols']
                mongo_query['hgnc_symbols'] = {'$in': hgnc_symbols}
                logger.debug("Adding hgnc_symbols: %s to query" %
                             ', '.join(hgnc_symbols))

            if query.get('gene_panels'):
                gene_panels = query['gene_panels']
                mongo_query['panels'] = {'$in': gene_panels}

        if query.get('chrom'):
            chromosome = query['chrom']
            mongo_query['chromosome'] = chromosome
            #Only check coordinates if there is a chromosome
            if (query.get('start') and query.get('end')):
                mongo_query['position'] = {
                                                '$lte': int(query['end'])
                                        }

                mongo_query['end'] = {
                                        '$gte': int(query['start'])
                                    }

        mongo_query_minor = []

        # A minor, excluding filter criteria will hide variants in general,
        # but can be overridden by an including, major filter criteria
        # such as a Pathogenic ClinSig.
        # If there are no major criteria given, all minor criteria are added as a
        # top level '$and' to the query.
        # If there is only one major criteria given without any minor, it will also be
        # added as a top level '$and'.
        # Otherwise, major criteria are added as a high level '$or' and all minor criteria
        # are joined together with them as a single lower level '$and'.

        mongo_query_minor = []

        if query.get('thousand_genomes_frequency') is not None:
            thousandg = query.get('thousand_genomes_frequency')
            if thousandg == '-1':
                mongo_query['thousand_genomes_frequency'] = {
                                                                '$exists': False
                                                            }

            else:
                # Replace comma with dot
                mongo_query_minor.append(
                    {
                        '$or': [
                            {
                                'thousand_genomes_frequency':
                                    {
                                        '$lt': float(thousandg)
                                    }
                                },
                            {
                                'thousand_genomes_frequency':
                                    {
                                        '$exists': False
                                    }
                            }
                        ]
                    }
                )
            logger.debug("Adding thousand_genomes_frequency to query")

        if query.get('exac_frequency') is not None:
            exac = query['exac_frequency']
            if exac == '-1':
                mongo_query['exac_frequency'] = {'$exists': False}
            else:
                mongo_query_minor.append({
                    '$or': [
                        {'exac_frequency': {'$lt': float(exac)}},
                        {'exac_frequency': {'$exists': False}}
                    ]
                })

        if query.get('local_obs') is not None:
            mongo_query_minor.append({
                '$or': [
                    {'local_obs_old': None},
                    {'local_obs_old': {'$lt': query['local_obs'] + 1}},
                ]
            })

        if query.get('cadd_score') is not None:
            cadd = query['cadd_score']
            cadd_query = {'cadd_score': {'$gt': float(cadd)}}
            logger.debug("Adding cadd_score: %s to query", cadd)

            if query.get('cadd_inclusive') is True:
                cadd_query = {
                    '$or': [
                        cadd_query,
                        {'cadd_score': {'$exists': False}}
                        ]}
                logger.debug("Adding cadd inclusive to query")

            mongo_query_minor.append(cadd_query)

        if query.get('genetic_models'):
            models = query['genetic_models']
            mongo_query_minor.append({'genetic_models': {'$in': models}})

            logger.debug("Adding genetic_models: %s to query" %
                         ', '.join(models))

        if query.get('functional_annotations'):
            functional = query['functional_annotations']
            mongo_query_minor.append({'genes.functional_annotation': {'$in': functional}})

            logger.debug("Adding functional_annotations %s to query" %
                         ', '.join(functional))

        if query.get('region_annotations'):
            region = query['region_annotations']
            mongo_query_minor.append({'genes.region_annotation': {'$in': region}})

            logger.debug("Adding region_annotations %s to query" %
                         ', '.join(region))

        if query.get('size'):
            size = query['size']
            size_query = {'length': {'$gt': int(size)}}
            logger.debug("Adding length: %s to query" % size)

            if query.get('size_inclusive') == 'yes':
                size_query = {
                    '$or': [
                        size_query,
                        {'length': {'$exists': False}}

                    ]}
                logger.debug("Adding size inclusive to query.")

            mongo_query_minor.append(size_query)

        if query.get('svtype'):
            svtype = query['svtype']
            mongo_query_minor.append({'sub_category': {'$in': svtype}})
            logger.debug("Adding SV_type %s to query" %
                         ', '.join(svtype))

        if query.get('depth'):
            logger.debug("add depth filter")
            mongo_query_minor.append({
                'tumor.read_depth': {
                    '$gt': query.get('depth'),
                }
            })

        if query.get('alt_count'):
            logger.debug("add min alt count filter")
            mongo_query_minor.append({
                'tumor.alt_depth': {
                    '$gt': query.get('alt_count'),
                }
            })

        if query.get('control_frequency'):
            logger.debug("add minimum control frequency filter")
            mongo_query_minor.append({
                'normal.alt_freq': {
                    '$lt': float(query.get('control_frequency')),
                }
            })

        mongo_query_major = None

        # Given a request to always return confident clinical variants,
        # add the clnsig query as a major criteria, but only
        # trust clnsig entries with trusted revstat levels.

        if query.get('clinsig'):
            rank = [int(item) for item in query['clinsig']]

            if query.get('clinsig_confident_always_returned') == True:

                trusted_revision_level = ['mult', 'single', 'exp', 'guideline']

                mongo_query_major = { "clnsig":
                        {
                            '$elemMatch': { 'value':
                                            { '$in': rank },
                                            'revstat':
                                            { '$in': trusted_revision_level }
                                          }
                         }
                    }

            else:
                logger.debug("add CLINSIG filter for rank: %s" %
                             ', '.join(str(query['clinsig'])))
                if mongo_query_minor:
                    mongo_query_minor.append({'clnsig.value': {'$in': rank}})
                else:
                    # if this is the only minor critera, use implicit and.
                    mongo_query['clnsig.value'] = {'$in': rank}

        if mongo_query_minor and mongo_query_major:
            mongo_query['$or'] = [ {'$and': mongo_query_minor }, mongo_query_major ]
        elif mongo_query_minor:
            mongo_query['$and'] = mongo_query_minor
        elif mongo_query_major:
            mongo_query['clnsig'] = mongo_query_major['clnsig']

        if variant_ids:
            mongo_query['variant_id'] = {'$in': variant_ids}

            logger.debug("Adding variant_ids %s to query" % ', '.join(variant_ids))

        logger.debug("mongo query: %s", mongo_query)

        return mongo_query
<|endoftext|>


<commit_before>import khmer
import numpy as np
from khmer.khmer_args import optimal_size
import os
import MinHash as MH
import pandas as pd
from multiprocessing import Pool  # Much faster without dummy (threading)
import multiprocessing
from itertools import *
import argparse

# Helper function that uses equations (2.1) and (2.7) that tells you where you need
# to set the threshold to ensure (with confidence 1-t) that you got all organisms
# with coverage >= c
def threshold_calc(k, c, p, confidence):
	delta = c*(1-np.sqrt(-2*np.log(1 - confidence)*(c+p) / float(c**2 * k)))
	if delta < 0:
		delta = 0
	return delta

# This will calculate the similarity indicies between one sketch and the sample NodeGraph
def compute_indicies(sketch, num_sample_kmers):
	#global sample_kmers
	num_hashes = len(sketch._kmers)
	num_training_kmers = sketch._true_num_kmers
	count = 0
	adjust = 0
	for kmer in sketch._kmers:
		if kmer != '':
			count += sample_kmers.get(kmer)
		else:
			adjust += 1  # Hash wasn't full, adjust accordingly
	intersection_cardinality = count  # Should adjust this by the FP rate of the NodGraph, but I don't think I can get that...
	containment_index = count / float(num_hashes - adjust)
	jaccard_index = num_training_kmers * containment_index / float(num_training_kmers + num_sample_kmers - num_training_kmers * containment_index)
	#print("Train %d sample %d" % (num_training_kmers, num_sample_kmers))
	# It can happen that the query file has less k-mers in it than the training file, so just round to nearest reasonable value
	if containment_index > 1:
		containment_index = 1
	elif containment_index < 0:
		containment_index = 0
	if jaccard_index > 1:
		jaccard_index = 1
	elif jaccard_index < 0:
		jaccard_index = 0
	return intersection_cardinality, containment_index, jaccard_index

def unwrap_compute_indicies(arg):
	return compute_indicies(*arg)

def restricted_float(x):
	x = float(x)
	if x < 0.0 or x >= 1.0:
		raise argparse.ArgumentTypeError("%r not in range [0.0, 1.0)"%(x,))
	return x

# Can read in with: test=pd.read_csv(os.path.abspath('../data/results.csv'),index_col=0)
def main():
	parser = argparse.ArgumentParser(description="This script creates a CSV file of similarity indicies between the"
									" input file and each of the sketches in the training/reference file.",
									formatter_class=argparse.ArgumentDefaultsHelpFormatter)
	parser.add_argument('-p', '--prime', help='Prime (for modding hashes)', default=9999999999971)
	parser.add_argument('-t', '--threads', type=int, help="Number of threads to use", default=multiprocessing.cpu_count())
	parser.add_argument('-n', '--num_hashes', type=int, help="Number of hashes to use.", default=500)
	parser.add_argument('-k', '--k_size', type=int, help="K-mer size", default=21)
	parser.add_argument('-f', '--force', action="store_true", help="Force creation of new NodeGraph.")
	parser.add_argument('-fp', '--fp_rate', type=restricted_float, help="False positive rate.", default=0.0001)
	parser.add_argument('-ct', '--containment_threshold', type=restricted_float,
						help="Only return results with containment index above this value", default=0.02)
	parser.add_argument('-c', '--confidence', type=restricted_float,
						help="Desired probability that all results were returned with containment index above threshold [-ct]",
						default=0.95)
	parser.add_argument('-ng', '--node_graph', help="NodeGraph/bloom filter location. Used if it exists; if not, one "
													"will be created and put in the same directory as the specified "
													"output CSV file.", default=None)
	parser.add_argument('in_file', help="Input file: FASTQ/A file (can be gzipped).")
	parser.add_argument('training_data', help="Training/reference data (HDF5 file created by MakeTrainingDatabase.py)")
	parser.add_argument('out_csv', help='Output CSV file')

	# Parse and check args
	args = parser.parse_args()
	ksize = args.k_size
	num_threads = args.threads
	query_file = os.path.abspath(args.in_file)
	if not os.path.exists(query_file):
		raise Exception("Query file %s does not exist." % query_file)
	# Node graph is stored in the output folder with name <InputFASTQ/A>.NodeGraph.K<k_size>
	if args.node_graph is None:  # If no node graph is specified, create one
		node_graph_out = os.path.join(os.path.dirname(os.path.abspath(args.out_csv)),
									os.path.basename(query_file) + ".NodeGraph.K" + str(ksize))
		if not os.path.exists(node_graph_out):  # Don't complain if the default location works
			print("Node graph not provided (via -ng). Creating one at: %s" % node_graph_out)
	elif os.path.exists(args.node_graph):  # If one is specified and it exists, use it
		node_graph_out = args.node_graph
	else:  # Otherwise, the specified one doesn't exist
		raise Exception("Provided NodeGraph %s does not exist." % args.node_graph)
	training_data = os.path.abspath(args.training_data)
	if not os.path.exists(training_data):
		raise Exception("Training/reference file %s does not exist." % training_data)
	results_file = os.path.abspath(args.out_csv)
	force = args.force
	fprate = args.fp_rate
	coverage_threshold = args.containment_threshold  # desired coverage cutoff
	confidence = args.confidence  # desired confidence that you got all the organisms with coverage >= desired coverage

	# Import all the training data
	sketches = MH.import_multiple_from_single_hdf5(training_data)
	# Check for issues with the sketches (can also check if all the kmers make sense (i.e. no '' or non-ACTG characters))
	num_hashes = len(sketches[0]._kmers)
	for i in range(len(sketches)):
		sketch = sketches[i]
		if len(sketch._kmers) != num_hashes:
			raise Exception("Unequal number of hashes for sketch of %s" % sketch.input_file_name)
		if sketch.ksize != ksize:
			raise Exception("Training/reference data uses K=%d while the value K=%d was requested. Culprit was %s." %
							(sketch.ksize, ksize, sketch.input_file_name))

	# Get names of training files for use as rows in returned tabular data
	training_file_names = []
	for i in range(len(sketches)):
		training_file_names.append(sketches[i].input_file_name)

	# Only form the Nodegraph if we need to
	global sample_kmers
	if not os.path.exists(node_graph_out) or force is True:
		hll = khmer.HLLCounter(0.01)
		hll.consume_seqfile(query_file)
		res = optimal_size(hll.estimate_cardinality(), fp_rate=fprate)
		sample_kmers = khmer.Nodegraph(ksize, res.htable_size, res.num_htables)
		sample_kmers.consume_seqfile(query_file)
		# Save the sample_kmers
		sample_kmers.save(node_graph_out)
	else:
		sample_kmers = khmer.load_nodegraph(node_graph_out)
		node_ksize = sample_kmers.ksize()
		if node_ksize != ksize:
			raise Exception("Node graph %s has wrong k-mer size of %d (input was %d). Try --force or change -k." % (
			node_graph_out, node_ksize, ksize))

	#num_sample_kmers = sample_kmers.n_unique_kmers()  # For some reason this only works when creating a new node graph, use the following instead
	num_sample_kmers = sample_kmers.n_occupied()

	# Compute all the indicies for all the training data
	pool = Pool(processes=num_threads)
	res = pool.map(unwrap_compute_indicies, zip(sketches, repeat(num_sample_kmers)))

	# Gather up the results in a nice form
	intersection_cardinalities = np.zeros(len(sketches))
	containment_indexes = np.zeros(len(sketches))
	jaccard_indexes = np.zeros(len(sketches))
	for i in range(len(res)):
		(intersection_cardinality, containment_index, jaccard_index) = res[i]
		intersection_cardinalities[i] = intersection_cardinality
		containment_indexes[i] = containment_index
		jaccard_indexes[i] = jaccard_index

	d = {'intersection': intersection_cardinalities, 'containment index': containment_indexes, 'jaccard index': jaccard_indexes}
	#df = pd.DataFrame(d, index=map(os.path.basename, training_file_names))
	df = pd.DataFrame(d, training_file_names)

	# Only get the rows above a certain threshold
	if coverage_threshold <= 0:
		est_threshold = 0
	else:
		est_threshold = threshold_calc(num_hashes, coverage_threshold, fprate, confidence)
	filtered_results = df[df['containment index'] > est_threshold].sort_values('containment index', ascending=False)
	# Export the results
	filtered_results.to_csv(results_file, index=True, encoding='utf-8')

if __name__ == "__main__":
	main()
<commit_msg>don't need to pass number of hashes to Query<commit_after>import khmer
import numpy as np
from khmer.khmer_args import optimal_size
import os
import MinHash as MH
import pandas as pd
from multiprocessing import Pool  # Much faster without dummy (threading)
import multiprocessing
from itertools import *
import argparse

# Helper function that uses equations (2.1) and (2.7) that tells you where you need
# to set the threshold to ensure (with confidence 1-t) that you got all organisms
# with coverage >= c
def threshold_calc(k, c, p, confidence):
	delta = c*(1-np.sqrt(-2*np.log(1 - confidence)*(c+p) / float(c**2 * k)))
	if delta < 0:
		delta = 0
	return delta

# This will calculate the similarity indicies between one sketch and the sample NodeGraph
def compute_indicies(sketch, num_sample_kmers):
	#global sample_kmers
	num_hashes = len(sketch._kmers)
	num_training_kmers = sketch._true_num_kmers
	count = 0
	adjust = 0
	for kmer in sketch._kmers:
		if kmer != '':
			count += sample_kmers.get(kmer)
		else:
			adjust += 1  # Hash wasn't full, adjust accordingly
	intersection_cardinality = count  # Should adjust this by the FP rate of the NodGraph, but I don't think I can get that...
	containment_index = count / float(num_hashes - adjust)
	jaccard_index = num_training_kmers * containment_index / float(num_training_kmers + num_sample_kmers - num_training_kmers * containment_index)
	#print("Train %d sample %d" % (num_training_kmers, num_sample_kmers))
	# It can happen that the query file has less k-mers in it than the training file, so just round to nearest reasonable value
	if containment_index > 1:
		containment_index = 1
	elif containment_index < 0:
		containment_index = 0
	if jaccard_index > 1:
		jaccard_index = 1
	elif jaccard_index < 0:
		jaccard_index = 0
	return intersection_cardinality, containment_index, jaccard_index

def unwrap_compute_indicies(arg):
	return compute_indicies(*arg)

def restricted_float(x):
	x = float(x)
	if x < 0.0 or x >= 1.0:
		raise argparse.ArgumentTypeError("%r not in range [0.0, 1.0)"%(x,))
	return x

# Can read in with: test=pd.read_csv(os.path.abspath('../data/results.csv'),index_col=0)
def main():
	parser = argparse.ArgumentParser(description="This script creates a CSV file of similarity indicies between the"
									" input file and each of the sketches in the training/reference file.",
									formatter_class=argparse.ArgumentDefaultsHelpFormatter)
	parser.add_argument('-p', '--prime', help='Prime (for modding hashes)', default=9999999999971)
	parser.add_argument('-t', '--threads', type=int, help="Number of threads to use", default=multiprocessing.cpu_count())
	#parser.add_argument('-n', '--num_hashes', type=int, help="Number of hashes to use.", default=500)
	parser.add_argument('-k', '--k_size', type=int, help="K-mer size", default=21)
	parser.add_argument('-f', '--force', action="store_true", help="Force creation of new NodeGraph.")
	parser.add_argument('-fp', '--fp_rate', type=restricted_float, help="False positive rate.", default=0.0001)
	parser.add_argument('-ct', '--containment_threshold', type=restricted_float,
						help="Only return results with containment index above this value", default=0.02)
	parser.add_argument('-c', '--confidence', type=restricted_float,
						help="Desired probability that all results were returned with containment index above threshold [-ct]",
						default=0.95)
	parser.add_argument('-ng', '--node_graph', help="NodeGraph/bloom filter location. Used if it exists; if not, one "
													"will be created and put in the same directory as the specified "
													"output CSV file.", default=None)
	parser.add_argument('in_file', help="Input file: FASTQ/A file (can be gzipped).")
	parser.add_argument('training_data', help="Training/reference data (HDF5 file created by MakeTrainingDatabase.py)")
	parser.add_argument('out_csv', help='Output CSV file')

	# Parse and check args
	args = parser.parse_args()
	ksize = args.k_size
	num_threads = args.threads
	query_file = os.path.abspath(args.in_file)
	if not os.path.exists(query_file):
		raise Exception("Query file %s does not exist." % query_file)
	# Node graph is stored in the output folder with name <InputFASTQ/A>.NodeGraph.K<k_size>
	if args.node_graph is None:  # If no node graph is specified, create one
		node_graph_out = os.path.join(os.path.dirname(os.path.abspath(args.out_csv)),
									os.path.basename(query_file) + ".NodeGraph.K" + str(ksize))
		if not os.path.exists(node_graph_out):  # Don't complain if the default location works
			print("Node graph not provided (via -ng). Creating one at: %s" % node_graph_out)
	elif os.path.exists(args.node_graph):  # If one is specified and it exists, use it
		node_graph_out = args.node_graph
	else:  # Otherwise, the specified one doesn't exist
		raise Exception("Provided NodeGraph %s does not exist." % args.node_graph)
	training_data = os.path.abspath(args.training_data)
	if not os.path.exists(training_data):
		raise Exception("Training/reference file %s does not exist." % training_data)
	results_file = os.path.abspath(args.out_csv)
	force = args.force
	fprate = args.fp_rate
	coverage_threshold = args.containment_threshold  # desired coverage cutoff
	confidence = args.confidence  # desired confidence that you got all the organisms with coverage >= desired coverage

	# Import all the training data
	sketches = MH.import_multiple_from_single_hdf5(training_data)
	# Check for issues with the sketches (can also check if all the kmers make sense (i.e. no '' or non-ACTG characters))
	num_hashes = len(sketches[0]._kmers)
	for i in range(len(sketches)):
		sketch = sketches[i]
		if len(sketch._kmers) != num_hashes:
			raise Exception("Unequal number of hashes for sketch of %s" % sketch.input_file_name)
		if sketch.ksize != ksize:
			raise Exception("Training/reference data uses K=%d while the value K=%d was requested. Culprit was %s." %
							(sketch.ksize, ksize, sketch.input_file_name))

	# Get names of training files for use as rows in returned tabular data
	training_file_names = []
	for i in range(len(sketches)):
		training_file_names.append(sketches[i].input_file_name)

	# Only form the Nodegraph if we need to
	global sample_kmers
	if not os.path.exists(node_graph_out) or force is True:
		hll = khmer.HLLCounter(0.01)
		hll.consume_seqfile(query_file)
		res = optimal_size(hll.estimate_cardinality(), fp_rate=fprate)
		sample_kmers = khmer.Nodegraph(ksize, res.htable_size, res.num_htables)
		sample_kmers.consume_seqfile(query_file)
		# Save the sample_kmers
		sample_kmers.save(node_graph_out)
	else:
		sample_kmers = khmer.load_nodegraph(node_graph_out)
		node_ksize = sample_kmers.ksize()
		if node_ksize != ksize:
			raise Exception("Node graph %s has wrong k-mer size of %d (input was %d). Try --force or change -k." % (
			node_graph_out, node_ksize, ksize))

	#num_sample_kmers = sample_kmers.n_unique_kmers()  # For some reason this only works when creating a new node graph, use the following instead
	num_sample_kmers = sample_kmers.n_occupied()

	# Compute all the indicies for all the training data
	pool = Pool(processes=num_threads)
	res = pool.map(unwrap_compute_indicies, zip(sketches, repeat(num_sample_kmers)))

	# Gather up the results in a nice form
	intersection_cardinalities = np.zeros(len(sketches))
	containment_indexes = np.zeros(len(sketches))
	jaccard_indexes = np.zeros(len(sketches))
	for i in range(len(res)):
		(intersection_cardinality, containment_index, jaccard_index) = res[i]
		intersection_cardinalities[i] = intersection_cardinality
		containment_indexes[i] = containment_index
		jaccard_indexes[i] = jaccard_index

	d = {'intersection': intersection_cardinalities, 'containment index': containment_indexes, 'jaccard index': jaccard_indexes}
	#df = pd.DataFrame(d, index=map(os.path.basename, training_file_names))
	df = pd.DataFrame(d, training_file_names)

	# Only get the rows above a certain threshold
	if coverage_threshold <= 0:
		est_threshold = 0
	else:
		est_threshold = threshold_calc(num_hashes, coverage_threshold, fprate, confidence)
	filtered_results = df[df['containment index'] > est_threshold].sort_values('containment index', ascending=False)
	# Export the results
	filtered_results.to_csv(results_file, index=True, encoding='utf-8')

if __name__ == "__main__":
	main()
<|endoftext|>


<commit_before>import khmer
import numpy as np
from khmer.khmer_args import optimal_size
import os
import MinHash as MH
import pandas as pd
from multiprocessing import Pool  # Much faster without dummy (threading)
import multiprocessing
from itertools import *
import argparse

# Helper function that uses equations (2.1) and (2.7) that tells you where you need
# to set the threshold to ensure (with confidence 1-t) that you got all organisms
# with coverage >= c
def threshold_calc(k, c, p, confidence):
	delta = c*(1-np.sqrt(-2*np.log(1 - confidence)*(c+p) / float(c**2 * k)))
	if delta < 0:
		delta = 0
	return delta

# This will calculate the similarity indicies between one sketch and the sample NodeGraph
def compute_indicies(sketch, num_sample_kmers):
	#global sample_kmers
	num_hashes = len(sketch._kmers)
	num_training_kmers = sketch._true_num_kmers
	count = 0
	adjust = 0
	for kmer in sketch._kmers:
		if kmer != '':
			count += sample_kmers.get(kmer)
		else:
			adjust += 1  # Hash wasn't full, adjust accordingly
	intersection_cardinality = count  # Should adjust this by the FP rate of the NodGraph, but I don't think I can get that...
	containment_index = count / float(num_hashes - adjust)
	jaccard_index = num_training_kmers * containment_index / float(num_training_kmers + num_sample_kmers - num_training_kmers * containment_index)
	#print("Train %d sample %d" % (num_training_kmers, num_sample_kmers))
	# It can happen that the query file has less k-mers in it than the training file, so just round to nearest reasonable value
	if containment_index > 1:
		containment_index = 1
	elif containment_index < 0:
		containment_index = 0
	if jaccard_index > 1:
		jaccard_index = 1
	elif jaccard_index < 0:
		jaccard_index = 0
	return intersection_cardinality, containment_index, jaccard_index

def unwrap_compute_indicies(arg):
	return compute_indicies(*arg)

def restricted_float(x):
	x = float(x)
	if x < 0.0 or x >= 1.0:
		raise argparse.ArgumentTypeError("%r not in range [0.0, 1.0)" % (x,))
	return x

# Can read in with: test=pd.read_csv(os.path.abspath('../data/results.csv'),index_col=0)
def main():
	parser = argparse.ArgumentParser(description="This script creates a CSV file of similarity indicies between the"
									" input file and each of the sketches in the training/reference file.",
									formatter_class=argparse.ArgumentDefaultsHelpFormatter)
	#parser.add_argument('-p', '--prime', help='Prime (for modding hashes)', default=9999999999971)
	parser.add_argument('-t', '--threads', type=int, help="Number of threads to use", default=multiprocessing.cpu_count())
	#parser.add_argument('-n', '--num_hashes', type=int, help="Number of hashes to use.", default=500)
	#parser.add_argument('-k', '--k_size', type=int, help="K-mer size", default=21)
	parser.add_argument('-f', '--force', action="store_true", help="Force creation of new NodeGraph.")
	parser.add_argument('-fp', '--fp_rate', type=restricted_float, help="False positive rate.", default=0.0001)
	parser.add_argument('-ct', '--containment_threshold', type=restricted_float,
						help="Only return results with containment index above this value", default=0.02)
	parser.add_argument('-c', '--confidence', type=restricted_float,
						help="Desired probability that all results were returned with containment index above threshold [-ct]",
						default=0.95)
	parser.add_argument('-ng', '--node_graph', help="NodeGraph/bloom filter location. Used if it exists; if not, one "
													"will be created and put in the same directory as the specified "
													"output CSV file.", default=None)
	parser.add_argument('in_file', help="Input file: FASTQ/A file (can be gzipped).")
	parser.add_argument('training_data', help="Training/reference data (HDF5 file created by MakeTrainingDatabase.py)")
	parser.add_argument('out_csv', help='Output CSV file')

	# Parse and check args
	args = parser.parse_args()
	#ksize = args.k_size
	training_data = os.path.abspath(args.training_data)
	if not os.path.exists(training_data):
		raise Exception("Training/reference file %s does not exist." % training_data)
	# Let's get the k-mer sizes in the training database
	ksizes = set()
	# Import all the training data
	sketches = MH.import_multiple_from_single_hdf5(training_data)
	# Check for issues with the sketches (can also check if all the kmers make sense (i.e. no '' or non-ACTG characters))
	if sketches[0]._kmers is None:
		raise Exception("For some reason, the k-mers were not saved when the database was created. Try running MakeDNADatabase.py again.")
	num_hashes = len(sketches[0]._kmers)
	for i in range(len(sketches)):
		sketch = sketches[i]
		if sketch._kmers is None:
			raise Exception(
				"For some reason, the k-mers were not saved when the database was created. Try running MakeDNADatabase.py again.")
		if len(sketch._kmers) != num_hashes:
			raise Exception("Unequal number of hashes for sketch of %s" % sketch.input_file_name)
		ksizes.add(sketch.ksize)
		if len(ksizes) > 1:
			raise Exception("Training/reference data uses different k-mer sizes. Culprit was %s." % (sketch.input_file_name))
	# Get the appropriate k-mer size
	ksize = ksizes.pop()
	num_threads = args.threads
	query_file = os.path.abspath(args.in_file)
	if not os.path.exists(query_file):
		raise Exception("Query file %s does not exist." % query_file)
	# Node graph is stored in the output folder with name <InputFASTQ/A>.NodeGraph.K<k_size>
	if args.node_graph is None:  # If no node graph is specified, create one
		node_graph_out = os.path.join(os.path.dirname(os.path.abspath(args.out_csv)),
									os.path.basename(query_file) + ".NodeGraph.K" + str(ksize))
		if not os.path.exists(node_graph_out):  # Don't complain if the default location works
			print("Node graph not provided (via -ng). Creating one at: %s" % node_graph_out)
	elif os.path.exists(args.node_graph):  # If one is specified and it exists, use it
		node_graph_out = args.node_graph
	else:  # Otherwise, the specified one doesn't exist
		raise Exception("Provided NodeGraph %s does not exist." % args.node_graph)
	results_file = os.path.abspath(args.out_csv)
	force = args.force
	fprate = args.fp_rate
	coverage_threshold = args.containment_threshold  # desired coverage cutoff
	confidence = args.confidence  # desired confidence that you got all the organisms with coverage >= desired coverage

	# Get names of training files for use as rows in returned tabular data
	training_file_names = []
	for i in range(len(sketches)):
		training_file_names.append(sketches[i].input_file_name)

	# Only form the Nodegraph if we need to
	global sample_kmers
	if not os.path.exists(node_graph_out) or force is True:
		hll = khmer.HLLCounter(0.01)
		hll.consume_seqfile(query_file)
		res = optimal_size(hll.estimate_cardinality(), fp_rate=fprate)
		sample_kmers = khmer.Nodegraph(ksize, res.htable_size, res.num_htables)
		sample_kmers.consume_seqfile(query_file)
		# Save the sample_kmers
		sample_kmers.save(node_graph_out)
	else:
		sample_kmers = khmer.load_nodegraph(node_graph_out)
		node_ksize = sample_kmers.ksize()
		if node_ksize != ksize:
			raise Exception("Node graph %s has wrong k-mer size of %d (input was %d). Try --force or change -k." % (
			node_graph_out, node_ksize, ksize))

	#num_sample_kmers = sample_kmers.n_unique_kmers()  # For some reason this only works when creating a new node graph, use the following instead
	num_sample_kmers = sample_kmers.n_occupied()

	# Compute all the indicies for all the training data
	pool = Pool(processes=num_threads)
	res = pool.map(unwrap_compute_indicies, zip(sketches, repeat(num_sample_kmers)))

	# Gather up the results in a nice form
	intersection_cardinalities = np.zeros(len(sketches))
	containment_indexes = np.zeros(len(sketches))
	jaccard_indexes = np.zeros(len(sketches))
	for i in range(len(res)):
		(intersection_cardinality, containment_index, jaccard_index) = res[i]
		intersection_cardinalities[i] = intersection_cardinality
		containment_indexes[i] = containment_index
		jaccard_indexes[i] = jaccard_index

	d = {'intersection': intersection_cardinalities, 'containment index': containment_indexes, 'jaccard index': jaccard_indexes}
	#df = pd.DataFrame(d, index=map(os.path.basename, training_file_names))
	df = pd.DataFrame(d, training_file_names)

	# Only get the rows above a certain threshold
	if coverage_threshold <= 0:
		est_threshold = 0
	else:
		est_threshold = threshold_calc(num_hashes, coverage_threshold, fprate, confidence)
	filtered_results = df[df['containment index'] > est_threshold].sort_values('containment index', ascending=False)
	# Export the results
	filtered_results.to_csv(results_file, index=True, encoding='utf-8')

if __name__ == "__main__":
	main()
<commit_msg>include full paths or just basenames optionally in output file<commit_after>import khmer
import numpy as np
from khmer.khmer_args import optimal_size
import os
import MinHash as MH
import pandas as pd
from multiprocessing import Pool  # Much faster without dummy (threading)
import multiprocessing
from itertools import *
import argparse

# Helper function that uses equations (2.1) and (2.7) that tells you where you need
# to set the threshold to ensure (with confidence 1-t) that you got all organisms
# with coverage >= c
def threshold_calc(k, c, p, confidence):
	delta = c*(1-np.sqrt(-2*np.log(1 - confidence)*(c+p) / float(c**2 * k)))
	if delta < 0:
		delta = 0
	return delta

# This will calculate the similarity indicies between one sketch and the sample NodeGraph
def compute_indicies(sketch, num_sample_kmers):
	#global sample_kmers
	num_hashes = len(sketch._kmers)
	num_training_kmers = sketch._true_num_kmers
	count = 0
	adjust = 0
	for kmer in sketch._kmers:
		if kmer != '':
			count += sample_kmers.get(kmer)
		else:
			adjust += 1  # Hash wasn't full, adjust accordingly
	intersection_cardinality = count  # Should adjust this by the FP rate of the NodGraph, but I don't think I can get that...
	containment_index = count / float(num_hashes - adjust)
	jaccard_index = num_training_kmers * containment_index / float(num_training_kmers + num_sample_kmers - num_training_kmers * containment_index)
	#print("Train %d sample %d" % (num_training_kmers, num_sample_kmers))
	# It can happen that the query file has less k-mers in it than the training file, so just round to nearest reasonable value
	if containment_index > 1:
		containment_index = 1
	elif containment_index < 0:
		containment_index = 0
	if jaccard_index > 1:
		jaccard_index = 1
	elif jaccard_index < 0:
		jaccard_index = 0
	return intersection_cardinality, containment_index, jaccard_index

def unwrap_compute_indicies(arg):
	return compute_indicies(*arg)

def restricted_float(x):
	x = float(x)
	if x < 0.0 or x >= 1.0:
		raise argparse.ArgumentTypeError("%r not in range [0.0, 1.0)" % (x,))
	return x

# Can read in with: test=pd.read_csv(os.path.abspath('../data/results.csv'),index_col=0)
def main():
	parser = argparse.ArgumentParser(description="This script creates a CSV file of similarity indicies between the"
									" input file and each of the sketches in the training/reference file.",
									formatter_class=argparse.ArgumentDefaultsHelpFormatter)
	#parser.add_argument('-p', '--prime', help='Prime (for modding hashes)', default=9999999999971)
	parser.add_argument('-t', '--threads', type=int, help="Number of threads to use", default=multiprocessing.cpu_count())
	#parser.add_argument('-n', '--num_hashes', type=int, help="Number of hashes to use.", default=500)
	#parser.add_argument('-k', '--k_size', type=int, help="K-mer size", default=21)
	parser.add_argument('-f', '--force', action="store_true", help="Force creation of new NodeGraph.")
	parser.add_argument('-fp', '--fp_rate', type=restricted_float, help="False positive rate.", default=0.0001)
	parser.add_argument('-ct', '--containment_threshold', type=restricted_float,
						help="Only return results with containment index above this value", default=0.02)
	parser.add_argument('-c', '--confidence', type=restricted_float,
						help="Desired probability that all results were returned with containment index above threshold [-ct]",
						default=0.95)
	parser.add_argument('-ng', '--node_graph', help="NodeGraph/bloom filter location. Used if it exists; if not, one "
													"will be created and put in the same directory as the specified "
													"output CSV file.", default=None)
	parser.add_argument('-b', '--base_name', action="store_true", help="Flag to indicate that only the base names (not the full path) should be saved in the output CSV file")
	parser.add_argument('in_file', help="Input file: FASTQ/A file (can be gzipped).")
	parser.add_argument('training_data', help="Training/reference data (HDF5 file created by MakeTrainingDatabase.py)")
	parser.add_argument('out_csv', help='Output CSV file')

	# Parse and check args
	args = parser.parse_args()
	#ksize = args.k_size
	base_name = args.base_name
	training_data = os.path.abspath(args.training_data)
	if not os.path.exists(training_data):
		raise Exception("Training/reference file %s does not exist." % training_data)
	# Let's get the k-mer sizes in the training database
	ksizes = set()
	# Import all the training data
	sketches = MH.import_multiple_from_single_hdf5(training_data)
	# Check for issues with the sketches (can also check if all the kmers make sense (i.e. no '' or non-ACTG characters))
	if sketches[0]._kmers is None:
		raise Exception("For some reason, the k-mers were not saved when the database was created. Try running MakeDNADatabase.py again.")
	num_hashes = len(sketches[0]._kmers)
	for i in range(len(sketches)):
		sketch = sketches[i]
		if sketch._kmers is None:
			raise Exception(
				"For some reason, the k-mers were not saved when the database was created. Try running MakeDNADatabase.py again.")
		if len(sketch._kmers) != num_hashes:
			raise Exception("Unequal number of hashes for sketch of %s" % sketch.input_file_name)
		ksizes.add(sketch.ksize)
		if len(ksizes) > 1:
			raise Exception("Training/reference data uses different k-mer sizes. Culprit was %s." % (sketch.input_file_name))
	# Get the appropriate k-mer size
	ksize = ksizes.pop()
	num_threads = args.threads
	query_file = os.path.abspath(args.in_file)
	if not os.path.exists(query_file):
		raise Exception("Query file %s does not exist." % query_file)
	# Node graph is stored in the output folder with name <InputFASTQ/A>.NodeGraph.K<k_size>
	if args.node_graph is None:  # If no node graph is specified, create one
		node_graph_out = os.path.join(os.path.dirname(os.path.abspath(args.out_csv)),
									os.path.basename(query_file) + ".NodeGraph.K" + str(ksize))
		if not os.path.exists(node_graph_out):  # Don't complain if the default location works
			print("Node graph not provided (via -ng). Creating one at: %s" % node_graph_out)
	elif os.path.exists(args.node_graph):  # If one is specified and it exists, use it
		node_graph_out = args.node_graph
	else:  # Otherwise, the specified one doesn't exist
		raise Exception("Provided NodeGraph %s does not exist." % args.node_graph)
	results_file = os.path.abspath(args.out_csv)
	force = args.force
	fprate = args.fp_rate
	coverage_threshold = args.containment_threshold  # desired coverage cutoff
	confidence = args.confidence  # desired confidence that you got all the organisms with coverage >= desired coverage

	# Get names of training files for use as rows in returned tabular data
	training_file_names = []
	for i in range(len(sketches)):
		training_file_names.append(sketches[i].input_file_name)

	# Only form the Nodegraph if we need to
	global sample_kmers
	if not os.path.exists(node_graph_out) or force is True:
		hll = khmer.HLLCounter(0.01)
		hll.consume_seqfile(query_file)
		res = optimal_size(hll.estimate_cardinality(), fp_rate=fprate)
		sample_kmers = khmer.Nodegraph(ksize, res.htable_size, res.num_htables)
		sample_kmers.consume_seqfile(query_file)
		# Save the sample_kmers
		sample_kmers.save(node_graph_out)
	else:
		sample_kmers = khmer.load_nodegraph(node_graph_out)
		node_ksize = sample_kmers.ksize()
		if node_ksize != ksize:
			raise Exception("Node graph %s has wrong k-mer size of %d (input was %d). Try --force or change -k." % (
			node_graph_out, node_ksize, ksize))

	#num_sample_kmers = sample_kmers.n_unique_kmers()  # For some reason this only works when creating a new node graph, use the following instead
	num_sample_kmers = sample_kmers.n_occupied()

	# Compute all the indicies for all the training data
	pool = Pool(processes=num_threads)
	res = pool.map(unwrap_compute_indicies, zip(sketches, repeat(num_sample_kmers)))

	# Gather up the results in a nice form
	intersection_cardinalities = np.zeros(len(sketches))
	containment_indexes = np.zeros(len(sketches))
	jaccard_indexes = np.zeros(len(sketches))
	for i in range(len(res)):
		(intersection_cardinality, containment_index, jaccard_index) = res[i]
		intersection_cardinalities[i] = intersection_cardinality
		containment_indexes[i] = containment_index
		jaccard_indexes[i] = jaccard_index

	d = {'intersection': intersection_cardinalities, 'containment index': containment_indexes, 'jaccard index': jaccard_indexes}
	#df = pd.DataFrame(d, index=map(os.path.basename, training_file_names))
	# Use only the basenames to label the rows (if requested)
	if base_name is True:
		df = pd.DataFrame(d, map(os.path.basename, training_file_names))
	else:
		df = pd.DataFrame(d, training_file_names)

	# Only get the rows above a certain threshold
	if coverage_threshold <= 0:
		est_threshold = 0
	else:
		est_threshold = threshold_calc(num_hashes, coverage_threshold, fprate, confidence)
	filtered_results = df[df['containment index'] > est_threshold].sort_values('containment index', ascending=False)
	# Export the results
	filtered_results.to_csv(results_file, index=True, encoding='utf-8')

if __name__ == "__main__":
	main()
<|endoftext|>


<commit_before>import sys
sys.path.append('../functions')

import time
import base64

from flask import Flask, redirect, url_for, request, render_template
from flask_socketio import SocketIO, emit

import runPipeline as run

#generate the app object
app = Flask(__name__)

#configure the upload settings
UPLOAD_FOLDER = '../../data'
ALLOWED_EXTENSIONS = 'tif'
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER

#generate the async handler
socketio = SocketIO(app)

def allowedFile(filename):
    return '.' in filename and filename.split('.')[1].lower() == ALLOWED_EXTENSIONS

@app.route('/')
def index():
    if request.method == 'GET':
        return render_template('index.html')

@socketio.on('upload', namespace='/')
def upload(message):
    #generate a unique id based on time of upload
    myID = str(time.time())[3:]

    #save the file
    rawDatList = message['file']
    for tp, rawDat in enumerate(rawDatList):
        fileDat = base64.b64decode(rawDat)
        with open(myID+'_' + str(tp) + '.tiff', 'wb') as f:
            f.write(fileDat)

    #emit a response on successful completion
    socketio.emit('response', {myID:myID})

@socketio.on('analyze', namespace='/')
def analyze(message):
    run.runPipeline(message['myID'])
    socketio.emit('complete',{})

@app.route('/results', methods=['GET']):
    myID = 'static/results/' + str(request.headers['myID']) + '.html'
    return render_template('results.html', myID=myID)

if __name__ == '__main__':
    socketio.run(app)
<commit_msg>added function signature<commit_after>import sys
sys.path.append('../functions')

import time
import base64

from flask import Flask, redirect, url_for, request, render_template
from flask_socketio import SocketIO, emit

import runPipeline as run

#generate the app object
app = Flask(__name__)

#configure the upload settings
UPLOAD_FOLDER = '../../data'
ALLOWED_EXTENSIONS = 'tif'
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER

#generate the async handler
socketio = SocketIO(app)

def allowedFile(filename):
    return '.' in filename and filename.split('.')[1].lower() == ALLOWED_EXTENSIONS

@app.route('/')
def index():
    if request.method == 'GET':
        return render_template('index.html')

@socketio.on('upload', namespace='/')
def upload(message):
    #generate a unique id based on time of upload
    myID = str(time.time())[3:]

    #save the file
    rawDatList = message['file']
    for tp, rawDat in enumerate(rawDatList):
        fileDat = base64.b64decode(rawDat)
        with open(myID+'_' + str(tp) + '.tiff', 'wb') as f:
            f.write(fileDat)

    #emit a response on successful completion
    socketio.emit('response', {myID:myID})

@socketio.on('analyze', namespace='/')
def analyze(message):
    run.runPipeline(message['myID'])
    socketio.emit('complete',{})

@app.route('/results', methods=['GET'])
def results():
    myID = 'static/results/' + str(request.headers['myID']) + '.html'
    return render_template('results.html', myID=myID)

if __name__ == '__main__':
    socketio.run(app)
<|endoftext|>


<commit_before>import nibabel as nib
import matplotlib.pyplot as plt
import numpy as np
import pickle
import pandas as pd

# Takes in a dictionary with (K, V) pair (subxxx_runyyy, modified 4D Array)
# Returns the masked with time collapsed and vol_mean
def generateMaskedBrain(allFiguresDict):
	maskedDict = {}
	volumeDict = {}
	for key, value in allFiguresDict.iteritems():
		data = value
		vol_mean = np.mean(data, axis = -1)
		percentile = np.percentile(np.ravel(vol_mean), q = 80)
		data[vol_mean > percentile, :] = True
		data[vol_mean < percentile, :] = False
		maskedDict[key] = vol_mean > percentile
		volumeDict[key] = vol_mean
	return maskedDict, volumeDict

#def generateMaskedBrain(paths):
	#for path in paths:
		#img = nib.load(path + '/filtered_func_data_mni.nii.gz')
		#print(img.shape)
		#data = img.get_data()

		#vol_shape = data.shape[:-1]
		#n_trs = data.shape[-1]
		#print(vol_shape, n_trs)
		#mean_vol = np.mean(data, axis = -1)
		#percent = np.percentile(np.ravel(mean_vol), q = 80)
		#print(percent)
		#plt.hist(np.ravel(mean_vol), bins = 100)
		#plt.show()

		# in_brain_mask = mean_vol > percent

		#data[mean_vol > percent, :] = True
		#data[mean_vol < percent, :] = False
		#print data[:,25,25,25]
		#return data
		# brainSave = save_obj(in_brain_tcs, 'test1')
		# print(in_brain_tcs.shape)

		# plt.imshow(mean_vol[:, :, 50])
		# plt.show()
		# plt.imshow(in_brain_mask[:, :, 50])
		# plt.show()

def generateMask(allFiguresDict, percent):
	maskedDict = {}
	volumeDict = {}
	for key, value in allFiguresDict.iteritems():
		data = value
		vol_mean = np.mean(data, axis = -1)
		percentile = np.percentile(np.ravel(vol_mean), q = percent)
		data[vol_mean > percentile, :] = True
		data[vol_mean < percentile, :] = False
		maskedDict[key] = vol_mean > percentile
		volumeDict[key] = vol_mean
	return maskedDict, volumeDict

def save_obj(obj, name):
	with open('/Users/mike/Downloads/ds105/sub001/model/model001/task001_run001.feat/' + name + '.pkl', 'wb') as f:
		pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL)

def load_obj(name):
	with open('/Users/mike/Downloads/ds105/sub001/model/model001/task001_run001.feat/' + name + '.pkl', 'rb') as f:
		return pickle.load(f)

# generateMaskedBrain(['/Users/edithho/Downloads/ds105/sub001/model/model001/task001_run002.feat'])
# test = load_obj('test1')
# print(test.shape)<commit_msg>Fixed Import Error<commit_after>import nibabel as nib
import matplotlib.pyplot as plt
import numpy as np
import pickle

# Takes in a dictionary with (K, V) pair (subxxx_runyyy, modified 4D Array)
# Returns the masked with time collapsed and vol_mean
def generateMaskedBrain(allFiguresDict):
	maskedDict = {}
	volumeDict = {}
	for key, value in allFiguresDict.iteritems():
		data = value
		vol_mean = np.mean(data, axis = -1)
		percentile = np.percentile(np.ravel(vol_mean), q = 80)
		data[vol_mean > percentile, :] = True
		data[vol_mean < percentile, :] = False
		maskedDict[key] = vol_mean > percentile
		volumeDict[key] = vol_mean
	return maskedDict, volumeDict

#def generateMaskedBrain(paths):
	#for path in paths:
		#img = nib.load(path + '/filtered_func_data_mni.nii.gz')
		#print(img.shape)
		#data = img.get_data()

		#vol_shape = data.shape[:-1]
		#n_trs = data.shape[-1]
		#print(vol_shape, n_trs)
		#mean_vol = np.mean(data, axis = -1)
		#percent = np.percentile(np.ravel(mean_vol), q = 80)
		#print(percent)
		#plt.hist(np.ravel(mean_vol), bins = 100)
		#plt.show()

		# in_brain_mask = mean_vol > percent

		#data[mean_vol > percent, :] = True
		#data[mean_vol < percent, :] = False
		#print data[:,25,25,25]
		#return data
		# brainSave = save_obj(in_brain_tcs, 'test1')
		# print(in_brain_tcs.shape)

		# plt.imshow(mean_vol[:, :, 50])
		# plt.show()
		# plt.imshow(in_brain_mask[:, :, 50])
		# plt.show()

def generateMask(allFiguresDict, percent):
	maskedDict = {}
	volumeDict = {}
	for key, value in allFiguresDict.iteritems():
		data = value
		vol_mean = np.mean(data, axis = -1)
		percentile = np.percentile(np.ravel(vol_mean), q = percent)
		data[vol_mean > percentile, :] = True
		data[vol_mean < percentile, :] = False
		maskedDict[key] = vol_mean > percentile
		volumeDict[key] = vol_mean
	return maskedDict, volumeDict

def save_obj(obj, name):
	with open('/Users/mike/Downloads/ds105/sub001/model/model001/task001_run001.feat/' + name + '.pkl', 'wb') as f:
		pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL)

def load_obj(name):
	with open('/Users/mike/Downloads/ds105/sub001/model/model001/task001_run001.feat/' + name + '.pkl', 'rb') as f:
		return pickle.load(f)

# generateMaskedBrain(['/Users/edithho/Downloads/ds105/sub001/model/model001/task001_run002.feat'])
# test = load_obj('test1')
# print(test.shape)<|endoftext|>


<commit_before>#! /usr/bin/python
import os, sys

def rmdir(dir,depth=0):
    import os
    path = os.path.abspath(dir)
    all_files = os.listdir(path)
    indent = '   ' * depth
    for i in all_files:
        if not i == 'CVS':
            print indent, i
            if os.path.isdir(os.path.join(path,i)):
                rmdir(os.path.join(path,i),depth+1)
            else:
                cmd = 'cd ' + path + ';rm -r ' + i + ';cvs rm ' + i + ';cd ..'
                print cmd
                os.system(cmd)

ignore = ['*.o', '*.pyc', '*.a', '*.so', 'core', '*.dll', '*.pyd']

from fnmatch import fnmatch

def allowed_file_type(file):
    for i in ignore:
        if fnmatch(file, i):
            return 0
    return 1

def adddir(dir,depth=0):
    import os
    path = os.path.abspath(dir)
    all_files = os.listdir(path)
    all_files = filter(allowed_file_type, all_files)
    indent = '   ' * depth

    if os.path.isdir(dir):
        print indent, dir
        cmd = 'cvs add ' + dir
        os.system(cmd)

    for i in all_files:
        if not i == 'CVS':
            print indent, i
            if os.path.isdir(os.path.join(path,i)):
                adddir(os.path.join(path,i),depth+1)
            else:
                cmd = 'cd ' + path + ';cvs add ' + i + ';cd ..'
                print cmd
                os.system(cmd)

<commit_msg>fixed file path stuff in adddir<commit_after>#! /usr/bin/python
import os, sys

def rmdir(dir,depth=0):
    import os
    path = os.path.abspath(dir)
    all_files = os.listdir(path)
    indent = '   ' * depth
    for i in all_files:
        if not i == 'CVS':
            print indent, i
            if os.path.isdir(os.path.join(path,i)):
                rmdir(os.path.join(path,i),depth+1)
            else:
                cmd = 'cd ' + path + ';rm -r ' + i + ';cvs rm ' + i + ';cd ..'
                print cmd
                os.system(cmd)

ignore = ['*.o', '*.pyc', '*.a', '*.so', 'core', '*.dll', '*.pyd']

from fnmatch import fnmatch

def allowed_file_type(file):
    for i in ignore:
        if fnmatch(file, i):
            return 0
    return 1

def adddir(dir,depth=1):
    import os
    #path = os.path.abspath(dir)
    path = dir
    all_files = os.listdir(path)
    all_files = filter(allowed_file_type, all_files)
    indent = '   ' * depth

    if os.path.isdir(dir):
        print indent[:-3], dir
        cmd = 'cvs add ' + dir
        os.system(cmd)

    for i in all_files:
        if not i == 'CVS':
            print indent, i
            if os.path.isdir(os.path.join(path,i)):
                adddir(os.path.join(path,i),depth+1)
            else:
                cmd = 'cd ' + path + ';cvs add ' + i + ';cd ..'
                print cmd
                os.system(cmd)

<|endoftext|>


<commit_before>#!/usr/bin/env python3
from __future__ import division
from __future__ import print_function
from __future__ import absolute_import
from __future__ import unicode_literals
import os
import sys
import errno
import logging
import argparse
# sys.path hack to access lib package in root directory.
sys.path.insert(1, os.path.dirname(sys.path[0]))
sys.path.insert(2, os.path.join(sys.path[1], 'lib'))
from lib import simplewrap
import consensus

ARG_DEFAULTS = {'input':sys.stdin, 'qual_thres':0, 'qual_format':'sanger', 'log':sys.stderr,
                'volume':logging.ERROR}
DESCRIPTION = """Tally statistics on errors in reads, compared to the rest of their (single-\
stranded) families.
Output columns without --all-repeats:
1. barcode
2. order
3. mate
4. number of reads
5. number of errors that occurred more than once
6-end. number of errors in each read.
With --all-repeats:
1. barcode
2. order
3. mate
4. number of reads
5-end. count of how many times each error occurred in the family."""

def make_argparser():

  # Need to use argparse.RawDescriptionHelpFormatter to preserve formatting in the
  # description of columns in the tsv output. But to still accommodate different
  # terminal widths, dynamic wrapping with simplewrap will be necessary.
  wrap = simplewrap.Wrapper().wrap

  parser = argparse.ArgumentParser(description=wrap(DESCRIPTION),
                                   formatter_class=argparse.RawDescriptionHelpFormatter)
  parser.set_defaults(**ARG_DEFAULTS)

  parser.add_argument('input', metavar='families.msa.tsv', nargs='?', type=argparse.FileType('r'),
    help='')
  parser.add_argument('-H', '--human', action='store_true')
  parser.add_argument('-r', '--all-repeats', action='store_true')
  parser.add_argument('-q', '--qual-thres', type=int)
  parser.add_argument('-F', '--qual-format', choices=('sanger', 'solexa', 'illumina1.3',
                                                      'illumina1.5', 'illumina1.8'))
  parser.add_argument('-o', '--overlap', action='store_true',
    help='Figure out whether there is overlap between mates in read pairs and deduplicate errors '
         'that appear twice because of it. Requires --bam.')
  parser.add_argument('-b', '--bam',
    help='The final duplex consensus reads, aligned to a reference. Used to find overlaps.')
  parser.add_argument('-l', '--log', type=argparse.FileType('w'),
    help='Print log messages to this file instead of to stderr. Warning: Will overwrite the file.')
  parser.add_argument('-Q', '--quiet', dest='volume', action='store_const', const=logging.CRITICAL)
  parser.add_argument('-v', '--verbose', dest='volume', action='store_const', const=logging.INFO)
  parser.add_argument('-D', '--debug', dest='volume', action='store_const', const=logging.DEBUG)

  return parser

def main(argv):

  parser = make_argparser()
  args = parser.parse_args(argv[1:])

  logging.basicConfig(stream=args.log, level=args.volume, format='%(message)s')
  tone_down_logger()

  print_family_errors(args.input, args.qual_thres, args.qual_format, args.human, args.all_repeats)

def print_family_errors(infile, qual_thres, qual_format, human, all_repeats):
  for family in parse_families(infile):
    for order in ('ab', 'ba'):
      for mate in (0, 1):
        seq_align, qual_align = family[order][mate]
        if not seq_align:
          continue
        consensus_seq = get_consensus_wrapper(seq_align, qual_align, qual_thres)
        errors = get_family_errors(consensus_seq, seq_align)
        error_types = group_errors(errors)
        errors_per_seq, repeated_errors, error_repeat_counts = tally_errors(error_types, len(seq_align))
        masked_alignment = mask_alignment(seq_align, errors)
        if human:
          for seq, seq_errors in zip(masked_alignment, errors_per_seq):
            print('{} errors: {}'.format(seq, seq_errors))
          if all_repeats:
            print('{} errors: {}, repeat errors: {}\n'.format(consensus_seq,
                                                              sum(errors_per_seq),
                                                              ', '.join(map(str, error_repeat_counts))))
          else:
            print('{} errors: {}, repeat errors: {}\n'.format(consensus_seq,
                                                              sum(errors_per_seq),
                                                              repeated_errors))
        elif all_repeats:
          print(family['bar'], order, mate, len(seq_align), *error_repeat_counts, sep='\t')
        else:
          print(family['bar'], order, mate, len(seq_align), repeated_errors, *errors_per_seq, sep='\t')

def parse_families(infile):
  """Parse a families.msa.tsv file.
  Yields a data structure for each family:
  family = {
    'bar': barcode,                                     # family 'AAACCGACACAGGACTAGGGATCA'
    'ab': (                                               # order ab
            ([seq1, seq2, seq3], [quals1, quals2, quals3]), # mate 1
            ([seq1, seq2], [quals1, quals2]),               # mate 2
          ),
    'ba': (                                               # order ba
            ([seq1, seq2], [quals1, quals2]),               # mate 1
            ([], []),                                       # mate 2
          )
  }
  That is, each family is a dict with the 'bar' key giving the barcode sequence, and a key for both
  orders ('ab' and 'ba'). The value for each order is a tuple of 2 values, one for each mate. Each
  value in the tuple is itself a 2-tuple containing the aligned bases and quality scores.
  Examples:
  Getting the sequences for mate 1 of order "ab":
  seq_align = family['ab'][0][0]
  Getting the quality scores:
  seq_align = family['ab'][0][1]
  Getting the sequences for mate 2 of order "ba":
  seq_align = family['ba'][1][0]
  """
  last_barcode = None
  family = {'bar':None, 'ab':(([],[]), ([],[])), 'ba':(([],[]), ([],[]))}
  for line in infile:
    fields = line.rstrip('\r\n').split('\t')
    barcode = fields[0]
    order = fields[1]
    mate = int(fields[2])-1
    seq = fields[4]
    quals = fields[5]
    if barcode != last_barcode:
      if last_barcode is not None:
        yield family
      family = {'bar':barcode, 'ab':(([],[]), ([],[])), 'ba':(([],[]), ([],[]))}
      last_barcode = barcode
    family[order][mate][0].append(seq)
    family[order][mate][1].append(quals)
  yield family

def get_consensus_wrapper(seqs, quals, qual_thres):
  """Encode outgoing strings as bytes and decode return value into str."""
  seqs_bytes = [bytes(seq, 'utf8') for seq in seqs]
  quals_bytes = [bytes(qual, 'utf8') for qual in quals]
  cons_bytes = consensus.get_consensus(seqs_bytes, quals_bytes, qual_thres=qual_thres)
  return str(cons_bytes, 'utf8')

def compare(consensus_seq, seq_align, all_repeats=False):
  num_seqs = len(seq_align)
  new_align = [''] * num_seqs
  errors = [0] * num_seqs
  if all_repeats:
    repeat_errors = []
  else:
    repeat_errors = 0
  # Walk along the alignment, one position at a time.
  for bases, cons_base in zip(zip(*seq_align), consensus_seq):
    # Tally how many of each base there are at this position.
    votes = {'A':0, 'C':0, 'G':0, 'T':0, '-':0, 'N':0}
    # Count votes and make new alignment without consensus bases.
    for i, base in enumerate(bases):
      votes[base] += 1
      if base == cons_base:
        new_align[i] += '.'
      else:
        new_align[i] += base
        errors[i] += 1
    # How often did each error occur at this position?
    # (counting each unique non-consensus base as an error)
    for base, vote in votes.items():
      if base != cons_base:
        if all_repeats:
          if vote > 0:
            repeat_errors.append(vote)
        elif vote > 1:
          repeat_errors += 1
    i += 1
  return errors, repeat_errors, new_align

def get_family_errors(consensus_seq, alignment):
  for coord, (cons_base, bases) in enumerate(zip(consensus_seq, zip(*alignment))):
    for seq_num, base in enumerate(bases):
      if base != cons_base:
        yield (seq_num, coord+1, base)

def group_errors(errors):
  error_types = []
  last_error = None
  current_types = []
  for error in sorted(errors, key=lambda error: error[1]):
    if last_error is not None and last_error[1] == error[1] and last_error[2] == error[2]:
      current_types.append(error)
    else:
      if current_types:
        error_types.append(current_types)
      current_types = [error]
    last_error = error
  if current_types:
    error_types.append(current_types)
  return error_types

def tally_errors(error_types, num_seqs):
  errors_per_seq = [0] * num_seqs
  repeated_errors = 0
  error_repeat_counts = []
  for error_type in error_types:
    error_repeat_counts.append(len(error_type))
    if len(error_type) > 1:
      repeated_errors += 1
    for error in error_type:
      errors_per_seq[error[0]] += 1
  return errors_per_seq, repeated_errors, error_repeat_counts

def mask_alignment(seq_alignment, errors):
  masked_alignment = [['.'] * len(seq) for seq in seq_alignment]
  for error in sorted(errors, key=lambda error: error[1]):
    seq_num = error[0]
    coord = error[1]
    base = error[2]
    masked_alignment[seq_num][coord-1] = base
  return [''.join(seq) for seq in masked_alignment]

def tone_down_logger():
  """Change the logging level names from all-caps to capitalized lowercase.
  E.g. "WARNING" -> "Warning" (turn down the volume a bit in your log files)"""
  for level in (logging.CRITICAL, logging.ERROR, logging.WARNING, logging.INFO, logging.DEBUG):
    level_name = logging.getLevelName(level)
    logging.addLevelName(level, level_name.capitalize())

def fail(message):
  logging.critical(message)
  if __name__ == '__main__':
    sys.exit(1)
  else:
    raise Exception('Unrecoverable error')

if __name__ == '__main__':
  try:
    sys.exit(main(sys.argv))
  except IOError as ioe:
    if ioe.errno != errno.EPIPE:
      raise
<commit_msg>utils/errstats.py: Fix bugs and restore to previous functionality.<commit_after>#!/usr/bin/env python3
from __future__ import division
from __future__ import print_function
from __future__ import absolute_import
from __future__ import unicode_literals
import os
import sys
import errno
import logging
import argparse
# sys.path hack to access lib package in root directory.
sys.path.insert(1, os.path.dirname(sys.path[0]))
sys.path.insert(2, os.path.join(sys.path[1], 'lib'))
from lib import simplewrap
import consensus

ARG_DEFAULTS = {'input':sys.stdin, 'qual_thres':0, 'log':sys.stderr,
                'volume':logging.ERROR}
DESCRIPTION = """Tally statistics on errors in reads, compared to the rest of their (single-\
stranded) families.
Output columns without --all-repeats:
1. barcode
2. order
3. mate
4. number of reads
5. number of errors that occurred more than once
6-end. number of errors in each read.
With --all-repeats:
1. barcode
2. order
3. mate
4. number of reads
5-end. count of how many times each error occurred in the family."""

def make_argparser():

  # Need to use argparse.RawDescriptionHelpFormatter to preserve formatting in the
  # description of columns in the tsv output. But to still accommodate different
  # terminal widths, dynamic wrapping with simplewrap will be necessary.
  wrap = simplewrap.Wrapper().wrap

  parser = argparse.ArgumentParser(description=wrap(DESCRIPTION),
                                   formatter_class=argparse.RawDescriptionHelpFormatter)
  parser.set_defaults(**ARG_DEFAULTS)

  parser.add_argument('input', metavar='families.msa.tsv', nargs='?', type=argparse.FileType('r'),
    help='')
  parser.add_argument('-H', '--human', action='store_true')
  parser.add_argument('-r', '--all-repeats', action='store_true')
  parser.add_argument('-q', '--qual-thres', type=int)
  parser.add_argument('-o', '--overlap', action='store_true',
    help='Figure out whether there is overlap between mates in read pairs and deduplicate errors '
         'that appear twice because of it. Requires --bam.')
  parser.add_argument('-b', '--bam',
    help='The final duplex consensus reads, aligned to a reference. Used to find overlaps.')
  parser.add_argument('-l', '--log', type=argparse.FileType('w'),
    help='Print log messages to this file instead of to stderr. Warning: Will overwrite the file.')
  parser.add_argument('-Q', '--quiet', dest='volume', action='store_const', const=logging.CRITICAL)
  parser.add_argument('-v', '--verbose', dest='volume', action='store_const', const=logging.INFO)
  parser.add_argument('-D', '--debug', dest='volume', action='store_const', const=logging.DEBUG)

  return parser

def main(argv):

  parser = make_argparser()
  args = parser.parse_args(argv[1:])

  logging.basicConfig(stream=args.log, level=args.volume, format='%(message)s')
  tone_down_logger()

  print_family_errors(args.input, args.qual_thres, args.human, args.all_repeats)

def print_family_errors(infile, qual_thres, human, all_repeats):
  for family in parse_families(infile):
    for order in ('ab', 'ba'):
      for mate in (0, 1):
        seq_align, qual_align = family[order][mate]
        if not seq_align:
          continue
        consensus_seq = get_consensus_wrapper(seq_align, qual_align, qual_thres, gapped=True)
        errors = get_family_errors(consensus_seq, seq_align)
        error_types = group_errors(errors)
        errors_per_seq, repeated_errors, error_repeat_counts = tally_errors(error_types, len(seq_align))
        masked_alignment = mask_alignment(seq_align, errors)
        if human:
          for seq, seq_errors in zip(masked_alignment, errors_per_seq):
            print('{} errors: {}'.format(seq, seq_errors))
          if all_repeats:
            print('{} errors: {}, repeat errors: {}\n'.format(consensus_seq,
                                                              sum(errors_per_seq),
                                                              ', '.join(map(str, error_repeat_counts))))
          else:
            print('{} errors: {}, repeat errors: {}\n'.format(consensus_seq,
                                                              sum(errors_per_seq),
                                                              repeated_errors))
        elif all_repeats:
          print(family['bar'], order, mate, len(seq_align), *error_repeat_counts, sep='\t')
        else:
          print(family['bar'], order, mate, len(seq_align), repeated_errors, *errors_per_seq, sep='\t')

def parse_families(infile):
  """Parse a families.msa.tsv file.
  Yields a data structure for each family:
  family = {
    'bar': barcode,                                     # family 'AAACCGACACAGGACTAGGGATCA'
    'ab': (                                               # order ab
            ([seq1, seq2, seq3], [quals1, quals2, quals3]), # mate 1
            ([seq1, seq2], [quals1, quals2]),               # mate 2
          ),
    'ba': (                                               # order ba
            ([seq1, seq2], [quals1, quals2]),               # mate 1
            ([], []),                                       # mate 2
          )
  }
  That is, each family is a dict with the 'bar' key giving the barcode sequence, and a key for both
  orders ('ab' and 'ba'). The value for each order is a tuple of 2 values, one for each mate. Each
  value in the tuple is itself a 2-tuple containing the aligned bases and quality scores.
  Examples:
  Getting the sequences for mate 1 of order "ab":
  seq_align = family['ab'][0][0]
  Getting the quality scores:
  seq_align = family['ab'][0][1]
  Getting the sequences for mate 2 of order "ba":
  seq_align = family['ba'][1][0]
  """
  last_barcode = None
  family = {'bar':None, 'ab':(([],[]), ([],[])), 'ba':(([],[]), ([],[]))}
  for line in infile:
    fields = line.rstrip('\r\n').split('\t')
    barcode = fields[0]
    order = fields[1]
    mate = int(fields[2])-1
    seq = fields[4]
    quals = fields[5]
    if barcode != last_barcode:
      if last_barcode is not None:
        yield family
      family = {'bar':barcode, 'ab':(([],[]), ([],[])), 'ba':(([],[]), ([],[]))}
      last_barcode = barcode
    family[order][mate][0].append(seq)
    family[order][mate][1].append(quals)
  yield family

def get_consensus_wrapper(seqs, quals, qual_thres, gapped=False):
  """Encode outgoing strings as bytes and decode return value into str."""
  seqs_bytes = [bytes(seq, 'utf8') for seq in seqs]
  quals_bytes = [bytes(qual, 'utf8') for qual in quals]
  cons_bytes = consensus.get_consensus(seqs_bytes, quals_bytes, qual_thres=qual_thres, gapped=gapped)
  return str(cons_bytes, 'utf8')

def compare(consensus_seq, seq_align, all_repeats=False):
  num_seqs = len(seq_align)
  new_align = [''] * num_seqs
  errors = [0] * num_seqs
  if all_repeats:
    repeat_errors = []
  else:
    repeat_errors = 0
  # Walk along the alignment, one position at a time.
  for bases, cons_base in zip(zip(*seq_align), consensus_seq):
    # Tally how many of each base there are at this position.
    votes = {'A':0, 'C':0, 'G':0, 'T':0, '-':0, 'N':0}
    # Count votes and make new alignment without consensus bases.
    for i, base in enumerate(bases):
      votes[base] += 1
      if base == cons_base:
        new_align[i] += '.'
      else:
        new_align[i] += base
        errors[i] += 1
    # How often did each error occur at this position?
    # (counting each unique non-consensus base as an error)
    for base, vote in votes.items():
      if base != cons_base:
        if all_repeats:
          if vote > 0:
            repeat_errors.append(vote)
        elif vote > 1:
          repeat_errors += 1
    i += 1
  return errors, repeat_errors, new_align

def get_family_errors(consensus_seq, alignment):
  errors = []
  for coord, (cons_base, bases) in enumerate(zip(consensus_seq, zip(*alignment))):
    for seq_num, base in enumerate(bases):
      if base != cons_base:
        errors.append((seq_num, coord+1, base))
  return errors

def group_errors(errors):
  error_types = []
  last_error = None
  current_types = []
  for error in sorted(errors, key=lambda error: error[1]):
    if last_error is not None and last_error[1] == error[1] and last_error[2] == error[2]:
      current_types.append(error)
    else:
      if current_types:
        error_types.append(current_types)
      current_types = [error]
    last_error = error
  if current_types:
    error_types.append(current_types)
  return error_types

def tally_errors(error_types, num_seqs):
  errors_per_seq = [0] * num_seqs
  repeated_errors = 0
  error_repeat_counts = []
  for error_type in error_types:
    error_repeat_counts.append(len(error_type))
    if len(error_type) > 1:
      repeated_errors += 1
    for error in error_type:
      errors_per_seq[error[0]] += 1
  return errors_per_seq, repeated_errors, error_repeat_counts

def mask_alignment(seq_alignment, errors):
  masked_alignment = [['.'] * len(seq) for seq in seq_alignment]
  for error in sorted(errors, key=lambda error: error[1]):
    seq_num = error[0]
    coord = error[1]
    base = error[2]
    masked_alignment[seq_num][coord-1] = base
  return [''.join(seq) for seq in masked_alignment]

def tone_down_logger():
  """Change the logging level names from all-caps to capitalized lowercase.
  E.g. "WARNING" -> "Warning" (turn down the volume a bit in your log files)"""
  for level in (logging.CRITICAL, logging.ERROR, logging.WARNING, logging.INFO, logging.DEBUG):
    level_name = logging.getLevelName(level)
    logging.addLevelName(level, level_name.capitalize())

def fail(message):
  logging.critical(message)
  if __name__ == '__main__':
    sys.exit(1)
  else:
    raise Exception('Unrecoverable error')

if __name__ == '__main__':
  try:
    sys.exit(main(sys.argv))
  except IOError as ioe:
    if ioe.errno != errno.EPIPE:
      raise
<|endoftext|>