<commit_before>from deuce.util import log as logging

from pecan import expose, response, request
from pecan.rest import RestController

import deuce

from deuce.controllers.blocks import BlocksController
from deuce.controllers.files import FilesController
from deuce.controllers.validation import *
from deuce.model import Vault

logger = logging.getLogger(__name__)

class VaultController(RestController):

    blocks = BlocksController()
    files = FilesController()

    @expose()
    def index(self):
        logger.warning('Invalid vault controller index request')
        response.status_code = 404

    @expose()
    @validate(vault_name=VaultPutRule)
    def put(self, vault_name):
        response.headers["Transaction-ID"] = request.context.request_id
        vault = Vault.create(request.project_id, vault_name,
                request.auth_token)
        # TODO: Need check and monitor failed vault.
        logger.info('Vault [{0}] created'.format(vault_name))
        response.status_code = 201 if vault else 500

    @expose()
    @validate(vault_id=VaultGetRule)
    def head(self, vault_id):
        """Returns the vault controller object"""

        response.headers["Transaction-ID"] = request.context.request_id
        if Vault.get(request.project_id, vault_id,
                request.auth_token):
            # weblint complains about the content-type header being
            # present as pecan doesn't intelligently add it or remove
            # it.
            del response.headers["Content-Type"]
            response.status_code = 204
            return response
        else:
            logger.error('Vault [{0}] does not exist'.format(vault_id))
            response.status_code = 404

    @expose('json')
    @validate(vault_id=VaultGetRule)
    def get_one(self, vault_id):
        """Returns the statistics on vault controller object"""
        response.headers["Transaction-ID"] = request.context.request_id

        vault = Vault.get(request.project_id, vault_id,
                request.auth_token)

        if vault:
            response.body_file = vault.get_vault_statistics(request.auth_token)
            response.status_code = 200
        else:
            logger.error('Vault [{0}] does not exist'.format(vault_id))
            response.status_code = 404

        return None

    @expose()
    @validate(vault_id=VaultPutRule)
    def delete(self, vault_id):
        response.headers["Transaction-ID"] = request.context.request_id
        vault = Vault.get(request.project_id, vault_id,
                request.auth_token)

        if vault:
            vault.delete(
                request.auth_token)
            logger.info('Vault [{0}] deleted'.format(vault_id))
            # weblint complains about the content-type header being present
            # as pecan doesn't intelligently add it or remove it.
            del response.headers["Content-Type"]
            response.status_code = 204
        else:
            logger.error('Vault [{0}] deletion failed; '
                'Vault does not exist'.format(vault_id))
            response.status_code = 404
<commit_msg>Bug Fix Statistics<commit_after>from deuce.util import log as logging

from pecan import expose, response, request
from pecan.rest import RestController

import deuce

from deuce.controllers.blocks import BlocksController
from deuce.controllers.files import FilesController
from deuce.controllers.validation import *
from deuce.model import Vault

logger = logging.getLogger(__name__)

class VaultController(RestController):

    blocks = BlocksController()
    files = FilesController()

    @expose()
    def index(self):
        logger.warning('Invalid vault controller index request')
        response.status_code = 404

    @expose()
    @validate(vault_name=VaultPutRule)
    def put(self, vault_name):
        response.headers["Transaction-ID"] = request.context.request_id
        vault = Vault.create(request.project_id, vault_name,
                request.auth_token)
        # TODO: Need check and monitor failed vault.
        logger.info('Vault [{0}] created'.format(vault_name))
        response.status_code = 201 if vault else 500

    @expose()
    @validate(vault_id=VaultGetRule)
    def head(self, vault_id):
        """Returns the vault controller object"""

        response.headers["Transaction-ID"] = request.context.request_id
        if Vault.get(request.project_id, vault_id,
                request.auth_token):
            # weblint complains about the content-type header being
            # present as pecan doesn't intelligently add it or remove
            # it.
            del response.headers["Content-Type"]
            response.status_code = 204
            return response
        else:
            logger.error('Vault [{0}] does not exist'.format(vault_id))
            response.status_code = 404

    @expose('json')
    @validate(vault_id=VaultGetRule)
    def get_one(self, vault_id):
        """Returns the statistics on vault controller object"""
        response.headers["Transaction-ID"] = request.context.request_id

        vault = Vault.get(request.project_id, vault_id,
                request.auth_token)

        if vault:
            response.status_code = 200
            return vault.get_vault_statistics(request.auth_token)
        else:
            logger.error('Vault [{0}] does not exist'.format(vault_id))
            response.status_code = 404
            return None

    @expose()
    @validate(vault_id=VaultPutRule)
    def delete(self, vault_id):
        response.headers["Transaction-ID"] = request.context.request_id
        vault = Vault.get(request.project_id, vault_id,
                request.auth_token)

        if vault:
            vault.delete(
                request.auth_token)
            logger.info('Vault [{0}] deleted'.format(vault_id))
            # weblint complains about the content-type header being present
            # as pecan doesn't intelligently add it or remove it.
            del response.headers["Content-Type"]
            response.status_code = 204
        else:
            logger.error('Vault [{0}] deletion failed; '
                'Vault does not exist'.format(vault_id))
            response.status_code = 404
<|endoftext|>


<commit_before>from django.core.urlresolvers import reverse
from django.test import TestCase, Client
from django.db import IntegrityError
from pages.models import Page, Comment

def create_page(key, display_name, definition):
    p = Page.objects.create(word=key, display_name=display_name, definition=definition)
    return p

def create_comment(page, comment, rank):
    c = Comment.objects.create(page=page, comment=comment, rating=rank)
    return c

class PageTest(TestCase):
    def test_page_create(self):
        """
        Test page creation with alphanumeric key.
        """
        word = 'alphanumeric'
        name = 'Alphanumeric'
        definition = 'a combination of alphabetic and numeric characters'
        page = create_page(word, name, definition)
        self.assertIsNotNone(page)
        self.assertEqual(word, page.word)
        self.assertEqual(name, page.display_name)
        self.assertEqual(definition, page.definition)

    def test_page_no_key(self):
        """
        Test page creation with no key
        """
        definition = 'no key'
        self.assertRaises(IntegrityError, create_page, None, 'word', definition)

    def test_page_unique_key(self):
        """
        Test that no two pages can be created with the same key
        """
        word = 'same_key'
        display_name = 'Same key'
        definition = 'Two words with the same key'
        page1 = create_page(word, display_name, definition)
        self.assertRaises(IntegrityError, create_page, word, display_name, definition)

    def test_page_description_blank(self):
        """
        Test that the description can remain blank
        """
        key = 'word'
        name = 'Word'
        page = create_page(key, name, None)
        self.assertIsInstance(page, Page)

    def test_generate_page_with_five_comments(self):
        page = create_page('word', 'Word', 'A definition of a word')
        for i in xrange(5):
            create_comment(page, 'This is a terrbile page', 1)
        self.assertEqual(5, page.comments.count())

    def test_successful_new_view(self):
        c = Client()
        d = dict(
            word='word',
            display_name='Word',
            definition='A definition of a word'
        )
        response = c.post(reverse('pages:new'), d)
        self.assertEqual(200, response.status_code)
        ctx = response.context
        print response.content
<commit_msg>fixed test sucessful new view<commit_after>from django.core.urlresolvers import reverse
from django.test import TestCase, Client
from django.db import IntegrityError
from pages.models import Page, Comment

def create_page(key, display_name, definition):
    p = Page.objects.create(word=key, display_name=display_name, definition=definition)
    return p

def create_comment(page, comment, rank):
    c = Comment.objects.create(page=page, comment=comment, rating=rank)
    return c

class PageTest(TestCase):
    def test_page_create(self):
        """
        Test page creation with alphanumeric key.
        """
        word = 'alphanumeric'
        name = 'Alphanumeric'
        definition = 'a combination of alphabetic and numeric characters'
        page = create_page(word, name, definition)
        self.assertIsNotNone(page)
        self.assertEqual(word, page.word)
        self.assertEqual(name, page.display_name)
        self.assertEqual(definition, page.definition)

    def test_page_no_key(self):
        """
        Test page creation with no key
        """
        definition = 'no key'
        self.assertRaises(IntegrityError, create_page, None, 'word', definition)

    def test_page_unique_key(self):
        """
        Test that no two pages can be created with the same key
        """
        word = 'same_key'
        display_name = 'Same key'
        definition = 'Two words with the same key'
        page1 = create_page(word, display_name, definition)
        self.assertRaises(IntegrityError, create_page, word, display_name, definition)

    def test_page_description_blank(self):
        """
        Test that the description can remain blank
        """
        key = 'word'
        name = 'Word'
        page = create_page(key, name, None)
        self.assertIsInstance(page, Page)

    def test_generate_page_with_five_comments(self):
        page = create_page('word', 'Word', 'A definition of a word')
        for i in xrange(5):
            create_comment(page, 'This is a terrbile page', 1)
        self.assertEqual(5, page.comments.count())

    def test_successful_new_view(self):
        c = Client()
        d = dict(
            word='kwargs',
            display_name='KU-Args',
            definition='Keyword arguments'
        )
        response = c.post(reverse('pages:new'), d)
        self.assertEqual(302, response.status_code)
        self.assertRegexpMatches(response['Location'], (d['word']))
<|endoftext|>


<commit_before>"""
Routines to construct new SymPy expressions transforming the provided input.
"""

from collections import Iterable, OrderedDict

import sympy
from sympy import collect, collect_const, flatten

from devito.dse.extended_sympy import Add, Eq, Mul
from devito.dse.inspection import count, estimate_cost, retrieve_indexed
from devito.dse.graph import temporaries_graph
from devito.dse.queries import q_indexed, q_op, q_leaf
from devito.interfaces import Indexed, TensorFunction
from devito.tools import as_tuple

__all__ = ['collect_nested', 'common_subexprs_elimination', 'freeze_expression',
           'xreplace_constrained', 'xreplace_indices', 'promote_scalar_expressions',
           'pow_to_mul']

def freeze_expression(expr):
    """
    Reconstruct ``expr`` turning all :class:`sympy.Mul` and :class:`sympy.Add`
    into, respectively, :class:`devito.Mul` and :class:`devito.Add`.
    """
    if expr.is_Atom or q_indexed(expr):
        return expr
    elif expr.is_Add:
        rebuilt_args = [freeze_expression(e) for e in expr.args]
        return Add(*rebuilt_args, evaluate=False)
    elif expr.is_Mul:
        rebuilt_args = [freeze_expression(e) for e in expr.args]
        return Mul(*rebuilt_args, evaluate=False)
    elif expr.is_Equality:
        rebuilt_args = [freeze_expression(e) for e in expr.args]
        return Eq(*rebuilt_args, evaluate=False)
    else:
        return expr.func(*[freeze_expression(e) for e in expr.args])

def promote_scalar_expressions(exprs, shape, indices, onstack):
    """
    Transform a collection of scalar expressions into tensor expressions.
    """
    processed = []

    # Fist promote the LHS
    graph = temporaries_graph(exprs)
    mapper = {}
    for k, v in graph.items():
        if v.is_scalar:
            # Create a new function symbol
            data = TensorFunction(name=k.name, shape=shape,
                                  dimensions=indices, onstack=onstack)
            indexed = Indexed(data.indexed, *indices)
            mapper[k] = indexed
            processed.append(Eq(indexed, v.rhs))
        else:
            processed.append(Eq(k, v.rhs))

    # Propagate the transformed LHS through the expressions
    processed = [Eq(n.lhs, n.rhs.xreplace(mapper)) for n in processed]

    return processed

def collect_nested(expr, aggressive=False):
    """
    Collect terms appearing in expr, checking all levels of the expression tree.

    :param expr: the expression to be factorized.
    """

    def run(expr):
        # Return semantic (rebuilt expression, factorization candidates)

        if expr.is_Number or expr.is_Symbol:
            return expr, [expr]
        elif q_indexed(expr) or expr.is_Atom:
            return expr, []
        elif expr.is_Add:
            rebuilt, candidates = zip(*[run(arg) for arg in expr.args])

            w_numbers = [i for i in rebuilt if any(j.is_Number for j in i.args)]
            wo_numbers = [i for i in rebuilt if i not in w_numbers]

            w_numbers = collect_const(expr.func(*w_numbers))
            wo_numbers = expr.func(*wo_numbers)

            if aggressive is True and wo_numbers:
                for i in flatten(candidates):
                    wo_numbers = collect(wo_numbers, i)

            rebuilt = expr.func(w_numbers, wo_numbers)
            return rebuilt, []
        elif expr.is_Mul:
            rebuilt, candidates = zip(*[run(arg) for arg in expr.args])
            rebuilt = collect_const(expr.func(*rebuilt))
            return rebuilt, flatten(candidates)
        else:
            rebuilt, candidates = zip(*[run(arg) for arg in expr.args])
            return expr.func(*rebuilt), flatten(candidates)

    return run(expr)[0]

def xreplace_constrained(exprs, make, rule, costmodel=lambda e: True, repeat=False):
    """
    Unlike ``xreplace``, which replaces all objects specified in a mapper,
    this function replaces all objects satisfying two criteria: ::

        * The "matching rule" -- a function returning True if a node within ``expr``
            satisfies a given property, and as such should be replaced;
        * A "cost model" -- a function triggering replacement only if a certain
            cost (e.g., operation count) is exceeded. This function is optional.

    Note that there is not necessarily a relationship between the set of nodes
    for which the matching rule returns True and those nodes passing the cost
    model check. It might happen for example that, given the expression ``a + b``,
    all of ``a``, ``b``, and ``a + b`` satisfy the matching rule, but only
    ``a + b`` satisfies the cost model.

    :param exprs: The target SymPy expression, or a collection of SymPy expressions.
    :param make: A function to construct symbols used for replacement.
                 The function takes as input an integer ID; ID is computed internally
                 and used as a unique identifier for the constructed symbols.
    :param rule: The matching rule (a lambda function).
    :param costmodel: The cost model (a lambda function, optional).
    :param repeat: Repeatedly apply ``xreplace`` until no more replacements are
                   possible (optional, defaults to False).
    """

    found = OrderedDict()
    rebuilt = []

    def replace(expr):
        temporary = found.get(expr)
        if temporary:
            return temporary
        else:
            temporary = make(replace.c)
            found[expr] = temporary
            replace.c += 1
            return temporary
    replace.c = 0  # Unique identifier for new temporaries

    def run(expr):
        if expr.is_Atom or q_indexed(expr):
            return expr, rule(expr)
        elif expr.is_Pow:
            base, flag = run(expr.base)
            return expr.func(base, expr.exp, evaluate=False), flag
        else:
            children = [run(a) for a in expr.args]
            matching = [a for a, flag in children if flag]
            other = [a for a, _ in children if a not in matching]
            if matching:
                matched = expr.func(*matching, evaluate=False)
                if len(matching) == len(children) and rule(expr):
                    # Go look for longer expressions first
                    return matched, True
                elif rule(matched) and costmodel(matched):
                    # Replace what I can replace, then give up
                    rebuilt = expr.func(*(other + [replace(matched)]), evaluate=False)
                    return rebuilt, False
                else:
                    # Replace flagged children, then give up
                    replaced = [replace(e) for e in matching if costmodel(e)]
                    unreplaced = [e for e in matching if not costmodel(e)]
                    rebuilt = expr.func(*(other + replaced + unreplaced), evaluate=False)
                    return rebuilt, False
            return expr.func(*other, evaluate=False), False

    # Process the provided expressions
    for expr in as_tuple(exprs):
        assert expr.is_Equality
        root = expr.rhs

        while True:
            ret, _ = run(root)
            if repeat and ret != root:
                root = ret
            else:
                rebuilt.append(expr.func(expr.lhs, ret))
                break

    # Post-process the output
    found = [Eq(v, k) for k, v in found.items()]

    return found + rebuilt, found

def xreplace_indices(exprs, mapper, candidates=None, only_rhs=False):
    """
    Create new expressions from ``exprs``, by replacing all index variables
    specified in mapper appearing as a tensor index. Only tensors whose symbolic
    name appears in ``candidates`` are considered if ``candidates`` is not None.
    """
    get = lambda i: i.rhs if only_rhs is True else i
    handle = flatten(retrieve_indexed(get(i)) for i in as_tuple(exprs))
    if candidates is not None:
        handle = [i for i in handle if i.base.label in candidates]
    mapper = dict(zip(handle, [i.xreplace(mapper) for i in handle]))
    replaced = [i.xreplace(mapper) for i in as_tuple(exprs)]
    return replaced if isinstance(exprs, Iterable) else replaced[0]

def common_subexprs_elimination(exprs, make, mode='default'):
    """
    Perform common subexpressions elimination.

    Note: the output is not guranteed to be topologically sorted.

    :param exprs: The target SymPy expression, or a collection of SymPy expressions.
    :param make: A function to construct symbols used for replacement.
                 The function takes as input an integer ID; ID is computed internally
                 and used as a unique identifier for the constructed symbols.
    """

    # Note: not defaulting to SymPy's CSE() function for three reasons:
    # - it also captures array index access functions (eg, i+1 in A[i+1] and B[i+1]);
    # - it sometimes "captures too much", losing factorization opportunities;
    # - very slow
    # TODO: a second "sympy" mode will be provided, relying on SymPy's CSE() but
    # also ensuring some sort of post-processing
    assert mode == 'default'  # Only supported mode ATM

    processed = list(exprs)
    mapped = []
    while True:
        # Detect redundancies
        counted = count(mapped + processed, q_op).items()
        targets = OrderedDict([(k, estimate_cost(k)) for k, v in counted if v > 1])
        if not targets:
            break

        # Create temporaries
        hit = max(targets.values())
        picked = [k for k, v in targets.items() if v == hit]
        mapper = OrderedDict([(e, make(len(mapped) + i)) for i, e in enumerate(picked)])

        # Apply repleacements
        processed = [e.xreplace(mapper) for e in processed]
        mapped = [e.xreplace(mapper) for e in mapped]
        mapped = [Eq(v, k) for k, v in reversed(list(mapper.items()))] + mapped

        # Prepare for the next round
        for k in picked:
            targets.pop(k)
    processed = mapped + processed

    # Simply renumber the temporaries in ascending order
    mapper = {i.lhs: j.lhs for i, j in zip(mapped, reversed(mapped))}
    processed = [e.xreplace(mapper) for e in processed]

    return processed

def compact_temporaries(exprs):
    """
    Drop temporaries consisting of single symbols.
    """
    g = temporaries_graph(exprs)

    mapper = {k: v.rhs for k, v in g.items()
              if v.is_scalar and (q_leaf(v.rhs) or v.rhs.is_Function)}

    processed = []
    for k, v in g.items():
        if k not in mapper:
            # The temporary /v/ is retained, and substitutions may be applied
            processed.append(v.xreplace(mapper))

    return processed

def pow_to_mul(expr):
    """
    Convert integer powers in an expression to Muls, like a**2 => a*a.
    Readapted from: ::

        stackoverflow.com/questions/14264431/expanding-algebraic-powers-in-python-sympy

    The readaptation was necessary to make it work with Indexed.
    """
    lhs, rhs = expr.args
    pows = [i for i in rhs.args if i.is_Pow]
    non_pows = [i for i in rhs.args if i not in pows]
    if not pows:
        return expr
    if any(not e.is_Integer or e <= 0 for b, e in (i.as_base_exp() for i in pows)):
        # Cannot handle powers containing non-integer non-positive exponents
        return expr
    muls = [sympy.Mul(*[b]*e, evaluate=False)
            for b, e in (i.as_base_exp() for i in pows)]
    rhs = rhs.func(*(muls + non_pows), evaluate=False)
    return expr.func(expr.lhs, rhs.func(*(muls + non_pows), evaluate=False))
<commit_msg>dse: Allow mapper-based subs in xreplace_constrained<commit_after>"""
Routines to construct new SymPy expressions transforming the provided input.
"""

from collections import Iterable, OrderedDict

import sympy
from sympy import collect, collect_const, flatten

from devito.dse.extended_sympy import Add, Eq, Mul
from devito.dse.inspection import count, estimate_cost, retrieve_indexed
from devito.dse.graph import temporaries_graph
from devito.dse.queries import q_indexed, q_op, q_leaf
from devito.interfaces import Indexed, TensorFunction
from devito.tools import as_tuple

__all__ = ['collect_nested', 'common_subexprs_elimination', 'freeze_expression',
           'xreplace_constrained', 'xreplace_indices', 'promote_scalar_expressions',
           'pow_to_mul']

def freeze_expression(expr):
    """
    Reconstruct ``expr`` turning all :class:`sympy.Mul` and :class:`sympy.Add`
    into, respectively, :class:`devito.Mul` and :class:`devito.Add`.
    """
    if expr.is_Atom or q_indexed(expr):
        return expr
    elif expr.is_Add:
        rebuilt_args = [freeze_expression(e) for e in expr.args]
        return Add(*rebuilt_args, evaluate=False)
    elif expr.is_Mul:
        rebuilt_args = [freeze_expression(e) for e in expr.args]
        return Mul(*rebuilt_args, evaluate=False)
    elif expr.is_Equality:
        rebuilt_args = [freeze_expression(e) for e in expr.args]
        return Eq(*rebuilt_args, evaluate=False)
    else:
        return expr.func(*[freeze_expression(e) for e in expr.args])

def promote_scalar_expressions(exprs, shape, indices, onstack):
    """
    Transform a collection of scalar expressions into tensor expressions.
    """
    processed = []

    # Fist promote the LHS
    graph = temporaries_graph(exprs)
    mapper = {}
    for k, v in graph.items():
        if v.is_scalar:
            # Create a new function symbol
            data = TensorFunction(name=k.name, shape=shape,
                                  dimensions=indices, onstack=onstack)
            indexed = Indexed(data.indexed, *indices)
            mapper[k] = indexed
            processed.append(Eq(indexed, v.rhs))
        else:
            processed.append(Eq(k, v.rhs))

    # Propagate the transformed LHS through the expressions
    processed = [Eq(n.lhs, n.rhs.xreplace(mapper)) for n in processed]

    return processed

def collect_nested(expr, aggressive=False):
    """
    Collect terms appearing in expr, checking all levels of the expression tree.

    :param expr: the expression to be factorized.
    """

    def run(expr):
        # Return semantic (rebuilt expression, factorization candidates)

        if expr.is_Number or expr.is_Symbol:
            return expr, [expr]
        elif q_indexed(expr) or expr.is_Atom:
            return expr, []
        elif expr.is_Add:
            rebuilt, candidates = zip(*[run(arg) for arg in expr.args])

            w_numbers = [i for i in rebuilt if any(j.is_Number for j in i.args)]
            wo_numbers = [i for i in rebuilt if i not in w_numbers]

            w_numbers = collect_const(expr.func(*w_numbers))
            wo_numbers = expr.func(*wo_numbers)

            if aggressive is True and wo_numbers:
                for i in flatten(candidates):
                    wo_numbers = collect(wo_numbers, i)

            rebuilt = expr.func(w_numbers, wo_numbers)
            return rebuilt, []
        elif expr.is_Mul:
            rebuilt, candidates = zip(*[run(arg) for arg in expr.args])
            rebuilt = collect_const(expr.func(*rebuilt))
            return rebuilt, flatten(candidates)
        else:
            rebuilt, candidates = zip(*[run(arg) for arg in expr.args])
            return expr.func(*rebuilt), flatten(candidates)

    return run(expr)[0]

def xreplace_constrained(exprs, make, rule=None, costmodel=lambda e: True, repeat=False):
    """
    Unlike ``xreplace``, which replaces all objects specified in a mapper,
    this function replaces all objects satisfying two criteria: ::

        * The "matching rule" -- a function returning True if a node within ``expr``
            satisfies a given property, and as such should be replaced;
        * A "cost model" -- a function triggering replacement only if a certain
            cost (e.g., operation count) is exceeded. This function is optional.

    Note that there is not necessarily a relationship between the set of nodes
    for which the matching rule returns True and those nodes passing the cost
    model check. It might happen for example that, given the expression ``a + b``,
    all of ``a``, ``b``, and ``a + b`` satisfy the matching rule, but only
    ``a + b`` satisfies the cost model.

    :param exprs: The target SymPy expression, or a collection of SymPy expressions.
    :param make: Either a mapper M: K -> V, indicating how to replace an expression
                 in K with a symbol in V, or a function, used to construct new, unique
                 symbols. Such a function should take as input a parameter, used to
                 enumerate the new symbols.
    :param rule: The matching rule (a lambda function). May be left unspecified if
                 ``make`` is a mapper.
    :param costmodel: The cost model (a lambda function, optional).
    :param repeat: Repeatedly apply ``xreplace`` until no more replacements are
                   possible (optional, defaults to False).
    """
    found = OrderedDict()
    rebuilt = []

    # Define /replace()/ based on the user-provided /make/
    if isinstance(make, dict):
        rule = rule if rule is not None else (lambda i: i in make)
        replace = lambda i: make[i]
    else:
        assert callable(make) and callable(rule)

        def replace(expr):
            if isinstance(make, dict):
                return make[expr]
            temporary = found.get(expr)
            if temporary:
                return temporary
            else:
                temporary = make(replace.c)
                found[expr] = temporary
                replace.c += 1
                return temporary
        replace.c = 0  # Unique identifier for new temporaries

    def run(expr):
        if expr.is_Atom or q_indexed(expr):
            return expr, rule(expr)
        elif expr.is_Pow:
            base, flag = run(expr.base)
            return expr.func(base, expr.exp, evaluate=False), flag
        else:
            children = [run(a) for a in expr.args]
            matching = [a for a, flag in children if flag]
            other = [a for a, _ in children if a not in matching]
            if matching:
                matched = expr.func(*matching, evaluate=False)
                if len(matching) == len(children) and rule(expr):
                    # Go look for longer expressions first
                    return matched, True
                elif rule(matched) and costmodel(matched):
                    # Replace what I can replace, then give up
                    rebuilt = expr.func(*(other + [replace(matched)]), evaluate=False)
                    return rebuilt, False
                else:
                    # Replace flagged children, then give up
                    replaced = [replace(e) for e in matching if costmodel(e)]
                    unreplaced = [e for e in matching if not costmodel(e)]
                    rebuilt = expr.func(*(other + replaced + unreplaced), evaluate=False)
                    return rebuilt, False
            return expr.func(*other, evaluate=False), False

    # Process the provided expressions
    for expr in as_tuple(exprs):
        assert expr.is_Equality
        root = expr.rhs

        while True:
            ret, _ = run(root)
            if repeat and ret != root:
                root = ret
            else:
                rebuilt.append(expr.func(expr.lhs, ret))
                break

    # Post-process the output
    found = [Eq(v, k) for k, v in found.items()]

    return found + rebuilt, found

def xreplace_indices(exprs, mapper, candidates=None, only_rhs=False):
    """
    Create new expressions from ``exprs``, by replacing all index variables
    specified in mapper appearing as a tensor index. Only tensors whose symbolic
    name appears in ``candidates`` are considered if ``candidates`` is not None.
    """
    get = lambda i: i.rhs if only_rhs is True else i
    handle = flatten(retrieve_indexed(get(i)) for i in as_tuple(exprs))
    if candidates is not None:
        handle = [i for i in handle if i.base.label in candidates]
    mapper = dict(zip(handle, [i.xreplace(mapper) for i in handle]))
    replaced = [i.xreplace(mapper) for i in as_tuple(exprs)]
    return replaced if isinstance(exprs, Iterable) else replaced[0]

def common_subexprs_elimination(exprs, make, mode='default'):
    """
    Perform common subexpressions elimination.

    Note: the output is not guranteed to be topologically sorted.

    :param exprs: The target SymPy expression, or a collection of SymPy expressions.
    :param make: A function to construct symbols used for replacement.
                 The function takes as input an integer ID; ID is computed internally
                 and used as a unique identifier for the constructed symbols.
    """

    # Note: not defaulting to SymPy's CSE() function for three reasons:
    # - it also captures array index access functions (eg, i+1 in A[i+1] and B[i+1]);
    # - it sometimes "captures too much", losing factorization opportunities;
    # - very slow
    # TODO: a second "sympy" mode will be provided, relying on SymPy's CSE() but
    # also ensuring some sort of post-processing
    assert mode == 'default'  # Only supported mode ATM

    processed = list(exprs)
    mapped = []
    while True:
        # Detect redundancies
        counted = count(mapped + processed, q_op).items()
        targets = OrderedDict([(k, estimate_cost(k)) for k, v in counted if v > 1])
        if not targets:
            break

        # Create temporaries
        hit = max(targets.values())
        picked = [k for k, v in targets.items() if v == hit]
        mapper = OrderedDict([(e, make(len(mapped) + i)) for i, e in enumerate(picked)])

        # Apply repleacements
        processed = [e.xreplace(mapper) for e in processed]
        mapped = [e.xreplace(mapper) for e in mapped]
        mapped = [Eq(v, k) for k, v in reversed(list(mapper.items()))] + mapped

        # Prepare for the next round
        for k in picked:
            targets.pop(k)
    processed = mapped + processed

    # Simply renumber the temporaries in ascending order
    mapper = {i.lhs: j.lhs for i, j in zip(mapped, reversed(mapped))}
    processed = [e.xreplace(mapper) for e in processed]

    return processed

def compact_temporaries(exprs):
    """
    Drop temporaries consisting of single symbols.
    """
    g = temporaries_graph(exprs)

    mapper = {k: v.rhs for k, v in g.items()
              if v.is_scalar and (q_leaf(v.rhs) or v.rhs.is_Function)}

    processed = []
    for k, v in g.items():
        if k not in mapper:
            # The temporary /v/ is retained, and substitutions may be applied
            processed.append(v.xreplace(mapper))

    return processed

def pow_to_mul(expr):
    """
    Convert integer powers in an expression to Muls, like a**2 => a*a.
    Readapted from: ::

        stackoverflow.com/questions/14264431/expanding-algebraic-powers-in-python-sympy

    The readaptation was necessary to make it work with Indexed.
    """
    lhs, rhs = expr.args
    pows = [i for i in rhs.args if i.is_Pow]
    non_pows = [i for i in rhs.args if i not in pows]
    if not pows:
        return expr
    if any(not e.is_Integer or e <= 0 for b, e in (i.as_base_exp() for i in pows)):
        # Cannot handle powers containing non-integer non-positive exponents
        return expr
    muls = [sympy.Mul(*[b]*e, evaluate=False)
            for b, e in (i.as_base_exp() for i in pows)]
    rhs = rhs.func(*(muls + non_pows), evaluate=False)
    return expr.func(expr.lhs, rhs.func(*(muls + non_pows), evaluate=False))
<|endoftext|>


<commit_before>import warnings

from errno import EAGAIN

import zmq
import diesel

from diesel import logmod
from diesel.util.queue import Queue
from diesel.util.event import Event

zctx = zmq.Context()

class DieselZMQSocket(object):
    '''Integrate zmq's super fast event loop with ours.
    '''
    def __init__(self, socket, bind=None, connect=None, linger_time=1000):
        self.socket = socket
        from diesel.runtime import current_app
        from diesel.hub import IntWrap

        if bind:
            assert not connect
            self.socket.bind(bind)
        elif connect:
            assert not bind
            self.socket.connect(connect)

        self.hub = current_app.hub
        self.fd = IntWrap(self.socket.getsockopt(zmq.FD))

        self.write_gate = Event()
        self.read_gate = Event()
        self.linger_time = linger_time
        self.hub.register(self.fd, self.handle_transition, self.error, self.error)
        self.handle_transition()
        self.destroyed = False
        self.sent = 0
        self.received = 0

    def send(self, message, flags=0):
        while True:
            try:
                self.socket.send(message, zmq.NOBLOCK | flags)
            except zmq.ZMQError, e:
                if e.errno == EAGAIN:
                    self.handle_transition() # force re-evaluation of EVENTS
                    self.write_gate.wait()
                else:
                    raise
            else:
                self.handle_transition()
                self.sent += 1
                break

    def recv(self, copy=True):
        while True:
            try:
                m = self.socket.recv(zmq.NOBLOCK, copy=copy)
            except zmq.ZMQError, e:
                if e.errno == EAGAIN:
                    self.handle_transition() # force re-evaluation of EVENTS
                    self.read_gate.wait()
                else:
                    raise
            else:
                self.handle_transition()
                self.received += 1
                return m

    def handle_transition(self):
        '''Handle state change.
        '''

        events = self.socket.getsockopt(zmq.EVENTS)
        if events & zmq.POLLIN:
            self.read_gate.set()
        else:
            self.read_gate.clear()

        if events & zmq.POLLOUT:
            self.write_gate.set()
        else:
            self.write_gate.clear()

    def error(self):
        raise RuntimeError("OH NOES, some weird zeromq FD error callback")

    def __del__(self):
        if not self.destroyed:
            self.hub.unregister(self.fd)
            self.socket.close(self.linger_time)

    def __enter__(self):
        return self

    def __exit__(self, *args):
        if not self.destroyed:
            self.hub.unregister(self.fd)
            self.socket.close(self.linger_time)

class DieselZMQService(object):
    """A ZeroMQ service that can handle multiple clients.

    Clients must maintain a steady flow of messages in order to maintain
    state in the service. A heartbeat of some sort. Or the timeout can be
    set to a sufficiently large value understanding that it will cause more
    resource consumption.

    """
    name = ''
    default_log_level = logmod.LOGLVL_DEBUG
    timeout = 10

    def __init__(self, uri, logger=None, log_level=None):
        self.uri = uri
        self.zmq_socket = None
        self.log = logger or None
        self.selected_log_level = log_level
        self.clients = {}
        self.outgoing = Queue()
        self.incoming = Queue()
        self.name = self.name or self.__class__.__name__
        if self.log and self.selected_log_level is not None:
            self.selected_log_level = None
            warnings.warn(
                "ignored `log_level` argument since `logger` was provided.",
                RuntimeWarning,
                stacklevel=2,
            )

    def _create_zeromq_server_socket(self):
        # TODO support other ZeroMQ socket types
        low_level_sock = zctx.socket(zmq.ROUTER)
        self.zmq_socket = DieselZMQSocket(low_level_sock, bind=self.uri)

    def _setup_the_logging_system(self):
        if not self.log:
            if self.selected_log_level is not None:
                log_level = self.selected_log_level
            else:
                log_level = self.default_log_level
            log_name = self.name or self.__class__.__name__
            self.log = diesel.log.sublog(log_name, verbosity=log_level)

    def _handle_client_requests_and_responses(self, remote_client):
        assert self.zmq_socket
        queues = [remote_client.incoming, remote_client.outgoing]
        while True:
            (evt, value) = diesel.first(waits=queues, sleep=self.timeout)
            if evt is remote_client.incoming:
                resp = self.handle_client_packet(value, remote_client.context)
            elif evt is remote_client.outgoing:
                resp = value
            elif evt == 'sleep':
                break
            if resp:
                if isinstance(resp, basestring):
                    output = [resp]
                else:
                    output = iter(resp)
                for part in output:
                    self.outgoing.put((remote_client.token, part))
        self._cleanup_client(remote_client)

    def _cleanup_client(self, remote_client):
        del self.clients[remote_client.token]
        self.cleanup_client(remote_client)
        self.log.debug("cleaned up client %r" % remote_client.token)

    def _receive_incoming_packets(self):
        assert self.zmq_socket
        socket = self.zmq_socket
        while True:
            # TODO support receiving data from other socket types
            token_msg = socket.recv(copy=False)
            assert token_msg.more
            token = token_msg.bytes
            packet_raw = socket.recv()
            self.incoming.put((token, packet_raw))

    def _handle_all_inbound_and_outbound_traffic(self):
        assert self.zmq_socket
        diesel.fork_child(self._receive_incoming_packets)
        queues = [self.incoming, self.outgoing]
        while True:
            (queue, (token, data)) = diesel.first(waits=queues)

            if queue is self.incoming:
                if token not in self.clients:
                    self._register_client(token, data)
                self.clients[token].incoming.put(data)

            elif queue is self.outgoing:
                self.zmq_socket.send(token, zmq.SNDMORE)
                self.zmq_socket.send(data)

    def _register_client(self, token, packet):
        self.clients[token] = remote = RemoteClient(token)
        diesel.fork_child(self._handle_client_requests_and_responses, remote)
        self.register_client(remote, packet)

    # Public API
    # ==========

    def run(self):
        self._create_zeromq_server_socket()
        self._setup_the_logging_system()
        self._handle_all_inbound_and_outbound_traffic()

    def handle_client_packet(self, packet, context):
        """Called with a bytestring packet and dictionary context.

        Return an iterable of bytestrings.

        """
        raise NotImplementedError()

    def cleanup_client(self, remote_client):
        """Called with a RemoteClient instance. Do any cleanup you need to."""
        pass

    def register_client(self, remote_client, packet):
        """Called with a RemoteClient instance. Do any registration here."""
        pass

class RemoteClient(object):
    def __init__(self, token):

        # The token is the identifier sent along with thq ZeroMQ packet that
        # uniquely identifies the remote client.

        self.token = token

        # The incoming queue is typically populated by the DieselZMQService
        # and represents a queue of messages send from the remote client.

        self.incoming = Queue()

        # The outgoing queue is where return values from the
        # DieselZMQService.handle_client_packet method are placed. Those values
        # are sent on to the remote client.
        #
        # Other diesel threads can stick values directly into outgoing queue
        # and the service will send them on as well. This allows for
        # asynchronous sending of messages to remote clients. That's why it's
        # called 'async' in the context.

        self.outgoing = Queue()

        # The context in general is a place where you can put data that is
        # related specifically to the remote client and it will exist as long
        # the remote client doesn't timeout.

        self.context = {'async':self.outgoing, 'token':token}

<commit_msg>Fix for a potential race.<commit_after>import warnings

from errno import EAGAIN

import zmq
import diesel

from diesel import logmod
from diesel.util.queue import Queue
from diesel.util.event import Event

zctx = zmq.Context()

class DieselZMQSocket(object):
    '''Integrate zmq's super fast event loop with ours.
    '''
    def __init__(self, socket, bind=None, connect=None, linger_time=1000):
        self.socket = socket
        from diesel.runtime import current_app
        from diesel.hub import IntWrap

        if bind:
            assert not connect
            self.socket.bind(bind)
        elif connect:
            assert not bind
            self.socket.connect(connect)

        self.hub = current_app.hub
        self.fd = IntWrap(self.socket.getsockopt(zmq.FD))

        self.write_gate = Event()
        self.read_gate = Event()
        self.linger_time = linger_time
        self.hub.register(self.fd, self.handle_transition, self.error, self.error)
        self.handle_transition()
        self.destroyed = False
        self.sent = 0
        self.received = 0

    def send(self, message, flags=0):
        while True:
            try:
                self.socket.send(message, zmq.NOBLOCK | flags)
            except zmq.ZMQError, e:
                if e.errno == EAGAIN:
                    self.handle_transition() # force re-evaluation of EVENTS
                    self.write_gate.wait()
                else:
                    raise
            else:
                self.handle_transition()
                self.sent += 1
                break

    def recv(self, copy=True):
        while True:
            try:
                m = self.socket.recv(zmq.NOBLOCK, copy=copy)
            except zmq.ZMQError, e:
                if e.errno == EAGAIN:
                    self.handle_transition() # force re-evaluation of EVENTS
                    self.read_gate.wait()
                else:
                    raise
            else:
                self.handle_transition()
                self.received += 1
                return m

    def handle_transition(self):
        '''Handle state change.
        '''

        events = self.socket.getsockopt(zmq.EVENTS)
        if events & zmq.POLLIN:
            self.read_gate.set()
        else:
            self.read_gate.clear()

        if events & zmq.POLLOUT:
            self.write_gate.set()
        else:
            self.write_gate.clear()

    def error(self):
        raise RuntimeError("OH NOES, some weird zeromq FD error callback")

    def __del__(self):
        if not self.destroyed:
            self.hub.unregister(self.fd)
            self.socket.close(self.linger_time)

    def __enter__(self):
        return self

    def __exit__(self, *args):
        if not self.destroyed:
            self.hub.unregister(self.fd)
            self.socket.close(self.linger_time)

class DieselZMQService(object):
    """A ZeroMQ service that can handle multiple clients.

    Clients must maintain a steady flow of messages in order to maintain
    state in the service. A heartbeat of some sort. Or the timeout can be
    set to a sufficiently large value understanding that it will cause more
    resource consumption.

    """
    name = ''
    default_log_level = logmod.LOGLVL_DEBUG
    timeout = 10

    def __init__(self, uri, logger=None, log_level=None):
        self.uri = uri
        self.zmq_socket = None
        self.log = logger or None
        self.selected_log_level = log_level
        self.clients = {}
        self.outgoing = Queue()
        self.incoming = Queue()
        self.name = self.name or self.__class__.__name__
        if self.log and self.selected_log_level is not None:
            self.selected_log_level = None
            warnings.warn(
                "ignored `log_level` argument since `logger` was provided.",
                RuntimeWarning,
                stacklevel=2,
            )

    def _create_zeromq_server_socket(self):
        # TODO support other ZeroMQ socket types
        low_level_sock = zctx.socket(zmq.ROUTER)
        self.zmq_socket = DieselZMQSocket(low_level_sock, bind=self.uri)

    def _setup_the_logging_system(self):
        if not self.log:
            if self.selected_log_level is not None:
                log_level = self.selected_log_level
            else:
                log_level = self.default_log_level
            log_name = self.name or self.__class__.__name__
            self.log = diesel.log.sublog(log_name, verbosity=log_level)

    def _handle_client_requests_and_responses(self, remote_client):
        assert self.zmq_socket
        queues = [remote_client.incoming, remote_client.outgoing]
        while True:
            (evt, value) = diesel.first(waits=queues, sleep=self.timeout)
            if evt is remote_client.incoming:
                resp = self.handle_client_packet(value, remote_client.context)
            elif evt is remote_client.outgoing:
                resp = value
            elif evt == 'sleep':
                break
            if resp:
                if isinstance(resp, basestring):
                    output = [resp]
                else:
                    output = iter(resp)
                for part in output:
                    self.outgoing.put((remote_client.token, part))
        self._cleanup_client(remote_client)

    def _cleanup_client(self, remote_client):
        del self.clients[remote_client.token]
        self.cleanup_client(remote_client)
        self.log.debug("cleaned up client %r" % remote_client.token)

    def _receive_incoming_packets(self):
        assert self.zmq_socket
        socket = self.zmq_socket
        while True:
            # TODO support receiving data from other socket types
            token_msg = socket.recv(copy=False)
            assert token_msg.more
            token = token_msg.bytes
            packet_raw = socket.recv()
            self.incoming.put((token, packet_raw))

    def _handle_all_inbound_and_outbound_traffic(self):
        assert self.zmq_socket
        diesel.fork_child(self._receive_incoming_packets)
        queues = [self.incoming, self.outgoing]
        while True:
            (queue, (token, data)) = diesel.first(waits=queues)

            if queue is self.incoming:
                if token not in self.clients:
                    self._register_client(token, data)
                self.clients[token].incoming.put(data)

            elif queue is self.outgoing:
                self.zmq_socket.send(token, zmq.SNDMORE)
                self.zmq_socket.send(data)

    def _register_client(self, token, packet):
        self.clients[token] = remote = RemoteClient(token)
        self.register_client(remote, packet)
        diesel.fork_child(self._handle_client_requests_and_responses, remote)

    # Public API
    # ==========

    def run(self):
        self._create_zeromq_server_socket()
        self._setup_the_logging_system()
        self._handle_all_inbound_and_outbound_traffic()

    def handle_client_packet(self, packet, context):
        """Called with a bytestring packet and dictionary context.

        Return an iterable of bytestrings.

        """
        raise NotImplementedError()

    def cleanup_client(self, remote_client):
        """Called with a RemoteClient instance. Do any cleanup you need to."""
        pass

    def register_client(self, remote_client, packet):
        """Called with a RemoteClient instance. Do any registration here."""
        pass

class RemoteClient(object):
    def __init__(self, token):

        # The token is the identifier sent along with thq ZeroMQ packet that
        # uniquely identifies the remote client.

        self.token = token

        # The incoming queue is typically populated by the DieselZMQService
        # and represents a queue of messages send from the remote client.

        self.incoming = Queue()

        # The outgoing queue is where return values from the
        # DieselZMQService.handle_client_packet method are placed. Those values
        # are sent on to the remote client.
        #
        # Other diesel threads can stick values directly into outgoing queue
        # and the service will send them on as well. This allows for
        # asynchronous sending of messages to remote clients. That's why it's
        # called 'async' in the context.

        self.outgoing = Queue()

        # The context in general is a place where you can put data that is
        # related specifically to the remote client and it will exist as long
        # the remote client doesn't timeout.

        self.context = {'async':self.outgoing, 'token':token}

<|endoftext|>


<commit_before>#! /usr/bin/env python2
#
# This file is part of khmer, http://github.com/ged-lab/khmer/, and is
# Copyright (C) Michigan State University, 2009-2015. It is licensed under
# the three-clause BSD license; see doc/LICENSE.txt.
# Contact: khmer-project@idyll.org
#
# pylint: disable=invalid-name,missing-docstring
"""
Eliminate reads with median k-mer abundance higher than
DESIRED_COVERAGE.  Output sequences will be placed in 'infile.keep'.

% python scripts/normalize-by-median.py [ -C <cutoff> ] <data1> <data2> ...

Use '-h' for parameter help.
"""

import sys
import screed
import os
import khmer
import textwrap
from itertools import izip
from khmer.khmer_args import (build_counting_args, add_loadhash_args,
                              report_on_config, info)
import argparse
from khmer.kfile import (check_space, check_space_for_hashtable,
                         check_valid_file_exists)
from khmer.utils import write_record, check_is_pair
DEFAULT_DESIRED_COVERAGE = 10

MAX_FALSE_POSITIVE_RATE = 0.8             # see Zhang et al.,
# http://arxiv.org/abs/1309.2975

# Iterate a collection in arbitrary batches
# from: http://stackoverflow.com/questions/4628290/pairs-from-single-list

def batchwise(coll, size):
    iter_coll = iter(coll)
    return izip(*[iter_coll] * size)

# Returns true if the pair of records are properly pairs

# pylint: disable=too-many-locals,too-many-branches
def normalize_by_median(input_filename, outfp, htable, args, report_fp=None):

    desired_coverage = args.cutoff
    ksize = htable.ksize()

    # In paired mode we read two records at a time
    batch_size = 1
    if args.paired:
        batch_size = 2

    index = -1
    total = 0
    discarded = 0
    for index, batch in enumerate(batchwise(screed.open(
            input_filename, parse_description=False), batch_size)):
        if index > 0 and index % 100000 == 0:
            print >>sys.stderr, '... kept {kept} of {total} or'\
                ' {perc:2}%'.format(kept=total - discarded, total=total,
                                    perc=int(100. - discarded /
                                             float(total) * 100.))
            print >>sys.stderr, '... in file', input_filename

            if report_fp:
                print >> report_fp, total, total - discarded, \
                    1. - (discarded / float(total))
                report_fp.flush()

        total += batch_size

        # If in paired mode, check that the reads are properly interleaved

        if args.paired:
            if not check_is_pair(batch[0], batch[1]):
                raise IOError('Error: Improperly interleaved pairs \
                    {b0} {b1}'.format(b0=batch[0].name, b1=batch[1].name))

        # Emit the batch of reads if any read passes the filter
        # and all reads are longer than K
        passed_filter = False
        passed_length = True
        for record in batch:
            if len(record.sequence) < ksize:
                passed_length = False
                continue

            seq = record.sequence.replace('N', 'A')
            med, _, _ = htable.get_median_count(seq)

            if med < desired_coverage:
                htable.consume(seq)
                passed_filter = True

        # Emit records if any passed
        if passed_length and passed_filter:
            for record in batch:
                write_record(record, outfp)
        else:
            discarded += batch_size

    if report_fp:
        print >> report_fp, total, total - discarded, \
            1. - (discarded / float(total))
        report_fp.flush()

    return total, discarded

def handle_error(error, output_name, input_name, fail_save, htable):
    print >> sys.stderr, '** ERROR:', error
    print >> sys.stderr, '** Failed on {name}: '.format(name=input_name)
    if fail_save:
        tablename = os.path.basename(input_name) + '.ct.failed'
        print >> sys.stderr, \
            '** ...dumping k-mer counting table to {tn}'.format(tn=tablename)
        htable.save(tablename)
    try:
        os.remove(output_name)
    except:  # pylint: disable=bare-except
        print >> sys.stderr, '** ERROR: problem removing corrupt filtered file'

def get_parser():
    epilog = ("""
    Discard sequences based on whether or not their median k-mer abundance lies
    above a specified cutoff. Kept sequences will be placed in <fileN>.keep.

    Paired end reads will be considered together if :option:`-p` is set. If
    either read will be kept, then both will be kept. This should result in
    keeping (or discarding) each sequencing fragment. This helps with retention
    of repeats, especially.

    With :option:`-s`/:option:`--savetable`, the k-mer counting table
    will be saved to the specified file after all sequences have been
    processed. With :option:`-d`, the k-mer counting table will be
    saved every d files for multifile runs; if :option:`-s` is set,
    the specified name will be used, and if not, the name `backup.ct`
    will be used.  :option:`-l`/:option:`--loadtable` will load the
    specified k-mer counting table before processing the specified
    files.  Note that these tables are are in the same format as those
    produced by :program:`load-into-counting.py` and consumed by
    :program:`abundance-dist.py`.

    :option:`-f`/:option:`--fault-tolerant` will force the program to continue
    upon encountering a formatting error in a sequence file; the k-mer counting
    table up to that point will be dumped, and processing will continue on the
    next file.

    To append reads to an output file (rather than overwriting it), send output
    to STDOUT with `--out -` and use UNIX file redirection syntax (`>>`) to
    append to the file.

    Example::

        normalize-by-median.py -k 17 tests/test-data/test-abund-read-2.fa

    Example::

""" "        normalize-by-median.py -p -k 17 tests/test-data/test-abund-read-paired.fa"  # noqa
    """

    Example::

""" "        normalize-by-median.py -k 17 -f tests/test-data/test-error-reads.fq tests/test-data/test-fastq-reads.fq"  # noqa
    """

    Example::

""" "        normalize-by-median.py -k 17 -d 2 -s test.ct tests/test-data/test-abund-read-2.fa tests/test-data/test-fastq-reads")   # noqa
    parser = build_counting_args(
        descr="Do digital normalization (remove mostly redundant sequences)",
        epilog=textwrap.dedent(epilog))
    parser.add_argument('-C', '--cutoff', type=int,
                        default=DEFAULT_DESIRED_COVERAGE)
    parser.add_argument('-p', '--paired', action='store_true')
    parser.add_argument('-s', '--savetable', metavar="filename", default='',
                        help='save the k-mer counting table to disk after all'
                        'reads are loaded.')
    parser.add_argument('-R', '--report',
                        metavar='filename', type=argparse.FileType('w'))
    parser.add_argument('-f', '--fault-tolerant', dest='force',
                        help='continue on next file if read errors are \
                         encountered', action='store_true')
    parser.add_argument('--save-on-failure', dest='fail_save',
                        action='store_false', default=True,
                        help='Save k-mer counting table when an error occurs')
    parser.add_argument('-d', '--dump-frequency', dest='dump_frequency',
                        type=int, help='dump k-mer counting table every d '
                        'files', default=-1)
    parser.add_argument('-o', '--out', metavar="filename",
                        dest='single_output_file',
                        type=argparse.FileType('w'),
                        default=None, help='only output a single file with '
                        'the specified filename; use a single dash "-" to '
                        'specify that output should go to STDOUT (the '
                        'terminal)')
    parser.add_argument('input_filenames', metavar='input_sequence_filename',
                        help='Input FAST[AQ] sequence filename.', nargs='+')
    parser.add_argument('--report-total-kmers', '-t', action='store_true',
                        help="Prints the total number of k-mers"
                        " post-normalization to stderr")
    parser.add_argument('--force', default=False, action='store_true',
                        help='Overwrite output file if it exists')
    add_loadhash_args(parser)
    return parser

def main():  # pylint: disable=too-many-branches,too-many-statements
    info('normalize-by-median.py', ['diginorm'])
    args = get_parser().parse_args()

    report_on_config(args)

    report_fp = args.report

    check_valid_file_exists(args.input_filenames)
    check_space(args.input_filenames, args.force)
    if args.savetable:
        check_space_for_hashtable(
            args.n_tables * args.min_tablesize, args.force)

    # list to save error files along with throwing exceptions
    if args.force:
        corrupt_files = []

    if args.loadtable:
        print 'loading k-mer counting table from', args.loadtable
        htable = khmer.load_counting_hash(args.loadtable)
    else:
        print >> sys.stderr, 'making k-mer counting table'
        htable = khmer.new_counting_hash(args.ksize, args.min_tablesize,
                                         args.n_tables)

    total = 0
    discarded = 0
    input_filename = None

    if args.single_output_file:
        outfp = args.single_output_file
        try:
            output_name = args.single_output_file.name
        except AttributeError:
            output_name = '<stdout>'

    for index, input_filename in enumerate(args.input_filenames):
        if not args.single_output_file:
            output_name = os.path.basename(input_filename) + '.keep'
            outfp = open(output_name, 'w')

        total_acc = 0
        discarded_acc = 0

        try:
            total_acc, discarded_acc = normalize_by_median(input_filename,
                                                           outfp, htable, args,
                                                           report_fp)
        except IOError as err:
            handle_error(err, output_name, input_filename, args.fail_save,
                         htable)
            if not args.force:
                print >> sys.stderr, '** Exiting!'

                sys.exit(1)
            else:
                print >> sys.stderr, '*** Skipping error file, moving on...'
                corrupt_files.append(input_filename)
        else:
            if total_acc == 0 and discarded_acc == 0:
                print >> sys.stderr, 'SKIPPED empty file', input_filename
            else:
                total += total_acc
                discarded += discarded_acc
                print >> sys.stderr, \
                    'DONE with {inp}; kept {kept} of {total} or {perc:2}%'\
                    .format(inp=input_filename, kept=total - discarded,
                            total=total, perc=int(100. - discarded /
                                                  float(total) * 100.))
                print >> sys.stderr, 'output in', output_name

        if (args.dump_frequency > 0 and
                index > 0 and index % args.dump_frequency == 0):
            print 'Backup: Saving k-mer counting file through', input_filename
            if args.savetable:
                hashname = args.savetable
                print '...saving to', hashname
            else:
                hashname = 'backup.ct'
                print 'Nothing given for savetable, saving to', hashname
            htable.save(hashname)

    if args.report_total_kmers:
        print >> sys.stderr, 'Total number of unique k-mers: {0}'.format(
            htable.n_unique_kmers())

    if args.savetable:
        print 'Saving k-mer counting table through', input_filename
        print '...saving to', args.savetable
        htable.save(args.savetable)

    fp_rate = khmer.calc_expected_collisions(htable)
    print >> sys.stderr, \
        'fp rate estimated to be {fpr:1.3f}'.format(fpr=fp_rate)

    if args.force and len(corrupt_files) > 0:
        print >> sys.stderr, "** WARNING: Finished with errors!"
        print >> sys.stderr, "** IOErrors occurred in the following files:"
        print >> sys.stderr, "\t", " ".join(corrupt_files)

    if fp_rate > MAX_FALSE_POSITIVE_RATE:
        print >> sys.stderr, "**"
        print >> sys.stderr, ("** ERROR: the k-mer counting table is too small"
                              " for this data set. Increase tablesize/# "
                              "tables.")
        print >> sys.stderr, "**"
        print >> sys.stderr, "** Do not use these results!!"
        if not args.force:
            sys.exit(1)

if __name__ == '__main__':
    main()

# vim: set ft=python ts=4 sts=4 sw=4 et tw=79:
<commit_msg>Added example to CLI<commit_after>#! /usr/bin/env python2
#
# This file is part of khmer, http://github.com/ged-lab/khmer/, and is
# Copyright (C) Michigan State University, 2009-2015. It is licensed under
# the three-clause BSD license; see doc/LICENSE.txt.
# Contact: khmer-project@idyll.org
#
# pylint: disable=invalid-name,missing-docstring
"""
Eliminate reads with median k-mer abundance higher than
DESIRED_COVERAGE.  Output sequences will be placed in 'infile.keep'.

% python scripts/normalize-by-median.py [ -C <cutoff> ] <data1> <data2> ...

Use '-h' for parameter help.
"""

import sys
import screed
import os
import khmer
import textwrap
from itertools import izip
from khmer.khmer_args import (build_counting_args, add_loadhash_args,
                              report_on_config, info)
import argparse
from khmer.kfile import (check_space, check_space_for_hashtable,
                         check_valid_file_exists)
from khmer.utils import write_record, check_is_pair
DEFAULT_DESIRED_COVERAGE = 10

MAX_FALSE_POSITIVE_RATE = 0.8             # see Zhang et al.,
# http://arxiv.org/abs/1309.2975

# Iterate a collection in arbitrary batches
# from: http://stackoverflow.com/questions/4628290/pairs-from-single-list

def batchwise(coll, size):
    iter_coll = iter(coll)
    return izip(*[iter_coll] * size)

# Returns true if the pair of records are properly pairs

# pylint: disable=too-many-locals,too-many-branches
def normalize_by_median(input_filename, outfp, htable, args, report_fp=None):

    desired_coverage = args.cutoff
    ksize = htable.ksize()

    # In paired mode we read two records at a time
    batch_size = 1
    if args.paired:
        batch_size = 2

    index = -1
    total = 0
    discarded = 0
    for index, batch in enumerate(batchwise(screed.open(
            input_filename, parse_description=False), batch_size)):
        if index > 0 and index % 100000 == 0:
            print >>sys.stderr, '... kept {kept} of {total} or'\
                ' {perc:2}%'.format(kept=total - discarded, total=total,
                                    perc=int(100. - discarded /
                                             float(total) * 100.))
            print >>sys.stderr, '... in file', input_filename

            if report_fp:
                print >> report_fp, total, total - discarded, \
                    1. - (discarded / float(total))
                report_fp.flush()

        total += batch_size

        # If in paired mode, check that the reads are properly interleaved

        if args.paired:
            if not check_is_pair(batch[0], batch[1]):
                raise IOError('Error: Improperly interleaved pairs \
                    {b0} {b1}'.format(b0=batch[0].name, b1=batch[1].name))

        # Emit the batch of reads if any read passes the filter
        # and all reads are longer than K
        passed_filter = False
        passed_length = True
        for record in batch:
            if len(record.sequence) < ksize:
                passed_length = False
                continue

            seq = record.sequence.replace('N', 'A')
            med, _, _ = htable.get_median_count(seq)

            if med < desired_coverage:
                htable.consume(seq)
                passed_filter = True

        # Emit records if any passed
        if passed_length and passed_filter:
            for record in batch:
                write_record(record, outfp)
        else:
            discarded += batch_size

    if report_fp:
        print >> report_fp, total, total - discarded, \
            1. - (discarded / float(total))
        report_fp.flush()

    return total, discarded

def handle_error(error, output_name, input_name, fail_save, htable):
    print >> sys.stderr, '** ERROR:', error
    print >> sys.stderr, '** Failed on {name}: '.format(name=input_name)
    if fail_save:
        tablename = os.path.basename(input_name) + '.ct.failed'
        print >> sys.stderr, \
            '** ...dumping k-mer counting table to {tn}'.format(tn=tablename)
        htable.save(tablename)
    try:
        os.remove(output_name)
    except:  # pylint: disable=bare-except
        print >> sys.stderr, '** ERROR: problem removing corrupt filtered file'

def get_parser():
    epilog = ("""
    Discard sequences based on whether or not their median k-mer abundance lies
    above a specified cutoff. Kept sequences will be placed in <fileN>.keep.

    Paired end reads will be considered together if :option:`-p` is set. If
    either read will be kept, then both will be kept. This should result in
    keeping (or discarding) each sequencing fragment. This helps with retention
    of repeats, especially.

    With :option:`-s`/:option:`--savetable`, the k-mer counting table
    will be saved to the specified file after all sequences have been
    processed. With :option:`-d`, the k-mer counting table will be
    saved every d files for multifile runs; if :option:`-s` is set,
    the specified name will be used, and if not, the name `backup.ct`
    will be used.  :option:`-l`/:option:`--loadtable` will load the
    specified k-mer counting table before processing the specified
    files.  Note that these tables are are in the same format as those
    produced by :program:`load-into-counting.py` and consumed by
    :program:`abundance-dist.py`.

    :option:`-f`/:option:`--fault-tolerant` will force the program to continue
    upon encountering a formatting error in a sequence file; the k-mer counting
    table up to that point will be dumped, and processing will continue on the
    next file.

    To append reads to an output file (rather than overwriting it), send output
    to STDOUT with `--out -` and use UNIX file redirection syntax (`>>`) to
    append to the file.

    Example::

        normalize-by-median.py -k 17 tests/test-data/test-abund-read-2.fa

    Example::

""" "        normalize-by-median.py -p -k 17 tests/test-data/test-abund-read-paired.fa"  # noqa
    """

    Example::

""" "        normalize-by-median.py -p -k 17 -o - tests/test-data/test-fastq-reads.fq >> appended-output.fq"  # noqa
    """

    Example::

""" "        normalize-by-median.py -k 17 -f tests/test-data/test-error-reads.fq tests/test-data/test-fastq-reads.fq"  # noqa
    """

    Example::

""" "        normalize-by-median.py -k 17 -d 2 -s test.ct tests/test-data/test-abund-read-2.fa tests/test-data/test-fastq-reads")   # noqa
    parser = build_counting_args(
        descr="Do digital normalization (remove mostly redundant sequences)",
        epilog=textwrap.dedent(epilog))
    parser.add_argument('-C', '--cutoff', type=int,
                        default=DEFAULT_DESIRED_COVERAGE)
    parser.add_argument('-p', '--paired', action='store_true')
    parser.add_argument('-s', '--savetable', metavar="filename", default='',
                        help='save the k-mer counting table to disk after all'
                        'reads are loaded.')
    parser.add_argument('-R', '--report',
                        metavar='filename', type=argparse.FileType('w'))
    parser.add_argument('-f', '--fault-tolerant', dest='force',
                        help='continue on next file if read errors are \
                         encountered', action='store_true')
    parser.add_argument('--save-on-failure', dest='fail_save',
                        action='store_false', default=True,
                        help='Save k-mer counting table when an error occurs')
    parser.add_argument('-d', '--dump-frequency', dest='dump_frequency',
                        type=int, help='dump k-mer counting table every d '
                        'files', default=-1)
    parser.add_argument('-o', '--out', metavar="filename",
                        dest='single_output_file',
                        type=argparse.FileType('w'),
                        default=None, help='only output a single file with '
                        'the specified filename; use a single dash "-" to '
                        'specify that output should go to STDOUT (the '
                        'terminal)')
    parser.add_argument('input_filenames', metavar='input_sequence_filename',
                        help='Input FAST[AQ] sequence filename.', nargs='+')
    parser.add_argument('--report-total-kmers', '-t', action='store_true',
                        help="Prints the total number of k-mers"
                        " post-normalization to stderr")
    parser.add_argument('--force', default=False, action='store_true',
                        help='Overwrite output file if it exists')
    add_loadhash_args(parser)
    return parser

def main():  # pylint: disable=too-many-branches,too-many-statements
    info('normalize-by-median.py', ['diginorm'])
    args = get_parser().parse_args()

    report_on_config(args)

    report_fp = args.report

    check_valid_file_exists(args.input_filenames)
    check_space(args.input_filenames, args.force)
    if args.savetable:
        check_space_for_hashtable(
            args.n_tables * args.min_tablesize, args.force)

    # list to save error files along with throwing exceptions
    if args.force:
        corrupt_files = []

    if args.loadtable:
        print 'loading k-mer counting table from', args.loadtable
        htable = khmer.load_counting_hash(args.loadtable)
    else:
        print >> sys.stderr, 'making k-mer counting table'
        htable = khmer.new_counting_hash(args.ksize, args.min_tablesize,
                                         args.n_tables)

    total = 0
    discarded = 0
    input_filename = None

    if args.single_output_file:
        outfp = args.single_output_file
        try:
            output_name = args.single_output_file.name
        except AttributeError:
            output_name = '<stdout>'

    for index, input_filename in enumerate(args.input_filenames):
        if not args.single_output_file:
            output_name = os.path.basename(input_filename) + '.keep'
            outfp = open(output_name, 'w')

        total_acc = 0
        discarded_acc = 0

        try:
            total_acc, discarded_acc = normalize_by_median(input_filename,
                                                           outfp, htable, args,
                                                           report_fp)
        except IOError as err:
            handle_error(err, output_name, input_filename, args.fail_save,
                         htable)
            if not args.force:
                print >> sys.stderr, '** Exiting!'

                sys.exit(1)
            else:
                print >> sys.stderr, '*** Skipping error file, moving on...'
                corrupt_files.append(input_filename)
        else:
            if total_acc == 0 and discarded_acc == 0:
                print >> sys.stderr, 'SKIPPED empty file', input_filename
            else:
                total += total_acc
                discarded += discarded_acc
                print >> sys.stderr, \
                    'DONE with {inp}; kept {kept} of {total} or {perc:2}%'\
                    .format(inp=input_filename, kept=total - discarded,
                            total=total, perc=int(100. - discarded /
                                                  float(total) * 100.))
                print >> sys.stderr, 'output in', output_name

        if (args.dump_frequency > 0 and
                index > 0 and index % args.dump_frequency == 0):
            print 'Backup: Saving k-mer counting file through', input_filename
            if args.savetable:
                hashname = args.savetable
                print '...saving to', hashname
            else:
                hashname = 'backup.ct'
                print 'Nothing given for savetable, saving to', hashname
            htable.save(hashname)

    if args.report_total_kmers:
        print >> sys.stderr, 'Total number of unique k-mers: {0}'.format(
            htable.n_unique_kmers())

    if args.savetable:
        print 'Saving k-mer counting table through', input_filename
        print '...saving to', args.savetable
        htable.save(args.savetable)

    fp_rate = khmer.calc_expected_collisions(htable)
    print >> sys.stderr, \
        'fp rate estimated to be {fpr:1.3f}'.format(fpr=fp_rate)

    if args.force and len(corrupt_files) > 0:
        print >> sys.stderr, "** WARNING: Finished with errors!"
        print >> sys.stderr, "** IOErrors occurred in the following files:"
        print >> sys.stderr, "\t", " ".join(corrupt_files)

    if fp_rate > MAX_FALSE_POSITIVE_RATE:
        print >> sys.stderr, "**"
        print >> sys.stderr, ("** ERROR: the k-mer counting table is too small"
                              " for this data set. Increase tablesize/# "
                              "tables.")
        print >> sys.stderr, "**"
        print >> sys.stderr, "** Do not use these results!!"
        if not args.force:
            sys.exit(1)

if __name__ == '__main__':
    main()

# vim: set ft=python ts=4 sts=4 sw=4 et tw=79:
<|endoftext|>


<commit_before>import pandas as pd
import numpy as np
import argparse

def fix_variant_type(var_clf):
    if 'missense' in var_clf.lower():
        return 'Missense_Mutation'
    elif 'splice_site' in var_clf.lower():
        return 'Splice_Site'
    elif 'frame_shift' in var_clf.lower():
        return 'Frame_Shift_Indel'
    elif 'in_frame' in var_clf.lower():
        return 'In_Frame_Indel'
    elif 'synonymous' in var_clf.lower():
        return 'Silent'
    elif 'nonsense' in var_clf.lower():
        return 'Nonsense'
    elif 'silent' in var_clf.lower():
        return 'Silent'
    elif 'nonstop' in var_clf.lower():
        return 'Nonstop_Mutation'
    elif 'translation_start_site' in var_clf.lower():
        return 'Translation_Start_Site'
    return var_clf

def generate_hgvs_syntax(df):
    hgvs_list = []
    for row in df.iterrows():
        row = row[1]  # grab the pandas series object
        seq_ont = row.ix['Sequence Ontology']
        pos = row.ix['Amino acid position']
        if seq_ont in ['SY', 'SL', 'SG', 'MS', 'CS']:
            ref = row.ix['Reference amino acid(s)']
            alt = row.ix['Alternate amino acid(s)']
            hgvs_string = 'p.{0}{1}{2}'.format(ref, pos, alt)
            hgvs_list.append(hgvs_string)
        elif seq_ont == 'II':
            ins_len = len(row.ix['Alternate base'][:-1]) / 3
            hgvs_string = 'p.K{0}_K{1}ins{2}'.format(pos, pos+1, 'K'*ins_len)
            hgvs_list.append(hgvs_string)
        elif seq_ont == 'ID':
            del_len = len(row.ix['Reference base']) / 3
            hgvs_string = 'p.K{0}_K{1}del{2}'.format(pos, pos+del_len-1, 'K'*del_len)
            hgvs_list.append(hgvs_string)
        elif seq_ont == 'FI' or seq_ont == 'FD':
            hgvs_string = 'p.K{0}fs*'.format(pos)
            hgvs_list.append(hgvs_string)
        elif seq_ont is np.nan:
            hgvs_list.append('p.?')
    return hgvs_list

def parse_arguments():
    parser = argparse.ArgumentParser()
    parser.add_argument('-m', '--maf',
                        type=str, action='store',
                        help='Filtered MAF file from broad website')
    parser.add_argument('-c', '--cravat',
                        type=str, action='store',
                        help='Cravat output which includes amino '
                        'acid info (Variant_Analysis.tsv)')
    parser.add_argument('-o', '--output',
                        type=str, action='store',
                        help='Modified MAF format acceptable for input')
    args = parser.parse_args()
    return vars(args)

def main(opts):
    # read in data
    cravat_df = pd.read_csv(opts['cravat'], sep='\t')
    broad_df = pd.read_csv(opts['maf'], sep='\t')

    # get hgvs strings from cravat output
    hgvs_list = generate_hgvs_syntax(cravat_df)
    broad_df['Protein_Change'] = hgvs_list

    # rename headers
    broad_df.rename(columns={'ttype': 'Tumor_Type',
                             'patient': 'Tumor_Sample',
                             'gene': 'Gene_Symbol',
                             'type': 'Variant_Classification',
                             'chr': 'Chromosome',
                             'ref_allele': 'Reference_Allele',
                             'newbase': 'Tumor_Allele',
                             'pos': 'Start_Position'},
                    inplace=True)

    # add end position column
    broad_df['End_Position'] = broad_df['Start_Position'] + broad_df['Tumor_Allele'].apply(lambda x: len(x) - 1)

    # fix variant classigfication column
    broad_df['Variant_Classification'] = broad_df['Variant_Classification'].apply(fix_variant_type)

    # add chr to chromosome names
    broad_df['Chromosome'] = broad_df['Chromosome'].astype(str).apply(lambda x: 'chr' + x)

    # output results to file
    cols_of_interest = ['Gene_Symbol', 'Tumor_Sample', 'Tumor_Type',
                        'Chromosome', 'Start_Position',
                        'End_Position', 'Variant_Classification',
                        'Reference_Allele', 'Tumor_Allele',
                        'Protein_Change']
    broad_df[cols_of_interest].to_csv(opts['output'], sep='\t', index=False)

if __name__ == "__main__":
    opts = parse_arguments()
    main(opts)
<commit_msg>Fixed bug where amino acid position was converted to float if NaN ocurred.<commit_after>import pandas as pd
import numpy as np
import argparse

def fix_variant_type(var_clf):
    if 'missense' in var_clf.lower():
        return 'Missense_Mutation'
    elif 'splice_site' in var_clf.lower():
        return 'Splice_Site'
    elif 'frame_shift' in var_clf.lower():
        return 'Frame_Shift_Indel'
    elif 'in_frame' in var_clf.lower():
        return 'In_Frame_Indel'
    elif 'synonymous' in var_clf.lower():
        return 'Silent'
    elif 'nonsense' in var_clf.lower():
        return 'Nonsense'
    elif 'silent' in var_clf.lower():
        return 'Silent'
    elif 'nonstop' in var_clf.lower():
        return 'Nonstop_Mutation'
    elif 'translation_start_site' in var_clf.lower():
        return 'Translation_Start_Site'
    return var_clf

def generate_hgvs_syntax(df):
    hgvs_list = []
    for row in df.iterrows():
        row = row[1]  # grab the pandas series object
        seq_ont = row.ix['Sequence Ontology']
        nulls_in_row = row.isnull()

        # fix bug related to pandas forcing conversion of
        # dtype to float if there exists a NaN
        if not nulls_in_row['Amino acid position']:
            pos = int(row.ix['Amino acid position'])
        else:
            pos = row.ix['Amino acid position']

        if seq_ont in ['SY', 'SL', 'SG', 'MS', 'CS']:
            ref = row.ix['Reference amino acid(s)']
            alt = row.ix['Alternate amino acid(s)']
            hgvs_string = 'p.{0}{1}{2}'.format(ref, pos, alt)
            hgvs_list.append(hgvs_string)
        elif seq_ont == 'II':
            ins_len = len(row.ix['Alternate base'][:-1]) / 3
            hgvs_string = 'p.K{0}_K{1}ins{2}'.format(pos, pos+1, 'K'*ins_len)
            hgvs_list.append(hgvs_string)
        elif seq_ont == 'ID':
            del_len = len(row.ix['Reference base']) / 3
            hgvs_string = 'p.K{0}_K{1}del{2}'.format(pos, pos+del_len-1, 'K'*del_len)
            hgvs_list.append(hgvs_string)
        elif seq_ont == 'FI' or seq_ont == 'FD':
            hgvs_string = 'p.K{0}fs*'.format(pos)
            hgvs_list.append(hgvs_string)
        elif seq_ont is np.nan:
            hgvs_list.append('p.?')
    return hgvs_list

def parse_arguments():
    parser = argparse.ArgumentParser()
    parser.add_argument('-m', '--maf',
                        type=str, action='store',
                        help='Filtered MAF file from broad website')
    parser.add_argument('-c', '--cravat',
                        type=str, action='store',
                        help='Cravat output which includes amino '
                        'acid info (Variant_Analysis.tsv)')
    parser.add_argument('-o', '--output',
                        type=str, action='store',
                        help='Modified MAF format acceptable for input')
    args = parser.parse_args()
    return vars(args)

def main(opts):
    # read in data
    cravat_df = pd.read_csv(opts['cravat'], sep='\t')
    broad_df = pd.read_csv(opts['maf'], sep='\t')

    # get hgvs strings from cravat output
    hgvs_list = generate_hgvs_syntax(cravat_df)
    broad_df['Protein_Change'] = hgvs_list

    # rename headers
    broad_df.rename(columns={'ttype': 'Tumor_Type',
                             'patient': 'Tumor_Sample',
                             'gene': 'Gene_Symbol',
                             'type': 'Variant_Classification',
                             'chr': 'Chromosome',
                             'ref_allele': 'Reference_Allele',
                             'newbase': 'Tumor_Allele',
                             'pos': 'Start_Position'},
                    inplace=True)

    # add end position column
    broad_df['End_Position'] = broad_df['Start_Position'] + broad_df['Tumor_Allele'].apply(lambda x: len(x) - 1)

    # fix variant classigfication column
    broad_df['Variant_Classification'] = broad_df['Variant_Classification'].apply(fix_variant_type)

    # add chr to chromosome names
    broad_df['Chromosome'] = broad_df['Chromosome'].astype(str).apply(lambda x: 'chr' + x)

    # output results to file
    cols_of_interest = ['Gene_Symbol', 'Tumor_Sample', 'Tumor_Type',
                        'Chromosome', 'Start_Position',
                        'End_Position', 'Variant_Classification',
                        'Reference_Allele', 'Tumor_Allele',
                        'Protein_Change']
    broad_df[cols_of_interest].to_csv(opts['output'], sep='\t', index=False)

if __name__ == "__main__":
    opts = parse_arguments()
    main(opts)
<|endoftext|>


<commit_before>#!/usr/bin/python3

import os
import sys

from .autorepr import autorepr
__all__ = ['RedirectStreams', 'TeeStreams', 'Tee']

def validate_init(self, *, stdout=None, stderr=None, noclobber=False, use_temp=False, mode='w'):
    self.use_temp = use_temp
    self.mode = mode
    self.noclobber = noclobber
    if noclobber:
        for i in stdout, stderr:
            if i is not None and os.path.lexists(i):
                raise RuntimeError("%s already exists" % i)
    self.stdout = stdout
    self.stderr = stderr

@autorepr
class RedirectStreams(object):
    """Redirect the standard streams somewhere"""
    __init__ = validate_init
    def __enter__(self):
        self.files = []
        if self.stdout is None:
            self.stdout = sys.stdout
        else:
            f = open(self.stdout, 'w')
            self.stdout = f
            self.files.append(f)
        if self.stderr is None:
            self.stderr = sys.stderr
        elif self.stderr == self.stdout:
            self.stderr = f
        else:
            f = open(self.stderr, 'w')
            self.stderr = f
            self.files.append(f)
        sys.stdout.flush() ; sys.stderr.flush()
        sys.stdout, sys.stderr = self.stdout, self.stderr
    def __exit__(self, etype, value, trace):
        sys.stdout.flush() ; sys.stderr.flush()
        sys.stdout = sys.__stdout__
        sys.stderr = sys.__stderr__
        for i in self.files: i.close()

class Tee(object):
    def __init__(*outputs):
        self.outputs = outputs
    def write(self, val):
        for i in self.outputs: i.write(val)
    def flush(self):
        for i in self.outputs: i.flush()
    def writelines(self, lines):
        for i in self.outputs: i.writelines(lines)
    def __repr__(self):
        return self.__class__.__name__ + "\n\t".join(repr(i) for i in self.outputs)

@autorepr
class TeeStreams(object):
    """Tee the standard streams to a file."""
    __init__ = validate_init
    def __enter__(self):
        self.files = []
        if self.stdout is None:
            self.stdout = sys.stdout
        else:
            f = open(self.stdout, 'w')
            self.stdout = Tee(sys.stdout, f)
            self.files.append(f)
        if self.stderr is None:
            self.stderr = sys.stderr
        elif self.stderr == self.stdout:
            self.stderr = Tee(sys.stderr, f)
        else:
            f = open(self.stderr, 'w')
            self.stderr = Tee(sys.stderr, f)
            self.files.append(f)
        sys.stdout.flush() ; sys.stderr.flush()
        sys.stdout, sys.stderr = self.stdout, self.stderr
    def __exit__(self, etype, value, trace):
        sys.stdout.flush() ; sys.stderr.flush()
        sys.stdout = sys.__stdout__
        sys.stderr = sys.__stderr__
        for i in self.files: i.close()

@autorepr
class LoggerFilelike(object):
    def __init__(self, logger, level):
        self.logger = logger
        self.level = level
    def __write__(self, message):
        self.logger(self.level, message)
    def __flush__(self):
        pass

@autorepr
class LogStderr(object):
    def __init__(self, logger, level):
        self.logger = logger
        self.level = level
    def __enter__(self):
        self.stderr = LoggerFilelike(self.logger, self.level)
        sys.stderr.flush()
        sys.stderr = self.stderr
    def __exit__(self, etype, value, trace):
        sys.stderr.flush()
        sys.stderr = sys.__stderr__
<commit_msg>fixing the context managers<commit_after>#!/usr/bin/python3

import os
import shutil
import sys
import tempfile

from .autorepr import autorepr
__all__ = ['RedirectStreams', 'TeeStreams', 'Tee', 'LoggerFilelike', 'LogStderr']

def validate_init(self, *, stdout=None, stderr=None, noclobber=False, use_temp=False, mode='w'):
    self.use_temp = use_temp
    self.mode = mode
    self.noclobber = noclobber
    if noclobber:
        for i in stdout, stderr:
            if i is not None and os.path.lexists(i):
                raise RuntimeError("%s already exists" % i)
    self.stdout = stdout
    self.stderr = stderr

@autorepr
class RedirectStreams(object):
    """Redirect the standard streams somewhere"""
    __init__ = validate_init
    def reset(self):
        self.files = [None,None]
        self.tmppaths = [None,None]
    def __enter__(self):
        self.reset()
        if self.stdout is not None:
            if self.noclobber and os.path.lexists(self.stdout):
                raise RuntimeError("%s already exists" % self.stdout)
            if self.use_temp:
                self.files[0] = tempfile.NamedTemporaryFile(delete=False)
                self.tmppaths[0] = self.files[0].name
            else:
                self.files[0] = open(self.stdout, self.mode)
            sys.stdout.flush()
            sys.stdout = self.files[0]
        if self.stderr is None:
            pass
        elif self.stderr == self.stdout:
            sys.stderr.flush()
            sys.stderr = self.files[0]
        else:
            if self.noclobber and os.path.lexists(self.stderr):
                raise RuntimeError("%s already exists" % self.stderr)
            if self.use_temp:
                self.files[1] = tempfile.NamedTemporaryFile(delete=False)
                self.tmppaths[1] = self.files[1].name
            else:
                self.files[1] = open(self.stderr, self.mode)
            sys.stderr.flush()
            sys.stderr = self.files[1]
    def __exit__(self, etype, value, trace):
        sys.stdout.flush() ; sys.stderr.flush()
        sys.stdout = sys.__stdout__
        sys.stderr = sys.__stderr__
        if self.use_temp:
            for tmpfile, path, dest in zip(self.files, self.temppaths, [self.stdout, self.stderr]):
                if tmpfile:
                    tmpfile.flush()
                    shutil.copyfile(path, dest)
                    os.remove(path)
        for i in self.files:
            if i is not None: i.close()
        self.reset()

class Tee(object):
    def __init__(*outputs):
        self.outputs = outputs
    def write(self, val):
        for i in self.outputs: i.write(val)
    def flush(self):
        for i in self.outputs: i.flush()
    def writelines(self, lines):
        for i in self.outputs: i.writelines(lines)
    def __repr__(self):
        return self.__class__.__name__ + "\n\t".join(repr(i) for i in self.outputs)

@autorepr
class TeeStreams(object):
    """Tee the standard streams to a file."""
    __init__ = validate_init
    def reset(self):
        self.tees = [None, None]
        self.files = [None, None]
        self.tmppaths = [None, None]
    def __enter__(self):
        self.reset()
        if self.stdout is not None:
            if self.noclobber and os.path.lexists(self.stdout):
                raise RuntimeError("%s already exists" % self.stdout)
            if self.use_temp:
                self.files[0] = tempfile.NamedTemporaryFile(delete=False)
                self.tmppaths[0] = self.files[0].name
            else:
                self.files[0] = open(self.stdout, self.mode)
            self.tees[0] = Tee(sys.stdout, self.files[0])
            sys.stdout.flush()
            sys.stdout = self.tees[0]
        if self.stderr is None:
            pass
        elif self.stderr == self.stdout:
            self.tees[1] = Tee(sys.stderr, f)
            sys.stderr.flush()
            sys.stderr = self.tees[1]
        else:
            if self.noclobber and os.path.lexists(self.stderr):
                raise RuntimeError("%s already exists" % self.stderr)
            if self.use_temp:
                self.files[1] = tempfile.NamedTemporaryFile(delete=False)
                self.tmppaths[1] = self.files[1].name
            else:
                self.files[1] = open(self.stderr, self.mode)
            self.tees[1] = Tee(sys.stderr, self.files[1])
            sys.stderr.flush()
            sys.stderr = self.tees[1]
    def __exit__(self, etype, value, trace):
        sys.stdout.flush() ; sys.stderr.flush()
        sys.stdout = sys.__stdout__
        sys.stderr = sys.__stderr__
        if self.use_temp:
            for tmpfile, path, dest in zip(self.files, self.temppaths, [self.stdout, self.stderr]):
                if tmpfile:
                    tmpfile.flush()
                    shutil.copyfile(path, dest)
                    os.remove(path)
        for i in self.files:
            if i is not None: i.close()
        self.reset()

@autorepr
class LoggerFilelike(object):
    def __init__(self, logger, level):
        self.logger = logger
        self.level = level
    def __write__(self, message):
        self.logger(self.level, message)
    def __flush__(self):
        pass

@autorepr
class LogStderr(object):
    def __init__(self, logger, level):
        self.logger = logger
        self.level = level
    def __enter__(self):
        self.stderr = LoggerFilelike(self.logger, self.level)
        sys.stderr.flush()
        sys.stderr = self.stderr
    def __exit__(self, etype, value, trace):
        sys.stderr.flush()
        sys.stderr = sys.__stderr__
<|endoftext|>


<commit_before>import os
import random
import subprocess

SOUNDS_DIR = '/System/Library/Sounds'
SOUNDS = os.listdir(SOUNDS_DIR)

if random.randrange(0, 100) < 4:
    to_play = os.path.join(SOUNDS_DIR, random.choice(SOUNDS))
    subprocess.call(['afplay', '-v', str(random.choice(range(10, 51))), to_play])

<commit_msg>Add shebang to random-system-sound.py<commit_after>#!/usr/bin/env python

import os
import random
import subprocess

SOUNDS_DIR = '/System/Library/Sounds'
SOUNDS = os.listdir(SOUNDS_DIR)

if random.randrange(0, 100) < 4:
    to_play = os.path.join(SOUNDS_DIR, random.choice(SOUNDS))
    subprocess.call(['afplay', '-v', str(random.choice(range(10, 51))), to_play])

<|endoftext|>