<commit_before>"""
   Decorator tools.

.. module:: decorator
    :platform: Unix, Windows
    :synopis: decorator tools.

.. moduleauthor:: Thomas Lehmann <thomas.lehmann.private@googlemail.com>

   =======
   License
   =======
   Copyright (c) 2015 Thomas Lehmann

   Permission is hereby granted, free of charge, to any person obtaining a copy of this
   software and associated documentation files (the "Software"), to deal in the Software
   without restriction, including without limitation the rights to use, copy, modify, merge,
   publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons
   to whom the Software is furnished to do so, subject to the following conditions:

   The above copyright notice and this permission notice shall be included in all copies
   or substantial portions of the Software.

   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
   INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
   IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
   DAMAGES OR OTHER LIABILITY,
   WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
"""
import sys
import json
import inspect

if sys.version.startswith("2."):
    from types import NoneType
else:
    NoneType = type(None)

class data(object):
    """
    decorator to provide data to a functor or method.

    The default is to get a parameter with name 'values' of type 'list'
    in your final function or method. For consistency reason this parameter
    will ALWAYS be a list.

    You can change the name using the parameter 'key'.
    Instead of using decorator parameter 'values' you can use 'file' to load
    the content of a file. If the filename ends with .json then it is
    loaded as json file.

    With parameter single you can run test decorated method as many entries
    are contained in the list.

    >>> @data(values=[1, 2, 3, 4])
    ... def test1(values): print(values)
    >>> test1()
    [1, 2, 3, 4]
    >>> @data(key='some_data', values=[4, 3, 2, 1])
    ... def test2(some_data): print(some_data)
    >>> test2()
    [4, 3, 2, 1]
    >>> @data(key='some_data', values=[2, 1], single=True)
    ... def test3(some_data): print(some_data)
    >>> test3()
    [2]
    [1]
    """

    def __init__(self, **kwargs):
        """
        Check the decorator parameters and keep the data for the call of the decorator.

        - valid parameters are 'key', 'file' and 'values'
        - you either can use 'file' OR 'values'
        - A file will be simple read as it is or as JSON if the name ends with '.json'
        """
        for key in kwargs:
            assert key in ['key', 'file', 'values', 'single']

        self.key = str(kwargs['key']) if 'key' in kwargs else "values"
        assert isinstance(self.key, str)
        self.values = kwargs['values'] if 'values' in kwargs else []
        assert isinstance(self.values, list)
        self.single = bool(kwargs['single']) if 'single' in kwargs else False

        if 'file' in kwargs:
            assert len(self.values) == 0
            if kwargs['file'].endswith(".json"):
                self.values.append(json.loads(open(kwargs['file'], 'r').read()))
            else:
                self.values.append(open(kwargs['file'], 'r').read())

    def __call__(self, function):
        """
        Providing the decorator for the function/method.

        When calling the decorator the defined values will
        be passed to the original function.

        :returns: decorator.
        """
        def decorator(*args):
            """ decorator function. """
            if self.single:
                for value in self.values:
                    arguments = {self.key: [value]}
                    function(*args, **arguments)
            else:
                arguments = {self.key: self.values}
                return function(*args, **arguments)

        decorator.__name__ = function.__name__
        decorator.__doc__ = function.__doc__
        return decorator

class validate_test_responsibility_for(object):

    """
    Decorator to ensure to test all methods for a given class.

    a class decorator that throws an exception when the test class does not
    implement all tests for all methods of the testable class (unit).
    The next code gives you an example on how it is used and what does happen:

    >>> class Value:
    ...     def __init__(self, value):
    ...         self.value = value
    ...
    >>> try:
    ...     import unittest
    ...     @validate_test_responsibility_for(Value)
    ...     class TestValue(unittest.TestCase):
    ...         pass
    ... except Exception as e:
    ...     print("|%s|" % str(e).strip())
    |...failed to provide test method 'TestValue.test_init' for method 'Value.__init__'|
    """

    def __init__(self, testable_class, include_class_name=False):
        """ store the class for test and checks all methods of that class. """
        if hasattr(testable_class, "decorated_object"):
            testable_class = testable_class.decorated_object

        self.testable_class = testable_class
        self.include_class_name = include_class_name
        self.methods_in_testable_class\
            = self.get_entries(self.testable_class, inspect.isfunction)\
            + self.get_entries(self.testable_class, inspect.ismethod)

    @staticmethod
    def get_entries(the_class, mode):
        """
        Query functions and method of a given class.

        get all entries by given mode (function or method) but the
        members of the concrete class only; not from its base
        """
        classes = {}
        for concrete_class in reversed(inspect.getmro(the_class)):
            classes[concrete_class] = {}
            for name, definition in dict(inspect.getmembers(concrete_class, mode)).items():
                if name.find("subclass") >= 0:
                    continue

                object_name = name
                is_base_method_only = False

                for known_class in classes:
                    if object_name not in classes[known_class]:
                        continue

                    if classes[known_class][object_name] == definition:
                        is_base_method_only = True
                        break

                if not is_base_method_only:
                    classes[concrete_class][object_name] = definition

        return list(classes[the_class].keys())

    def __call__(self, test_class):
        """ called when instantiated; then we have to verify for the required test methods. """
        self.verify(test_class)
        return test_class

    @staticmethod
    def get_test_method(name, prefix=""):
        """ adjusting final test method name. """
        # no underscores wanted (change "__init__" => "init")
        final_name = name.strip("_").lower()

        # ensure more readable name (like "equal" instead of "eq")
        if name == "__eq__":
            final_name = "equal"
        elif name == "__lt__":
            final_name = "less"
        elif name == "__gt__":
            final_name = "greater"

        return "test_" + prefix.lower() + final_name

    def verify(self, test_class):
        """ verification that for each testable method a test method does exist. """
        methods_in_test_class\
            = self.get_entries(test_class, inspect.isfunction)\
            + self.get_entries(test_class, inspect.ismethod)

        missing = []
        for testable_method in self.methods_in_testable_class:
            prefix = ""
            if self.include_class_name:
                prefix = self.testable_class.__name__ + "_"

            test_method = self.get_test_method(testable_method, prefix)
            if test_method in methods_in_test_class:
                continue

            missing.append((test_class.__name__ + "." + test_method,
                            self.testable_class.__name__ + "." + testable_method))

        if len(missing) > 0:
            # creates message with all missing methods throwing an exception for it
            message = ""
            for test_method, testable_method in missing:
                message += "\n...failed to provide test method '%s' for method '%s'" \
                           % (test_method, testable_method)
            raise Exception(message)

class validate_types(object):
    """
    A function or method decorator that validates types of parameters.

    >>> try:
    ...     @validate_types(ptypes=[int, int], r=int)
    ...     def sum_of_two_integers(a, b):
    ...         return a + b
    ...     sum_of_two_integers(1, "world")
    ... except TypeError as e:
    ...     print("|%s|" % str(e))
    |1. parameter - expected type is <class 'int'>, given type is <class 'str'>|
    """

    def __init__(self, ptypes=[], r=NoneType, offset=0):
        """
        :param: ptypes: list of parameter types like "int", "float" or "str"
        :param: e: return type
        :param: offset: number of parameters to ignore (from the beginning
        """
        self.parameter_types = ptypes
        self.return_type = r
        self.offset = offset

    def __call__(self, function):
        """
        :param: function: that's the real function which should be decorated
        :rtype: decorator function (wrapper)
        """
        def wrapper(*args, **kwargs):
            """
            The wrapper checks the types of the parameter of the real function
            and raises an exception if the type doesn't fit. If fine the
            real function is executed and the return value is given back (as usual).

            :param: args: any argument
            :param: kwargs: any keyword argument
            :rtype: the result of the wrapped function
            """
            self.check_parameter_types(*args)
            result = function(*args, **kwargs)
            self.check_return_type(result)
            return result
        return wrapper

    def check_parameter_types(self, *args):
        """
        :param: args: any parameter (basically of the wrapped function)
        """
        parameter = self.offset
        for given_type, expected_type in \
                zip([type(entry) for entry in args[self.offset:]], self.parameter_types):
            if not given_type == expected_type:
                raise TypeError("%d. parameter - expected type is %s, given type is %s"
                                % (parameter, expected_type, given_type))
            parameter += 1

    def check_return_type(self, value):
        """
        :param: value: return value of wrapped function
        """
        if not type(value) == self.return_type:
            raise TypeError("expected return type is %s, given return type is %s"
                            % (self.return_type, type(value)))

def singleton(the_class):
    """ decorator for a class to make a singleton out of it. """
    class_instances = {}

    def get_instance(*args, **kwargs):
        """
        creating or just return the one and only class instance.

        The singleton depends on the parameters used in __init__
        """
        key = (the_class, args, str(kwargs))
        if key not in class_instances:
            class_instances[key] = the_class(*args, **kwargs)
        return class_instances[key]

    return get_instance
<commit_msg>handling of type/class in different Python versions<commit_after>"""
   Decorator tools.

.. module:: decorator
    :platform: Unix, Windows
    :synopis: decorator tools.

.. moduleauthor:: Thomas Lehmann <thomas.lehmann.private@googlemail.com>

   =======
   License
   =======
   Copyright (c) 2015 Thomas Lehmann

   Permission is hereby granted, free of charge, to any person obtaining a copy of this
   software and associated documentation files (the "Software"), to deal in the Software
   without restriction, including without limitation the rights to use, copy, modify, merge,
   publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons
   to whom the Software is furnished to do so, subject to the following conditions:

   The above copyright notice and this permission notice shall be included in all copies
   or substantial portions of the Software.

   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
   INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
   IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
   DAMAGES OR OTHER LIABILITY,
   WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
"""
import sys
import json
import inspect

if sys.version.startswith("2."):
    from types import NoneType
else:
    NoneType = type(None)

class data(object):
    """
    decorator to provide data to a functor or method.

    The default is to get a parameter with name 'values' of type 'list'
    in your final function or method. For consistency reason this parameter
    will ALWAYS be a list.

    You can change the name using the parameter 'key'.
    Instead of using decorator parameter 'values' you can use 'file' to load
    the content of a file. If the filename ends with .json then it is
    loaded as json file.

    With parameter single you can run test decorated method as many entries
    are contained in the list.

    >>> @data(values=[1, 2, 3, 4])
    ... def test1(values): print(values)
    >>> test1()
    [1, 2, 3, 4]
    >>> @data(key='some_data', values=[4, 3, 2, 1])
    ... def test2(some_data): print(some_data)
    >>> test2()
    [4, 3, 2, 1]
    >>> @data(key='some_data', values=[2, 1], single=True)
    ... def test3(some_data): print(some_data)
    >>> test3()
    [2]
    [1]
    """

    def __init__(self, **kwargs):
        """
        Check the decorator parameters and keep the data for the call of the decorator.

        - valid parameters are 'key', 'file' and 'values'
        - you either can use 'file' OR 'values'
        - A file will be simple read as it is or as JSON if the name ends with '.json'
        """
        for key in kwargs:
            assert key in ['key', 'file', 'values', 'single']

        self.key = str(kwargs['key']) if 'key' in kwargs else "values"
        assert isinstance(self.key, str)
        self.values = kwargs['values'] if 'values' in kwargs else []
        assert isinstance(self.values, list)
        self.single = bool(kwargs['single']) if 'single' in kwargs else False

        if 'file' in kwargs:
            assert len(self.values) == 0
            if kwargs['file'].endswith(".json"):
                self.values.append(json.loads(open(kwargs['file'], 'r').read()))
            else:
                self.values.append(open(kwargs['file'], 'r').read())

    def __call__(self, function):
        """
        Providing the decorator for the function/method.

        When calling the decorator the defined values will
        be passed to the original function.

        :returns: decorator.
        """
        def decorator(*args):
            """ decorator function. """
            if self.single:
                for value in self.values:
                    arguments = {self.key: [value]}
                    function(*args, **arguments)
            else:
                arguments = {self.key: self.values}
                return function(*args, **arguments)

        decorator.__name__ = function.__name__
        decorator.__doc__ = function.__doc__
        return decorator

class validate_test_responsibility_for(object):

    """
    Decorator to ensure to test all methods for a given class.

    a class decorator that throws an exception when the test class does not
    implement all tests for all methods of the testable class (unit).
    The next code gives you an example on how it is used and what does happen:

    >>> class Value:
    ...     def __init__(self, value):
    ...         self.value = value
    ...
    >>> try:
    ...     import unittest
    ...     @validate_test_responsibility_for(Value)
    ...     class TestValue(unittest.TestCase):
    ...         pass
    ... except Exception as e:
    ...     print("|%s|" % str(e).strip())
    |...failed to provide test method 'TestValue.test_init' for method 'Value.__init__'|
    """

    def __init__(self, testable_class, include_class_name=False):
        """ store the class for test and checks all methods of that class. """
        if hasattr(testable_class, "decorated_object"):
            testable_class = testable_class.decorated_object

        self.testable_class = testable_class
        self.include_class_name = include_class_name
        self.methods_in_testable_class\
            = self.get_entries(self.testable_class, inspect.isfunction)\
            + self.get_entries(self.testable_class, inspect.ismethod)

    @staticmethod
    def get_entries(the_class, mode):
        """
        Query functions and method of a given class.

        get all entries by given mode (function or method) but the
        members of the concrete class only; not from its base
        """
        classes = {}
        for concrete_class in reversed(inspect.getmro(the_class)):
            classes[concrete_class] = {}
            for name, definition in dict(inspect.getmembers(concrete_class, mode)).items():
                if name.find("subclass") >= 0:
                    continue

                object_name = name
                is_base_method_only = False

                for known_class in classes:
                    if object_name not in classes[known_class]:
                        continue

                    if classes[known_class][object_name] == definition:
                        is_base_method_only = True
                        break

                if not is_base_method_only:
                    classes[concrete_class][object_name] = definition

        return list(classes[the_class].keys())

    def __call__(self, test_class):
        """ called when instantiated; then we have to verify for the required test methods. """
        self.verify(test_class)
        return test_class

    @staticmethod
    def get_test_method(name, prefix=""):
        """ adjusting final test method name. """
        # no underscores wanted (change "__init__" => "init")
        final_name = name.strip("_").lower()

        # ensure more readable name (like "equal" instead of "eq")
        if name == "__eq__":
            final_name = "equal"
        elif name == "__lt__":
            final_name = "less"
        elif name == "__gt__":
            final_name = "greater"

        return "test_" + prefix.lower() + final_name

    def verify(self, test_class):
        """ verification that for each testable method a test method does exist. """
        methods_in_test_class\
            = self.get_entries(test_class, inspect.isfunction)\
            + self.get_entries(test_class, inspect.ismethod)

        missing = []
        for testable_method in self.methods_in_testable_class:
            prefix = ""
            if self.include_class_name:
                prefix = self.testable_class.__name__ + "_"

            test_method = self.get_test_method(testable_method, prefix)
            if test_method in methods_in_test_class:
                continue

            missing.append((test_class.__name__ + "." + test_method,
                            self.testable_class.__name__ + "." + testable_method))

        if len(missing) > 0:
            # creates message with all missing methods throwing an exception for it
            message = ""
            for test_method, testable_method in missing:
                message += "\n...failed to provide test method '%s' for method '%s'" \
                           % (test_method, testable_method)
            raise Exception(message)

class validate_types(object):
    """
    A function or method decorator that validates types of parameters.

    >>> try:
    ...     @validate_types(ptypes=[int, int], r=int)
    ...     def sum_of_two_integers(a, b):
    ...         return a + b
    ...     sum_of_two_integers(1, "world")
    ... except TypeError as e:
    ...     print("|%s|" % str(e).replace('<class', '<type'))
    |1. parameter - expected type is <class 'int'>, given type is <class 'str'>|
    """

    def __init__(self, ptypes=[], r=NoneType, offset=0):
        """
        :param: ptypes: list of parameter types like "int", "float" or "str"
        :param: e: return type
        :param: offset: number of parameters to ignore (from the beginning
        """
        self.parameter_types = ptypes
        self.return_type = r
        self.offset = offset

    def __call__(self, function):
        """
        :param: function: that's the real function which should be decorated
        :rtype: decorator function (wrapper)
        """
        def wrapper(*args, **kwargs):
            """
            The wrapper checks the types of the parameter of the real function
            and raises an exception if the type doesn't fit. If fine the
            real function is executed and the return value is given back (as usual).

            :param: args: any argument
            :param: kwargs: any keyword argument
            :rtype: the result of the wrapped function
            """
            self.check_parameter_types(*args)
            result = function(*args, **kwargs)
            self.check_return_type(result)
            return result
        return wrapper

    def check_parameter_types(self, *args):
        """
        :param: args: any parameter (basically of the wrapped function)
        """
        parameter = self.offset
        for given_type, expected_type in \
                zip([type(entry) for entry in args[self.offset:]], self.parameter_types):
            if not given_type == expected_type:
                raise TypeError("%d. parameter - expected type is %s, given type is %s"
                                % (parameter, expected_type, given_type))
            parameter += 1

    def check_return_type(self, value):
        """
        :param: value: return value of wrapped function
        """
        if not type(value) == self.return_type:
            raise TypeError("expected return type is %s, given return type is %s"
                            % (self.return_type, type(value)))

def singleton(the_class):
    """ decorator for a class to make a singleton out of it. """
    class_instances = {}

    def get_instance(*args, **kwargs):
        """
        creating or just return the one and only class instance.

        The singleton depends on the parameters used in __init__
        """
        key = (the_class, args, str(kwargs))
        if key not in class_instances:
            class_instances[key] = the_class(*args, **kwargs)
        return class_instances[key]

    return get_instance
<|endoftext|>


<commit_before>'''
Build the pipeline workflow by plumbing the stages together.
'''

from ruffus import Pipeline, suffix, formatter, add_inputs, output_from
from stages import Stages

def make_pipeline(state):
    '''Build the pipeline by constructing stages and connecting them together'''
    # Build an empty pipeline
    pipeline = Pipeline(name='hiplexpipe')
    # Get a list of paths to all the FASTQ files
    fastq_files = state.config.get_option('fastqs')
    # Stages are dependent on the state
    stages = Stages(state)

    # The original FASTQ files
    # This is a dummy stage. It is useful because it makes a node in the
    # pipeline graph, and gives the pipeline an obvious starting point.
    pipeline.originate(
        task_func=stages.original_fastqs,
        name='original_fastqs',
        output=fastq_files)

    # Align paired end reads in FASTQ to the reference producing a BAM file
    pipeline.transform(
        task_func=stages.align_bwa,
        name='align_bwa',
        input=output_from('original_fastqs'),
        # Match the R1 (read 1) FASTQ file and grab the path and sample name.
        # This will be the first input to the stage.
        # We assume the sample name may consist of only alphanumeric
        # characters.
        # Hi-Plex example: OHI031002-P02F04_S318_L001_R1_001.fastq
        # new sample name = OHI031002-P02F04
        # 100ng-GSP4-cy0-251_S251_L001_R2_001.fastq.gz
        # HardSeed-4C-096_S96_L001_R1_001.fastq.gz
        filter=formatter(
            '.+/(?P<sample>[a-zA-Z0-9-]+)_(?P<readid>[a-zA-Z0-9-]+)_(?P<lane>[a-zA-Z0-9]+)_R1_(?P<lib>[a-zA-Z0-9-:]+).fastq'),
        # Add one more inputs to the stage:
        #    1. The corresponding R2 FASTQ file
        # Hi-Plex example: OHI031002-P02F04_S318_L001_R2_001.fastq
        add_inputs=add_inputs(
            '{path[0]}/{sample[0]}_{readid[0]}_{lane[0]}_R2_{lib[0]}.fastq'),
        # Add an "extra" argument to the state (beyond the inputs and outputs)
        # which is the sample name. This is needed within the stage for finding out
        # sample specific configuration options
        extras=['{sample[0]}', '{readid[0]}', '{lane[0]}', '{lib[0]}'],
        # The output file name is the sample name with a .bam extension.
        output='alignments/{sample[0]}_{readid[0]}/{sample[0]}_{readid[0]}.bam')

    # Call variants using undr_rover
    pipeline.transform(
        task_func=stages.apply_undr_rover,
        name='apply_undr_rover',
        input=output_from('original_fastqs'),
        # Match the R1 (read 1) FASTQ file and grab the path and sample name.
        # This will be the first input to the stage.
        # We assume the sample name may consist of only alphanumeric
        # characters.
        # Hi-Plex example: OHI031002-P02F04_S318_L001_R1_001.fastq
        filter=formatter(
            '.+/(?P<sample>[a-zA-Z0-9-]+)_(?P<readid>[a-zA-Z0-9-]+)_(?P<lane>[a-zA-Z0-9]+)_R1_(?P<lib>[a-zA-Z0-9-:]+).fastq'),
        # Add one more inputs to the stage:
        #    1. The corresponding R2 FASTQ file
        # Hi-Plex example: OHI031002-P02F04_S318_L001_R2_001.fastq

        add_inputs=add_inputs(
            '{path[0]}/{sample[0]}_{readid[0]}_{lane[0]}_R2_{lib[0]}.fastq'),
        # extras=['{sample[0]}', '{readid[0]}', '{lane[0]}', '{lib[0]}'],
        extras=['{sample[0]}', '{readid[0]}'],
        # The output file name is the sample name with a .bam extension.
        output='variants/undr_rover/{sample[0]}_{readid[0]}.vcf')

    # Sort the BAM file using Picard
    pipeline.transform(
        task_func=stages.sort_bam_picard,
        name='sort_bam_picard',
        input=output_from('align_bwa'),
        filter=suffix('.bam'),
        output='.sort.bam')

    # High quality and primary alignments
    pipeline.transform(
        task_func=stages.primary_bam,
        name='primary_bam',
        input=output_from('sort_bam_picard'),
        filter=suffix('.sort.bam'),
        output='.primary.bam')

    # index bam file
    pipeline.transform(
        task_func=stages.index_sort_bam_picard,
        name='index_bam',
        input=output_from('primary_bam'),
        filter=suffix('.primary.bam'),
        output='.primary.bam.bai')

    # Clip the primer_seq from BAM File
    (pipeline.transform(
        task_func=stages.clip_bam,
        name='clip_bam',
        input=output_from('primary_bam'),
        filter=suffix('.primary.bam'),
        output='.primerclipped.bam')
        .follows('index_bam'))

    # # Sort the BAM file using Picard
    # pipeline.transform(
    #     task_func=stages.sort_bam_picard,
    #     name='sort_bam_picard',
    #     input=output_from('clip_bam'),
    #     filter=suffix('.clip.bam'),
    #     output='.sort.bam')

    # # samtools index sorted bam file
    # pipeline.transform(
    #     task_func=stages.index_sort_bam_picard,
    #     name='index_sort_bam_picard',
    #     input=output_from('clip_bam'),
    #     filter=suffix('.primerclipped.bam'),
    #     output='.primerclipped.bam.bai')

    # # Coverage using Picard
    # (pipeline.transform(
    #     task_func=stages.target_coverage,
    #     name='target_coverage',
    #     input=output_from('sort_bam_picard'),
    #     # filter=suffix('.sort.bam'),
    #     filter=formatter(
    #         '.+/(?P<sample>[a-zA-Z0-9-_]+).sort.bam'),
    #     output='coverage/{sample[0]}.coverage.txt')
    #     .follows('sort_bam_picard'))

    ###### GATK VARIANT CALLING ######
    # Call variants using GATK
    pipeline.transform(
        task_func=stages.call_haplotypecaller_gatk,
        name='call_haplotypecaller_gatk',
        input=output_from('clip_bam'),
        # filter=suffix('.merged.dedup.realn.bam'),
        filter=formatter('.+/(?P<sample>[a-zA-Z0-9-_]+).sort.primerclipped.bam'),
        output='variants/gatk/{sample[0]}.g.vcf')
        # .follows('index_sort_bam_picard'))

    # Combine G.VCF files for all samples using GATK
    pipeline.merge(
        task_func=stages.combine_gvcf_gatk,
        name='combine_gvcf_gatk',
        input=output_from('call_haplotypecaller_gatk'),
        output='variants/gatk/ALL.combined.vcf')

    # Genotype G.VCF files using GATK
    pipeline.transform(
        task_func=stages.genotype_gvcf_gatk,
        name='genotype_gvcf_gatk',
        input=output_from('combine_gvcf_gatk'),
        filter=suffix('.combined.vcf'),
        output='.raw.vcf')

    # Annotate VCF file using GATK
    pipeline.transform(
       task_func=stages.variant_annotator_gatk,
       name='variant_annotator_gatk',
       input=output_from('genotype_gvcf_gatk'),
       filter=suffix('.raw.vcf'),
       output='.raw.annotate.vcf')

    # ------- RECAL
    # SNP recalibration using GATK
    pipeline.transform(
        task_func=stages.snp_recalibrate_gatk,
        name='snp_recalibrate_gatk',
        input=output_from('variant_annotator_gatk'),
        filter=suffix('.raw.annotate.vcf'),
        output=['.snp_recal', '.snp_tranches', '.snp_plots.R'])

    # Apply SNP recalibration using GATK
    (pipeline.transform(
        task_func=stages.apply_snp_recalibrate_gatk,
        name='apply_snp_recalibrate_gatk',
        input=output_from('variant_annotator_gatk'),
        filter=suffix('.raw.annotate.vcf'),
        add_inputs=add_inputs(['variants/ALL.snp_recal', 'variants/ALL.snp_tranches']),
        output='.recal_SNP.vcf')
        .follows('snp_recalibrate_gatk'))

    # INDEL recalibration using GATK
    pipeline.transform(
        task_func=stages.indel_recalibrate_gatk,
        name='indel_recalibrate_gatk',
        input=output_from('apply_snp_recalibrate_gatk'),
        filter=suffix('.recal_SNP.vcf'),
        output=['.indel_recal', '.indel_tranches', '.indel_plots.R'])

    # Apply INDEL recalibration using GATK
    (pipeline.transform(
        task_func=stages.apply_indel_recalibrate_gatk,
        name='apply_indel_recalibrate_gatk',
        input=output_from('apply_snp_recalibrate_gatk'),
        filter=suffix('.recal_SNP.vcf'),
        add_inputs=add_inputs(
            ['variants/ALL.indel_recal', 'variants/ALL.indel_tranches']),
        output='.raw.annotate.vqsr.vcf')
        .follows('indel_recalibrate_gatk'))

    # Apply VariantFiltration using GATK
    (pipeline.transform(
        task_func=stages.apply_variant_filtration_gatk_lenient,
        name='apply_variant_filtration_gatk_lenient',
        input=output_from('apply_indel_recalibrate_gatk'),
        filter=suffix('.raw.annotate.vqsr.vcf'),
        output='.raw.annotate.vqsr.filtered_lenient.vcf')
        .follows('apply_indel_recalibrate_gatk'))

    # ------- RECAL

    # -------- VEP ----------
    # Apply NORM
    (pipeline.transform(
        task_func=stages.apply_vt,
        name='apply_vt',
        input=output_from('apply_variant_filtration_gatk_lenient'),
        filter=suffix('.raw.annotate.vqsr.filtered_lenient.vcf'),
        # add_inputs=add_inputs(['variants/ALL.indel_recal', 'variants/ALL.indel_tranches']),
        output='.raw.annotate.vqsr.filtered_lenient.norm.vcf')
        .follows('apply_variant_filtration_gatk_lenient'))

    # Apply VEP
    (pipeline.transform(
        task_func=stages.apply_vep,
        name='apply_vep',
        input=output_from('apply_vt'),
        filter=suffix('.raw.annotate.vqsr.filtered_lenient.norm.vcf'),
        # add_inputs=add_inputs(['variants/ALL.indel_recal', 'variants/ALL.indel_tranches']),
        output='.raw.annotate.vqsr.filtered_lenient.norm.vep.vcf')
        .follows('apply_vt'))

    # Apply SnpEff
    (pipeline.transform(
        task_func=stages.apply_snpeff,
        name='apply_snpeff',
        input=output_from('apply_vep'),
        filter=suffix('.raw.annotate.vqsr.filtered_lenient.norm.vep.vcf'),
        # add_inputs=add_inputs(['variants/ALL.indel_recal', 'variants/ALL.indel_tranches']),
        output='.raw.annotate.vqsr.filtered_lenient.norm.vep.snpeff.vcf')
        .follows('apply_vep'))

    # Apply vcfanno
    (pipeline.transform(
        task_func=stages.apply_vcfanno,
        name='apply_vcfanno',
        input=output_from('apply_snpeff'),
        filter=suffix('.raw.annotate.vqsr.filtered_lenient.norm.vep.snpeff.vcf'),
        # add_inputs=add_inputs(['variants/ALL.indel_recal', 'variants/ALL.indel_tranches']),
        output='.annotated.vcf')
        .follows('apply_snpeff'))

    # -------- VEP ----------
    ###### GATK VARIANT CALLING ######

    # Concatenate undr_rover vcf files
    pipeline.merge(
        task_func=stages.apply_cat_vcf,
        name='apply_cat_vcf',
        input=output_from('apply_undr_rover'),
        output='variants/undr_rover/ur.vcf.gz')

    # Apple VEP on concatenated undr_rover vcf file
    (pipeline.transform(
        task_func=stages.apply_vep,
        name='apply_vep_ur',
        input=output_from('apply_cat_vcf'),
        filter=suffix('.vcf.gz'),
        output='.vep.vcf')
        .follows('apply_cat_vcf'))

    # Apply vcfanno on concatenated/vep undr_rover vcf file
    (pipeline.transform(
        task_func=stages.apply_vcfanno,
        name='apply_vcfanno_ur',
        input=output_from('apply_vep_ur'),
        filter=suffix('.vep.vcf'),
        output='.vep.anno.vcf')
        .follows('apply_vep_ur'))

    # Apply snpeff
    (pipeline.transform(
        task_func=stages.apply_snpeff,
        name='apply_snpeff_ur',
        input=output_from('apply_vcfanno_ur'),
        filter=suffix('.vep.anno.vcf'),
        output='.vep.anno.snpeff.vcf')
        .follows('apply_vcfanno_ur'))

    # Apply HomopolymerRun
    pipeline.transform(
        task_func=stages.apply_homopolymer_ann,
        name='apply_homopolymer_ann',
        input=output_from('apply_snpeff_ur'),
        filter=suffix('.vep.anno.snpeff.vcf'),
        output='.annotated.vcf')

    # Apply summarize multi coverage
    (pipeline.merge(
        task_func=stages.apply_multicov,
        name='apply_multicov',
        input=output_from('primary_bam'),
        output='coverage/all.multicov.txt')
        .follows('index_sort_bam_picard'))

    # Apply summarize picard coverage
    # (pipeline.merge(
    #     task_func=stages.apply_summarize_picard,
    #     name='apply_summarize_picard',
    #     input=output_from('target_coverage'),
    #     output='coverage/all.hsmetrics.txt')
    #     .follows('target_coverage'))

    # Apply summarize multicov coverage plots
    (pipeline.merge(
        task_func=stages.apply_multicov_plots,
        name='apply_multicov_plots',
        input=output_from('apply_multicov'),
        output='coverage/coverage_analysis_main.html')
        .follows('apply_multicov'))

    return pipeline
<commit_msg>bug fixes<commit_after>'''
Build the pipeline workflow by plumbing the stages together.
'''

from ruffus import Pipeline, suffix, formatter, add_inputs, output_from
from stages import Stages

def make_pipeline(state):
    '''Build the pipeline by constructing stages and connecting them together'''
    # Build an empty pipeline
    pipeline = Pipeline(name='hiplexpipe')
    # Get a list of paths to all the FASTQ files
    fastq_files = state.config.get_option('fastqs')
    # Stages are dependent on the state
    stages = Stages(state)

    # The original FASTQ files
    # This is a dummy stage. It is useful because it makes a node in the
    # pipeline graph, and gives the pipeline an obvious starting point.
    pipeline.originate(
        task_func=stages.original_fastqs,
        name='original_fastqs',
        output=fastq_files)

    # Align paired end reads in FASTQ to the reference producing a BAM file
    pipeline.transform(
        task_func=stages.align_bwa,
        name='align_bwa',
        input=output_from('original_fastqs'),
        # Match the R1 (read 1) FASTQ file and grab the path and sample name.
        # This will be the first input to the stage.
        # We assume the sample name may consist of only alphanumeric
        # characters.
        # Hi-Plex example: OHI031002-P02F04_S318_L001_R1_001.fastq
        # new sample name = OHI031002-P02F04
        # 100ng-GSP4-cy0-251_S251_L001_R2_001.fastq.gz
        # HardSeed-4C-096_S96_L001_R1_001.fastq.gz
        filter=formatter(
            '.+/(?P<sample>[a-zA-Z0-9-]+)_(?P<readid>[a-zA-Z0-9-]+)_(?P<lane>[a-zA-Z0-9]+)_R1_(?P<lib>[a-zA-Z0-9-:]+).fastq'),
        # Add one more inputs to the stage:
        #    1. The corresponding R2 FASTQ file
        # Hi-Plex example: OHI031002-P02F04_S318_L001_R2_001.fastq
        add_inputs=add_inputs(
            '{path[0]}/{sample[0]}_{readid[0]}_{lane[0]}_R2_{lib[0]}.fastq'),
        # Add an "extra" argument to the state (beyond the inputs and outputs)
        # which is the sample name. This is needed within the stage for finding out
        # sample specific configuration options
        extras=['{sample[0]}', '{readid[0]}', '{lane[0]}', '{lib[0]}'],
        # The output file name is the sample name with a .bam extension.
        output='alignments/{sample[0]}_{readid[0]}/{sample[0]}_{readid[0]}.bam')

    # Call variants using undr_rover
    pipeline.transform(
        task_func=stages.apply_undr_rover,
        name='apply_undr_rover',
        input=output_from('original_fastqs'),
        # Match the R1 (read 1) FASTQ file and grab the path and sample name.
        # This will be the first input to the stage.
        # We assume the sample name may consist of only alphanumeric
        # characters.
        # Hi-Plex example: OHI031002-P02F04_S318_L001_R1_001.fastq
        filter=formatter(
            '.+/(?P<sample>[a-zA-Z0-9-]+)_(?P<readid>[a-zA-Z0-9-]+)_(?P<lane>[a-zA-Z0-9]+)_R1_(?P<lib>[a-zA-Z0-9-:]+).fastq'),
        # Add one more inputs to the stage:
        #    1. The corresponding R2 FASTQ file
        # Hi-Plex example: OHI031002-P02F04_S318_L001_R2_001.fastq

        add_inputs=add_inputs(
            '{path[0]}/{sample[0]}_{readid[0]}_{lane[0]}_R2_{lib[0]}.fastq'),
        # extras=['{sample[0]}', '{readid[0]}', '{lane[0]}', '{lib[0]}'],
        extras=['{sample[0]}', '{readid[0]}'],
        # The output file name is the sample name with a .bam extension.
        output='variants/undr_rover/{sample[0]}_{readid[0]}.vcf')

    # Sort the BAM file using Picard
    pipeline.transform(
        task_func=stages.sort_bam_picard,
        name='sort_bam_picard',
        input=output_from('align_bwa'),
        filter=suffix('.bam'),
        output='.sort.bam')

    # High quality and primary alignments
    pipeline.transform(
        task_func=stages.primary_bam,
        name='primary_bam',
        input=output_from('sort_bam_picard'),
        filter=suffix('.sort.bam'),
        output='.primary.bam')

    # index bam file
    pipeline.transform(
        task_func=stages.index_sort_bam_picard,
        name='index_bam',
        input=output_from('primary_bam'),
        filter=suffix('.primary.bam'),
        output='.primary.bam.bai')

    # Clip the primer_seq from BAM File
    (pipeline.transform(
        task_func=stages.clip_bam,
        name='clip_bam',
        input=output_from('primary_bam'),
        filter=suffix('.primary.bam'),
        output='.primerclipped.bam')
        .follows('index_bam'))

    # # Sort the BAM file using Picard
    # pipeline.transform(
    #     task_func=stages.sort_bam_picard,
    #     name='sort_bam_picard',
    #     input=output_from('clip_bam'),
    #     filter=suffix('.clip.bam'),
    #     output='.sort.bam')

    # # samtools index sorted bam file
    # pipeline.transform(
    #     task_func=stages.index_sort_bam_picard,
    #     name='index_sort_bam_picard',
    #     input=output_from('clip_bam'),
    #     filter=suffix('.primerclipped.bam'),
    #     output='.primerclipped.bam.bai')

    # # Coverage using Picard
    # (pipeline.transform(
    #     task_func=stages.target_coverage,
    #     name='target_coverage',
    #     input=output_from('sort_bam_picard'),
    #     # filter=suffix('.sort.bam'),
    #     filter=formatter(
    #         '.+/(?P<sample>[a-zA-Z0-9-_]+).sort.bam'),
    #     output='coverage/{sample[0]}.coverage.txt')
    #     .follows('sort_bam_picard'))

    ###### GATK VARIANT CALLING ######
    # Call variants using GATK
    pipeline.transform(
        task_func=stages.call_haplotypecaller_gatk,
        name='call_haplotypecaller_gatk',
        input=output_from('clip_bam'),
        # filter=suffix('.merged.dedup.realn.bam'),
        filter=formatter('.+/(?P<sample>[a-zA-Z0-9-_]+).primary.primerclipped.bam'),
        output='variants/gatk/{sample[0]}.g.vcf')
        # .follows('index_sort_bam_picard'))

    # Combine G.VCF files for all samples using GATK
    pipeline.merge(
        task_func=stages.combine_gvcf_gatk,
        name='combine_gvcf_gatk',
        input=output_from('call_haplotypecaller_gatk'),
        output='variants/gatk/ALL.combined.vcf')

    # Genotype G.VCF files using GATK
    pipeline.transform(
        task_func=stages.genotype_gvcf_gatk,
        name='genotype_gvcf_gatk',
        input=output_from('combine_gvcf_gatk'),
        filter=suffix('.combined.vcf'),
        output='.raw.vcf')

    # Annotate VCF file using GATK
    pipeline.transform(
       task_func=stages.variant_annotator_gatk,
       name='variant_annotator_gatk',
       input=output_from('genotype_gvcf_gatk'),
       filter=suffix('.raw.vcf'),
       output='.raw.annotate.vcf')

    # ------- RECAL
    # SNP recalibration using GATK
    pipeline.transform(
        task_func=stages.snp_recalibrate_gatk,
        name='snp_recalibrate_gatk',
        input=output_from('variant_annotator_gatk'),
        filter=suffix('.raw.annotate.vcf'),
        output=['.snp_recal', '.snp_tranches', '.snp_plots.R'])

    # Apply SNP recalibration using GATK
    (pipeline.transform(
        task_func=stages.apply_snp_recalibrate_gatk,
        name='apply_snp_recalibrate_gatk',
        input=output_from('variant_annotator_gatk'),
        filter=suffix('.raw.annotate.vcf'),
        add_inputs=add_inputs(['variants/ALL.snp_recal', 'variants/ALL.snp_tranches']),
        output='.recal_SNP.vcf')
        .follows('snp_recalibrate_gatk'))

    # INDEL recalibration using GATK
    pipeline.transform(
        task_func=stages.indel_recalibrate_gatk,
        name='indel_recalibrate_gatk',
        input=output_from('apply_snp_recalibrate_gatk'),
        filter=suffix('.recal_SNP.vcf'),
        output=['.indel_recal', '.indel_tranches', '.indel_plots.R'])

    # Apply INDEL recalibration using GATK
    (pipeline.transform(
        task_func=stages.apply_indel_recalibrate_gatk,
        name='apply_indel_recalibrate_gatk',
        input=output_from('apply_snp_recalibrate_gatk'),
        filter=suffix('.recal_SNP.vcf'),
        add_inputs=add_inputs(
            ['variants/ALL.indel_recal', 'variants/ALL.indel_tranches']),
        output='.raw.annotate.vqsr.vcf')
        .follows('indel_recalibrate_gatk'))

    # Apply VariantFiltration using GATK
    (pipeline.transform(
        task_func=stages.apply_variant_filtration_gatk_lenient,
        name='apply_variant_filtration_gatk_lenient',
        input=output_from('apply_indel_recalibrate_gatk'),
        filter=suffix('.raw.annotate.vqsr.vcf'),
        output='.raw.annotate.vqsr.filtered_lenient.vcf')
        .follows('apply_indel_recalibrate_gatk'))

    # ------- RECAL

    # -------- VEP ----------
    # Apply NORM
    (pipeline.transform(
        task_func=stages.apply_vt,
        name='apply_vt',
        input=output_from('apply_variant_filtration_gatk_lenient'),
        filter=suffix('.raw.annotate.vqsr.filtered_lenient.vcf'),
        # add_inputs=add_inputs(['variants/ALL.indel_recal', 'variants/ALL.indel_tranches']),
        output='.raw.annotate.vqsr.filtered_lenient.norm.vcf')
        .follows('apply_variant_filtration_gatk_lenient'))

    # Apply VEP
    (pipeline.transform(
        task_func=stages.apply_vep,
        name='apply_vep',
        input=output_from('apply_vt'),
        filter=suffix('.raw.annotate.vqsr.filtered_lenient.norm.vcf'),
        # add_inputs=add_inputs(['variants/ALL.indel_recal', 'variants/ALL.indel_tranches']),
        output='.raw.annotate.vqsr.filtered_lenient.norm.vep.vcf')
        .follows('apply_vt'))

    # Apply SnpEff
    (pipeline.transform(
        task_func=stages.apply_snpeff,
        name='apply_snpeff',
        input=output_from('apply_vep'),
        filter=suffix('.raw.annotate.vqsr.filtered_lenient.norm.vep.vcf'),
        # add_inputs=add_inputs(['variants/ALL.indel_recal', 'variants/ALL.indel_tranches']),
        output='.raw.annotate.vqsr.filtered_lenient.norm.vep.snpeff.vcf')
        .follows('apply_vep'))

    # Apply vcfanno
    (pipeline.transform(
        task_func=stages.apply_vcfanno,
        name='apply_vcfanno',
        input=output_from('apply_snpeff'),
        filter=suffix('.raw.annotate.vqsr.filtered_lenient.norm.vep.snpeff.vcf'),
        # add_inputs=add_inputs(['variants/ALL.indel_recal', 'variants/ALL.indel_tranches']),
        output='.annotated.vcf')
        .follows('apply_snpeff'))

    # -------- VEP ----------
    ###### GATK VARIANT CALLING ######

    # Concatenate undr_rover vcf files
    pipeline.merge(
        task_func=stages.apply_cat_vcf,
        name='apply_cat_vcf',
        input=output_from('apply_undr_rover'),
        output='variants/undr_rover/ur.vcf.gz')

    # Apple VEP on concatenated undr_rover vcf file
    (pipeline.transform(
        task_func=stages.apply_vep,
        name='apply_vep_ur',
        input=output_from('apply_cat_vcf'),
        filter=suffix('.vcf.gz'),
        output='.vep.vcf')
        .follows('apply_cat_vcf'))

    # Apply vcfanno on concatenated/vep undr_rover vcf file
    (pipeline.transform(
        task_func=stages.apply_vcfanno,
        name='apply_vcfanno_ur',
        input=output_from('apply_vep_ur'),
        filter=suffix('.vep.vcf'),
        output='.vep.anno.vcf')
        .follows('apply_vep_ur'))

    # Apply snpeff
    (pipeline.transform(
        task_func=stages.apply_snpeff,
        name='apply_snpeff_ur',
        input=output_from('apply_vcfanno_ur'),
        filter=suffix('.vep.anno.vcf'),
        output='.vep.anno.snpeff.vcf')
        .follows('apply_vcfanno_ur'))

    # Apply HomopolymerRun
    pipeline.transform(
        task_func=stages.apply_homopolymer_ann,
        name='apply_homopolymer_ann',
        input=output_from('apply_snpeff_ur'),
        filter=suffix('.vep.anno.snpeff.vcf'),
        output='.annotated.vcf')

    # Apply summarize multi coverage
    (pipeline.merge(
        task_func=stages.apply_multicov,
        name='apply_multicov',
        input=output_from('primary_bam'),
        filter=suffix('.primary.bam'),
        output='coverage/all.multicov.txt')
        .follows('index_bam'))

    # Apply summarize picard coverage
    # (pipeline.merge(
    #     task_func=stages.apply_summarize_picard,
    #     name='apply_summarize_picard',
    #     input=output_from('target_coverage'),
    #     output='coverage/all.hsmetrics.txt')
    #     .follows('target_coverage'))

    # Apply summarize multicov coverage plots
    (pipeline.merge(
        task_func=stages.apply_multicov_plots,
        name='apply_multicov_plots',
        input=output_from('apply_multicov'),
        output='coverage/coverage_analysis_main.html')
        .follows('apply_multicov'))

    return pipeline
<|endoftext|>


<commit_before>'''
Build the pipeline workflow by plumbing the stages together.
'''

from ruffus import Pipeline, suffix, formatter, add_inputs, output_from
from stages import Stages

def make_pipeline(state):
    '''Build the pipeline by constructing stages and connecting them together'''
    # Build an empty pipeline
    pipeline = Pipeline(name='hiplexpipe')
    # Get a list of paths to all the FASTQ files
    fastq_files = state.config.get_option('fastqs')
    # Stages are dependent on the state
    stages = Stages(state)

    # The original FASTQ files
    # This is a dummy stage. It is useful because it makes a node in the
    # pipeline graph, and gives the pipeline an obvious starting point.
    pipeline.originate(
        task_func=stages.original_fastqs,
        name='original_fastqs',
        output=fastq_files)

    # Align paired end reads in FASTQ to the reference producing a BAM file
    pipeline.transform(
        task_func=stages.align_bwa,
        name='align_bwa',
        input=output_from('original_fastqs'),
        # Match the R1 (read 1) FASTQ file and grab the path and sample name.
        # This will be the first input to the stage.
        # We assume the sample name may consist of only alphanumeric
        # characters.
        # Hi-Plex example: OHI031002-P02F04_S318_L001_R1_001.fastq
        # new sample name = OHI031002-P02F04
        filter=formatter(
            '.+/(?P<sample>[a-zA-Z0-9-]+)_(?P<readid>[a-zA-Z0-9-]+)_(?P<lane>[a-zA-Z0-9]+)_R1_(?P<lib>[a-zA-Z0-9-:]+).fastq'),
        # Add one more inputs to the stage:
        #    1. The corresponding R2 FASTQ file
        # Hi-Plex example: OHI031002-P02F04_S318_L001_R2_001.fastq
        add_inputs=add_inputs(
            '{path[0]}/{sample[0]}_{readid[0]}_{lane[0]}_R2_{lib[0]}.fastq'),
        # Add an "extra" argument to the state (beyond the inputs and outputs)
        # which is the sample name. This is needed within the stage for finding out
        # sample specific configuration options
        extras=['{sample[0]}', '{readid[0]}', '{lane[0]}', '{lib[0]}'],
        # The output file name is the sample name with a .bam extension.
        output='alignments/{sample[0]}_{readid[0]}/{sample[0]}_{readid[0]}.bam')

    # Call variants using undr_rover
    pipeline.transform(
        task_func=stages.apply_undr_rover,
        name='apply_undr_rover',
        input=output_from('original_fastqs'),
        # Match the R1 (read 1) FASTQ file and grab the path and sample name.
        # This will be the first input to the stage.
        # We assume the sample name may consist of only alphanumeric
        # characters.
        # Hi-Plex example: OHI031002-P02F04_S318_L001_R1_001.fastq
        filter=formatter(
            '.+/(?P<sample>[a-zA-Z0-9-]+)_(?P<readid>[a-zA-Z0-9-]+)_(?P<lane>[a-zA-Z0-9]+)_R1_(?P<lib>[a-zA-Z0-9-:]+).fastq'),
        # Add one more inputs to the stage:
        #    1. The corresponding R2 FASTQ file
        # Hi-Plex example: OHI031002-P02F04_S318_L001_R2_001.fastq

        add_inputs=add_inputs(
            '{path[0]}/{sample[0]}_{readid[0]}_{lane[0]}_R2_{lib[0]}.fastq'),
        # extras=['{sample[0]}', '{readid[0]}', '{lane[0]}', '{lib[0]}'],
        extras=['{sample[0]}', '{readid[0]}'],
        # The output file name is the sample name with a .bam extension.
        output='variants/undr_rover/{sample[0]}_{readid[0]}.vcf')

    # Sort the BAM file using Picard
    pipeline.transform(
        task_func=stages.sort_bam_picard,
        name='sort_bam_picard',
        input=output_from('align_bwa'),
        filter=suffix('.bam'),
        output='.sort.bam')

    # samtools index sorted bam file
    pipeline.transform(
        task_func=stages.index_sort_bam_picard,
        name='index_sort_bam_picard',
        input=output_from('sort_bam_picard'),
        filter=suffix('.sort.bam'),
        output='.sort.bam.bai')

    # Coverage using Picard
    (pipeline.transform(
        task_func=stages.target_coverage,
        name='target_coverage',
        input=output_from('sort_bam_picard'),
        # filter=suffix('.sort.bam'),
        filter=formatter(
            '.+/(?P<sample>[a-zA-Z0-9-_]+).sort.bam'),
        output='coverage/{sample[0]}.coverage.txt')
        .follows('sort_bam_picard'))

    # # Coverage using bam
    # (pipeline.transform(
    #     task_func=stages.target_coverage_bamutil,
    #     name='target_coverage_bamutil',
    #     input=output_from('sort_bam_picard'),
    #     # filter=suffix('.sort.bam'),
    #     filter=formatter(
    #         '.+/(?P<sample>[a-zA-Z0-9-_]+).sort.bam'),
    #     output='coverage/{sample[0]}.bamutil.txt')
    #     .follows('index_sort_bam_picard'))
    #
    # # Coverage using bam with interval
    # (pipeline.transform(
    #     task_func=stages.target_coverage_bamutil_interval,
    #     name='target_coverage_bamutil_interval',
    #     input=output_from('sort_bam_picard'),
    #     # filter=suffix('.sort.bam'),
    #     filter=formatter(
    #         '.+/(?P<sample>[a-zA-Z0-9-_]+).sort.bam'),
    #     output='coverage/{sample[0]}.bamutil2.txt')
    #     .follows('index_sort_bam_picard'))

    # Apply samtools
    pipeline.merge(
        task_func=stages.apply_samtools_mpileup,
        name='apply_samtools_mpileup',
        input=output_from('sort_bam_picard'),
        # filter=suffix('.sort.bam'),
        #filter=formatter('.+/(?P<sample>[a-zA-Z0-9-]+).sort.bam'),
        output='variants/all.mpileup')
        #filter=formatter('.+/(?P<sample>[a-zA-Z0-9-]+).sort.bam'),
        #output='variants/all.bcf')
        #.follows('sort_bam_picard'))

    # Apply bcftools
    (pipeline.transform(
        task_func=stages.apply_bcftools,
        name='apply_bcftools',
        input=output_from('apply_samtools_mpileup'),
        filter=suffix('.mpileup'),
        # add_inputs=add_inputs(['variants/ALL.indel_recal', 'variants/ALL.indel_tranches']),
        output='.raw.vcf')
        .follows('apply_samtools_mpileup'))

    # Apply NORM
    (pipeline.transform(
        task_func=stages.apply_vt,
        name='apply_vt',
        input=output_from('apply_bcftools'),
        filter=suffix('.raw.vcf'),
        # add_inputs=add_inputs(['variants/ALL.indel_recal', 'variants/ALL.indel_tranches']),
        output='.raw.vt.vcf')
        .follows('sort_bam_picard'))

    # Apply VEP
    (pipeline.transform(
        task_func=stages.apply_vep,
        name='apply_vep',
        input=output_from('apply_vt'),
        filter=suffix('.raw.vt.vcf'),
        # add_inputs=add_inputs(['variants/ALL.indel_recal', 'variants/ALL.indel_tranches']),
        output='.raw.vt.vep.vcf')
        .follows('apply_vt'))

    # Apply BCF
    # (pipeline.transform(
    #     task_func=stages.apply_bcf,
    #     name='apply_bcf',
    #     input=output_from('apply_vep'),
    #     filter=suffix('.raw.vqsr.vt.vep.vcf'),
    #     # add_inputs=add_inputs(['variants/ALL.indel_recal', 'variants/ALL.indel_tranches']),
    #     output='.raw.vt.vep.bcf.vcf')
    #     .follows('apply_vep'))

    # Apply SnpEff
    (pipeline.transform(
        task_func=stages.apply_snpeff,
        name='apply_snpeff',
        input=output_from('apply_vep'),
        filter=suffix('.raw.vt.vep.vcf'),
        # add_inputs=add_inputs(['variants/ALL.indel_recal', 'variants/ALL.indel_tranches']),
        output='.raw.vt.vep.snpeff.vcf')
        .follows('apply_vep'))

    # Apply vcfanno
    (pipeline.transform(
        task_func=stages.apply_vcfanno,
        name='apply_vcfanno',
        input=output_from('apply_snpeff'),
        filter=suffix('.raw.vt.vep.snpeff.vcf'),
        # add_inputs=add_inputs(['variants/ALL.indel_recal', 'variants/ALL.indel_tranches']),
        output='.annotated.vcf')
        .follows('apply_snpeff'))

    # Concatenate undr_rover vcf files
    pipeline.merge(
        task_func=stages.apply_cat_vcf,
        name='apply_cat_vcf',
        input=output_from('apply_undr_rover'),
        output='variants/undr_rover/ur.vcf.gz')

    # Apple VEP on concatenated undr_rover vcf file
    pipeline.merge(
        task_func=stages.apply_vep2,
        name='apply_vep2',
        input=output_from('apply_cat_vcf'),
        output='variants/undr_rover/ur.vep.vcf')

    return pipeline
<commit_msg>Add UR vep<commit_after>'''
Build the pipeline workflow by plumbing the stages together.
'''

from ruffus import Pipeline, suffix, formatter, add_inputs, output_from
from stages import Stages

def make_pipeline(state):
    '''Build the pipeline by constructing stages and connecting them together'''
    # Build an empty pipeline
    pipeline = Pipeline(name='hiplexpipe')
    # Get a list of paths to all the FASTQ files
    fastq_files = state.config.get_option('fastqs')
    # Stages are dependent on the state
    stages = Stages(state)

    # The original FASTQ files
    # This is a dummy stage. It is useful because it makes a node in the
    # pipeline graph, and gives the pipeline an obvious starting point.
    pipeline.originate(
        task_func=stages.original_fastqs,
        name='original_fastqs',
        output=fastq_files)

    # Align paired end reads in FASTQ to the reference producing a BAM file
    pipeline.transform(
        task_func=stages.align_bwa,
        name='align_bwa',
        input=output_from('original_fastqs'),
        # Match the R1 (read 1) FASTQ file and grab the path and sample name.
        # This will be the first input to the stage.
        # We assume the sample name may consist of only alphanumeric
        # characters.
        # Hi-Plex example: OHI031002-P02F04_S318_L001_R1_001.fastq
        # new sample name = OHI031002-P02F04
        filter=formatter(
            '.+/(?P<sample>[a-zA-Z0-9-]+)_(?P<readid>[a-zA-Z0-9-]+)_(?P<lane>[a-zA-Z0-9]+)_R1_(?P<lib>[a-zA-Z0-9-:]+).fastq'),
        # Add one more inputs to the stage:
        #    1. The corresponding R2 FASTQ file
        # Hi-Plex example: OHI031002-P02F04_S318_L001_R2_001.fastq
        add_inputs=add_inputs(
            '{path[0]}/{sample[0]}_{readid[0]}_{lane[0]}_R2_{lib[0]}.fastq'),
        # Add an "extra" argument to the state (beyond the inputs and outputs)
        # which is the sample name. This is needed within the stage for finding out
        # sample specific configuration options
        extras=['{sample[0]}', '{readid[0]}', '{lane[0]}', '{lib[0]}'],
        # The output file name is the sample name with a .bam extension.
        output='alignments/{sample[0]}_{readid[0]}/{sample[0]}_{readid[0]}.bam')

    # Call variants using undr_rover
    pipeline.transform(
        task_func=stages.apply_undr_rover,
        name='apply_undr_rover',
        input=output_from('original_fastqs'),
        # Match the R1 (read 1) FASTQ file and grab the path and sample name.
        # This will be the first input to the stage.
        # We assume the sample name may consist of only alphanumeric
        # characters.
        # Hi-Plex example: OHI031002-P02F04_S318_L001_R1_001.fastq
        filter=formatter(
            '.+/(?P<sample>[a-zA-Z0-9-]+)_(?P<readid>[a-zA-Z0-9-]+)_(?P<lane>[a-zA-Z0-9]+)_R1_(?P<lib>[a-zA-Z0-9-:]+).fastq'),
        # Add one more inputs to the stage:
        #    1. The corresponding R2 FASTQ file
        # Hi-Plex example: OHI031002-P02F04_S318_L001_R2_001.fastq

        add_inputs=add_inputs(
            '{path[0]}/{sample[0]}_{readid[0]}_{lane[0]}_R2_{lib[0]}.fastq'),
        # extras=['{sample[0]}', '{readid[0]}', '{lane[0]}', '{lib[0]}'],
        extras=['{sample[0]}', '{readid[0]}'],
        # The output file name is the sample name with a .bam extension.
        output='variants/undr_rover/{sample[0]}_{readid[0]}.vcf')

    # Sort the BAM file using Picard
    pipeline.transform(
        task_func=stages.sort_bam_picard,
        name='sort_bam_picard',
        input=output_from('align_bwa'),
        filter=suffix('.bam'),
        output='.sort.bam')

    # samtools index sorted bam file
    pipeline.transform(
        task_func=stages.index_sort_bam_picard,
        name='index_sort_bam_picard',
        input=output_from('sort_bam_picard'),
        filter=suffix('.sort.bam'),
        output='.sort.bam.bai')

    # Coverage using Picard
    (pipeline.transform(
        task_func=stages.target_coverage,
        name='target_coverage',
        input=output_from('sort_bam_picard'),
        # filter=suffix('.sort.bam'),
        filter=formatter(
            '.+/(?P<sample>[a-zA-Z0-9-_]+).sort.bam'),
        output='coverage/{sample[0]}.coverage.txt')
        .follows('sort_bam_picard'))

    # # Coverage using bam
    # (pipeline.transform(
    #     task_func=stages.target_coverage_bamutil,
    #     name='target_coverage_bamutil',
    #     input=output_from('sort_bam_picard'),
    #     # filter=suffix('.sort.bam'),
    #     filter=formatter(
    #         '.+/(?P<sample>[a-zA-Z0-9-_]+).sort.bam'),
    #     output='coverage/{sample[0]}.bamutil.txt')
    #     .follows('index_sort_bam_picard'))
    #
    # # Coverage using bam with interval
    # (pipeline.transform(
    #     task_func=stages.target_coverage_bamutil_interval,
    #     name='target_coverage_bamutil_interval',
    #     input=output_from('sort_bam_picard'),
    #     # filter=suffix('.sort.bam'),
    #     filter=formatter(
    #         '.+/(?P<sample>[a-zA-Z0-9-_]+).sort.bam'),
    #     output='coverage/{sample[0]}.bamutil2.txt')
    #     .follows('index_sort_bam_picard'))

    # Apply samtools
    pipeline.merge(
        task_func=stages.apply_samtools_mpileup,
        name='apply_samtools_mpileup',
        input=output_from('sort_bam_picard'),
        # filter=suffix('.sort.bam'),
        #filter=formatter('.+/(?P<sample>[a-zA-Z0-9-]+).sort.bam'),
        output='variants/all.mpileup')
        #filter=formatter('.+/(?P<sample>[a-zA-Z0-9-]+).sort.bam'),
        #output='variants/all.bcf')
        #.follows('sort_bam_picard'))

    # Apply bcftools
    (pipeline.transform(
        task_func=stages.apply_bcftools,
        name='apply_bcftools',
        input=output_from('apply_samtools_mpileup'),
        filter=suffix('.mpileup'),
        # add_inputs=add_inputs(['variants/ALL.indel_recal', 'variants/ALL.indel_tranches']),
        output='.raw.vcf')
        .follows('apply_samtools_mpileup'))

    # Apply NORM
    (pipeline.transform(
        task_func=stages.apply_vt,
        name='apply_vt',
        input=output_from('apply_bcftools'),
        filter=suffix('.raw.vcf'),
        # add_inputs=add_inputs(['variants/ALL.indel_recal', 'variants/ALL.indel_tranches']),
        output='.raw.vt.vcf')
        .follows('sort_bam_picard'))

    # Apply VEP
    (pipeline.transform(
        task_func=stages.apply_vep,
        name='apply_vep',
        input=output_from('apply_vt'),
        filter=suffix('.raw.vt.vcf'),
        # add_inputs=add_inputs(['variants/ALL.indel_recal', 'variants/ALL.indel_tranches']),
        output='.raw.vt.vep.vcf')
        .follows('apply_vt'))

    # Apply BCF
    # (pipeline.transform(
    #     task_func=stages.apply_bcf,
    #     name='apply_bcf',
    #     input=output_from('apply_vep'),
    #     filter=suffix('.raw.vqsr.vt.vep.vcf'),
    #     # add_inputs=add_inputs(['variants/ALL.indel_recal', 'variants/ALL.indel_tranches']),
    #     output='.raw.vt.vep.bcf.vcf')
    #     .follows('apply_vep'))

    # Apply SnpEff
    (pipeline.transform(
        task_func=stages.apply_snpeff,
        name='apply_snpeff',
        input=output_from('apply_vep'),
        filter=suffix('.raw.vt.vep.vcf'),
        # add_inputs=add_inputs(['variants/ALL.indel_recal', 'variants/ALL.indel_tranches']),
        output='.raw.vt.vep.snpeff.vcf')
        .follows('apply_vep'))

    # Apply vcfanno
    (pipeline.transform(
        task_func=stages.apply_vcfanno,
        name='apply_vcfanno',
        input=output_from('apply_snpeff'),
        filter=suffix('.raw.vt.vep.snpeff.vcf'),
        # add_inputs=add_inputs(['variants/ALL.indel_recal', 'variants/ALL.indel_tranches']),
        output='.annotated.vcf')
        .follows('apply_snpeff'))

    # Concatenate undr_rover vcf files
    pipeline.merge(
        task_func=stages.apply_cat_vcf,
        name='apply_cat_vcf',
        input=output_from('apply_undr_rover'),
        output='variants/undr_rover/ur.vcf.gz')

    # Apple VEP on concatenated undr_rover vcf file
    (pipeline.transform(
        task_func=stages.apply_vep2,
        name='apply_vep2',
        input=output_from('apply_cat_vcf'),
        output='variants/undr_rover/ur.vep.vcf')
        .follows('apply_cat_vcf'))

    return pipeline
<|endoftext|>


<commit_before>
from base64 import b64decode

from biothings.utils.web.es_dsl import AsyncSearch
from biothings.web.pipeline import ESQueryBuilder, ESResultTransform

from controller import OpenAPI, Swagger
from utils import decoder

class SmartAPIQueryBuilder(ESQueryBuilder):

    def default_string_query(self, q, options):

        search = AsyncSearch()

        if q == '__all__':
            search = search.query()

        elif q == '__any__' and self.allow_random_query:
            search = search.query('function_score', random_score={})

        # elasticsearch query string syntax
        elif ":" in q or " AND " in q or " OR " in q:
            search = search.query('query_string', query=q)

        else:  # simple text search
            query = {
                "query": {
                    "dis_max": {
                        "queries": [
                            {"term": {"_id": {"value": q, "boost": 5}}},
                            {"term": {"_meta.slug": {"value": q, "boost": 3}}},
                            {"match": {"info.title": {"query": q, "boost": 1.5}}},
                            {"term": {"servers.url": {"value": q, "boost": 1.1}}},
                            # ---------------------------------------------
                            {"query_string": {"query": q}},  # base score
                            # ---------------------------------------------
                            {"wildcard": {"info.title": {"value": q + "*", "boost": 0.8}}},
                            {"wildcard": {"info.description": {"value": q + "*", "boost": 0.5}}},
                        ]
                    }
                }
            }
            search = AsyncSearch()
            search = search.update_from_dict(query)

        search = search.params(rest_total_hits_as_int=True)
        search = search.source(exclude=['_raw'], include=options._source)

        if options.authors:  # '"Chunlei Wu"'
            search = search.filter('terms', info__contact__name__raw=options.authors)

        if options.tags:  # '"chemical", "drug"'
            search = search.filter('terms', tags__name__raw=options.tags)

        return search

    def default_match_query(self, q, scopes, options):
        search = super().default_match_query(q, scopes, options)
        search = search.source(include=options._source or ['_*'])
        return search

    def _apply_extras(self, search, options):
        """
        Process non-query options and customize their behaviors.
        Customized aggregation syntax string is translated here.
        """

        # add aggregations
        facet_size = options.facet_size or 10
        for agg in options.aggs or []:
            term, bucket = agg, search.aggs
            while term:
                if self.allow_nested_query and \
                        '(' in term and term.endswith(')'):
                    _term, term = term[:-1].split('(', 1)
                else:
                    _term, term = term, ''
                bucket = bucket.bucket(
                    _term, 'terms', field=_term, size=facet_size)

        # add es params
        if isinstance(options.sort, list):
            # accept '-' prefixed field names
            search = search.sort(*options.sort)

        # OVERRIDE
        # -------------------------------------------------------
        # if isinstance(options._source, list):
        #     if 'all' not in options._source:
        #         search = search.source(options._source)
        # -------------------------------------------------------

        for key, value in options.items():
            if key in ('from', 'size', 'explain', 'version'):
                search = search.extra(**{key: value})

        return search

class SmartAPIResultTransform(ESResultTransform):

    def transform_hit(self, path, doc, options):

        if path == '':
            doc.pop('_index')
            doc.pop('_type', None)    # not available by default on es7
            doc.pop('sort', None)     # added when using sort
            doc.pop('_node', None)    # added when using explain
            doc.pop('_shard', None)   # added when using explain

            # OVERRIDE STARTS HERE

            if "_raw" in doc:
                _raw = b64decode(doc.pop('_raw'))
                _raw = decoder.decompress(_raw)
                _raw = decoder.to_dict(_raw)
                doc.update(_raw)

            if options.raw == 0:
                for key in list(doc.keys()):
                    if key.startswith('_'):
                        doc.pop(key)

            if isinstance(doc.get('paths'), list):
                doc['paths'] = {
                    item['path']: item.get('pathitem', {})
                    for item in doc['paths']
                }

            # NOTE
            # Root field filtering in transform stage (if necessary)
            # ---------------------------------------------------------------
            # if options._source:
            #     fields = {field.split('.')[0] for field in options._source}
            #     for key in list(doc.keys()):
            #         if key not in fields:
            #             doc.pop(key)

            # field ordering
            if not options.sorted:

                if 'openapi' in doc:
                    _doc = OpenAPI(doc)
                    _doc.order()
                    # meta fields appear first
                    for key in list(doc.keys()):
                        if not key.startswith('_'):
                            doc.pop(key)
                    doc.update(_doc)

                elif 'swagger' in doc:
                    _doc = Swagger(doc)
                    _doc.order()
                    # meta fields appear first
                    for key in list(doc.keys()):
                        if not key.startswith('_'):
                            doc.pop(key)
                    doc.update(_doc)
<commit_msg>further experiment search result relevance<commit_after>
from base64 import b64decode

from biothings.utils.web.es_dsl import AsyncSearch
from biothings.web.pipeline import ESQueryBuilder, ESResultTransform

from controller import OpenAPI, Swagger
from utils import decoder

class SmartAPIQueryBuilder(ESQueryBuilder):

    def default_string_query(self, q, options):

        search = AsyncSearch()

        if q == '__all__':
            search = search.query()

        elif q == '__any__' and self.allow_random_query:
            search = search.query('function_score', random_score={})

        # elasticsearch query string syntax
        elif ":" in q or " AND " in q or " OR " in q:
            search = search.query('query_string', query=q)

        # term search
        elif q.startswith('"') and q.endswith('"'):
            query = {
                "query": {
                    "dis_max": {
                        "queries": [
                            {"match": {"info.title": {"query": q, "boost": 1.5, "operator": "AND"}}},
                            {"query_string": {"query": q, "default_operator": "AND"}}  # base score
                        ]
                    }
                }
            }
            search = AsyncSearch()
            search = search.update_from_dict(query)

        else:  # simple text search
            query = {
                "query": {
                    "dis_max": {
                        "queries": [
                            {"term": {"_id": {"value": q, "boost": 5}}},
                            {"term": {"_meta.slug": {"value": q, "boost": 3}}},
                            {"match": {"info.title": {"query": q, "boost": 1.5}}},
                            {"term": {"servers.url": {"value": q, "boost": 1.1}}},
                            # ---------------------------------------------
                            {"query_string": {"query": q}},  # base score
                            # ---------------------------------------------
                            {"wildcard": {"info.title": {"value": q + "*", "boost": 0.8}}},
                            {"wildcard": {"info.description": {"value": q + "*", "boost": 0.5}}},
                        ]
                    }
                }
            }
            search = AsyncSearch()
            search = search.update_from_dict(query)

        search = search.params(rest_total_hits_as_int=True)
        search = search.source(exclude=['_raw'], include=options._source)

        if options.authors:  # '"Chunlei Wu"'
            search = search.filter('terms', info__contact__name__raw=options.authors)

        if options.tags:  # '"chemical", "drug"'
            search = search.filter('terms', tags__name__raw=options.tags)

        return search

    def default_match_query(self, q, scopes, options):
        search = super().default_match_query(q, scopes, options)
        search = search.source(include=options._source or ['_*'])
        return search

    def _apply_extras(self, search, options):
        """
        Process non-query options and customize their behaviors.
        Customized aggregation syntax string is translated here.
        """

        # add aggregations
        facet_size = options.facet_size or 10
        for agg in options.aggs or []:
            term, bucket = agg, search.aggs
            while term:
                if self.allow_nested_query and \
                        '(' in term and term.endswith(')'):
                    _term, term = term[:-1].split('(', 1)
                else:
                    _term, term = term, ''
                bucket = bucket.bucket(
                    _term, 'terms', field=_term, size=facet_size)

        # add es params
        if isinstance(options.sort, list):
            # accept '-' prefixed field names
            search = search.sort(*options.sort)

        # OVERRIDE
        # -------------------------------------------------------
        # if isinstance(options._source, list):
        #     if 'all' not in options._source:
        #         search = search.source(options._source)
        # -------------------------------------------------------

        for key, value in options.items():
            if key in ('from', 'size', 'explain', 'version'):
                search = search.extra(**{key: value})

        return search

class SmartAPIResultTransform(ESResultTransform):

    def transform_hit(self, path, doc, options):

        if path == '':
            doc.pop('_index')
            doc.pop('_type', None)    # not available by default on es7
            doc.pop('sort', None)     # added when using sort
            doc.pop('_node', None)    # added when using explain
            doc.pop('_shard', None)   # added when using explain

            # OVERRIDE STARTS HERE

            if "_raw" in doc:
                _raw = b64decode(doc.pop('_raw'))
                _raw = decoder.decompress(_raw)
                _raw = decoder.to_dict(_raw)
                doc.update(_raw)

            if options.raw == 0:
                for key in list(doc.keys()):
                    if key.startswith('_'):
                        doc.pop(key)

            if isinstance(doc.get('paths'), list):
                doc['paths'] = {
                    item['path']: item.get('pathitem', {})
                    for item in doc['paths']
                }

            # NOTE
            # Root field filtering in transform stage (if necessary)
            # ---------------------------------------------------------------
            # if options._source:
            #     fields = {field.split('.')[0] for field in options._source}
            #     for key in list(doc.keys()):
            #         if key not in fields:
            #             doc.pop(key)

            # field ordering
            if not options.sorted:

                if 'openapi' in doc:
                    _doc = OpenAPI(doc)
                    _doc.order()
                    # meta fields appear first
                    for key in list(doc.keys()):
                        if not key.startswith('_'):
                            doc.pop(key)
                    doc.update(_doc)

                elif 'swagger' in doc:
                    _doc = Swagger(doc)
                    _doc.order()
                    # meta fields appear first
                    for key in list(doc.keys()):
                        if not key.startswith('_'):
                            doc.pop(key)
                    doc.update(_doc)
<|endoftext|>


<commit_before>from django.contrib import admin
from order.models import *
from forms import *
from django.utils.translation import ugettext_lazy as _
import copy

# Register your models here.

class ScientificMissionAdmin(admin.ModelAdmin):
    # Hide this link on the main menu.
    def get_model_perms(self, request):
        perms = super(ScientificMissionAdmin, self).get_model_perms(request)
        perms['list_hide'] = True
        return perms

admin.site.register(ScientificMission, ScientificMissionAdmin)

class SuperOrder(admin.ModelAdmin):
    # Shows the requests according to the user permission.
    # Users defined as superuser or NIRA Admin can see all orders.
    def get_queryset(self, request):
        qs = super(SuperOrder, self).get_queryset(request)
        if request.user.projectmember.is_nira_admin or request.user.is_superuser:
            return qs
        return qs.filter(requester=request.user)

    # If not superuser or NIRA Admin, the status field becomes read-only.
    def get_readonly_fields(self, request, obj=None):
        ro_fields = super(SuperOrder, self).get_readonly_fields(request, obj)
        if not request.user.is_superuser and not request.user.projectmember.is_nira_admin:
            ro_fields = list(ro_fields) + ['status']
        return ro_fields

    # If superuser or NIRA Admin, show the requester and protocol fields.
    def get_fieldsets(self, request, obj=None):
        fieldsets = copy.deepcopy(super(SuperOrder, self).get_fieldsets(request, obj))
        if request.user.projectmember.is_nira_admin or request.user.is_superuser:
            fieldsets[0][1]['fields'].append('requester')
            fieldsets.append((_('Administrative system'), {'fields': ('protocol',)}),)
        return fieldsets

    # If not superuser or NIRA Admin, set the requester as the current user and the status as Open.
    def save_model(self, request, obj, form, change):
        if not request.user.is_superuser and not request.user.projectmember.is_nira_admin:
            if not change:
                obj.requester = ProjectMember.objects.get(user=request.user)
                obj.status = 'o'
        obj.save()

class OrderAdmin(SuperOrder):
    list_display = ('id_order', 'date_modified', 'order_date', 'status', 'type_of_order', 'requester')
    list_per_page = 15
    list_filter = ('status', 'type_of_order', 'requester',)

    # Disable the option to add order.
    def has_add_permission(self, request, obj=None):
        return False

    # Disable the option to delete order.
    def has_delete_permission(self, request, obj=None):
        return False

    # Hide the order link on the main menu.
    def get_model_perms(self, request):
        perms = super(OrderAdmin, self).get_model_perms(request)
        perms['list_hide'] = True
        return perms

admin.site.register(Order, OrderAdmin)

class EventAdmin(SuperOrder):
    fieldsets = [
        (None, {
            'fields': ['status']
        }),
        (_('Event Info'), {
            'fields': ('name', 'url', 'value', 'start_date', 'end_date', 'invitation')
        }),
        (_('Purpose'), {
            'fields': ('justification',)
        }),
    ]

    list_display = ('order_number', 'status', 'requester', 'name', 'order_date')

    list_display_links = ('order_number', 'status', 'name', 'order_date')

    form = EventAdminForm

admin.site.register(Event, EventAdmin)

class HardwareSoftwareAdmin(SuperOrder):
    fieldsets = [
        (None, {
            'fields': ['status']
        }),
        (_('Info about the equipment, supplies or miscellaneous'), {
            'fields': ('type', 'quantity', 'url')
        }),
        (_('Purpose'), {
            'fields': ('justification',)
        }),
    ]

    list_display = ('order_number', 'status', 'requester', 'type', 'order_date')

    list_display_links = ('order_number', 'status', 'type', 'order_date')

    form = HardwareSoftwareAdminForm

    # If superuser or NIRA admin, show origin and category fields.
    def get_fieldsets(self, request, obj=None):
        fieldsets = copy.deepcopy(super(HardwareSoftwareAdmin, self).get_fieldsets(request, obj))
        if request.user.projectmember.is_nira_admin or request.user.is_superuser:
            fieldsets[0][1]['fields'].append('origin')
            fieldsets[0][1]['fields'].append('category')
            fieldsets[0][1]['fields'].append('institution')
        return fieldsets

admin.site.register(HardwareSoftware, HardwareSoftwareAdmin)

class ServiceAdmin(SuperOrder):
    fieldsets = [
        (None, {
            'fields': ['status']
        }),
        (_('Service Info'), {
            'fields': ('type',)
        }),
        (_('Purpose'), {
            'fields': ('justification',)
        }),
    ]

    list_display = ('order_number', 'status', 'requester', 'type', 'order_date')

    list_display_links = ('order_number', 'status', 'type', 'order_date')

    form = ServiceAdminForm

    # If superuser or NIRA admin, show origin field.
    def get_fieldsets(self, request, obj=None):
        fieldsets = copy.deepcopy(super(ServiceAdmin, self).get_fieldsets(request, obj))
        if request.user.projectmember.is_nira_admin or request.user.is_superuser:
            fieldsets[0][1]['fields'].append('origin')
        return fieldsets

admin.site.register(Service, ServiceAdmin)

class TicketAdmin(SuperOrder):
    fieldsets = [
        (None, {
            'fields': ['status']
        }),
        (_('Ticket Info'), {
            'fields': ('type_transportation', 'type', 'origin', 'destination', 'outbound_date',
                       'outbound_date_preference', 'inbound_date', 'inbound_date_preference', 'note')
        }),
        (_('Purpose of the trip'), {
            'fields': ('justification',)
        }),
    ]

    list_display = ('order_number', 'status', 'requester', 'origin', 'destination', 'outbound_date', 'inbound_date',
                    'order_date')

    list_display_links = ('order_number', 'status', 'origin', 'destination', 'outbound_date', 'inbound_date',
                          'order_date')

    form = TicketAdminForm

admin.site.register(Ticket, TicketAdmin)

class DailyStipendAdmin(SuperOrder):
    fieldsets = [
        (None, {
            'fields': ['status']
        }),
        (_('Daily stipend Info'), {
            'fields': ('origin', 'destination', 'departure', 'arrival')
        }),
        (_('Purpose'), {
            'fields': ('justification',)
        }),
    ]

    list_display = ('order_number', 'status', 'requester', 'origin', 'destination', 'order_date')

    list_display_links = ('order_number', 'status', 'origin', 'destination', 'order_date')

    # If superuser or NIRA admin, show receiver and mission fields.
    def get_fieldsets(self, request, obj=None):
        fieldsets = copy.deepcopy(super(DailyStipendAdmin, self).get_fieldsets(request, obj))
        if request.user.projectmember.is_nira_admin or request.user.is_superuser:
            fieldsets[0][1]['fields'].append('receiver')
            fieldsets[0][1]['fields'].append('mission')
        return fieldsets

admin.site.register(DailyStipend, DailyStipendAdmin)

class ReimbursementAdmin(SuperOrder):
    fieldsets = [
        (None, {
            'fields': ['status']
        }),
        (_('Reimbursement Info'), {
            'fields': ('why',)
        }),
        (_('Purpose'), {
            'fields': ('justification',)
        }),
    ]

    list_display = ('order_number', 'status', 'requester', 'why', 'order_date')

    list_display_links = ('order_number', 'status', 'why', 'order_date')

admin.site.register(Reimbursement, ReimbursementAdmin)<commit_msg>Bug fix<commit_after>from django.contrib import admin
from order.models import *
from forms import *
from django.utils.translation import ugettext_lazy as _
import copy

# Register your models here.

class ScientificMissionAdmin(admin.ModelAdmin):
    # Hide this link on the main menu.
    def get_model_perms(self, request):
        perms = super(ScientificMissionAdmin, self).get_model_perms(request)
        perms['list_hide'] = True
        return perms

admin.site.register(ScientificMission, ScientificMissionAdmin)

class SuperOrder(admin.ModelAdmin):
    # Shows the requests according to the user permission.
    # Users defined as superuser or NIRA Admin can see all orders.
    def get_queryset(self, request):
        qs = super(SuperOrder, self).get_queryset(request)
        if request.user.projectmember.is_nira_admin or request.user.is_superuser:
            return qs
        return qs.filter(requester=request.user.projectmember)

    # If not superuser or NIRA Admin, the status field becomes read-only.
    def get_readonly_fields(self, request, obj=None):
        ro_fields = super(SuperOrder, self).get_readonly_fields(request, obj)
        if not request.user.is_superuser and not request.user.projectmember.is_nira_admin:
            ro_fields = list(ro_fields) + ['status']
        return ro_fields

    # If superuser or NIRA Admin, show the requester and protocol fields.
    def get_fieldsets(self, request, obj=None):
        fieldsets = copy.deepcopy(super(SuperOrder, self).get_fieldsets(request, obj))
        if request.user.projectmember.is_nira_admin or request.user.is_superuser:
            fieldsets[0][1]['fields'].append('requester')
            fieldsets.append((_('Administrative system'), {'fields': ('protocol',)}),)
        return fieldsets

    # If not superuser or NIRA Admin, set the requester as the current user and the status as Open.
    def save_model(self, request, obj, form, change):
        if not request.user.is_superuser and not request.user.projectmember.is_nira_admin:
            if not change:
                obj.requester = ProjectMember.objects.get(user=request.user)
                obj.status = 'o'
        obj.save()

class OrderAdmin(SuperOrder):
    list_display = ('id_order', 'date_modified', 'order_date', 'status', 'type_of_order', 'requester')
    list_per_page = 15
    list_filter = ('status', 'type_of_order', 'requester',)

    # Disable the option to add order.
    def has_add_permission(self, request, obj=None):
        return False

    # Disable the option to delete order.
    def has_delete_permission(self, request, obj=None):
        return False

    # Hide the order link on the main menu.
    def get_model_perms(self, request):
        perms = super(OrderAdmin, self).get_model_perms(request)
        perms['list_hide'] = True
        return perms

admin.site.register(Order, OrderAdmin)

class EventAdmin(SuperOrder):
    fieldsets = [
        (None, {
            'fields': ['status']
        }),
        (_('Event Info'), {
            'fields': ('name', 'url', 'value', 'start_date', 'end_date', 'invitation')
        }),
        (_('Purpose'), {
            'fields': ('justification',)
        }),
    ]

    list_display = ('order_number', 'status', 'requester', 'name', 'order_date')

    list_display_links = ('order_number', 'status', 'name', 'order_date')

    form = EventAdminForm

admin.site.register(Event, EventAdmin)

class HardwareSoftwareAdmin(SuperOrder):
    fieldsets = [
        (None, {
            'fields': ['status']
        }),
        (_('Info about the equipment, supplies or miscellaneous'), {
            'fields': ('type', 'quantity', 'url')
        }),
        (_('Purpose'), {
            'fields': ('justification',)
        }),
    ]

    list_display = ('order_number', 'status', 'requester', 'type', 'order_date')

    list_display_links = ('order_number', 'status', 'type', 'order_date')

    form = HardwareSoftwareAdminForm

    # If superuser or NIRA admin, show origin and category fields.
    def get_fieldsets(self, request, obj=None):
        fieldsets = copy.deepcopy(super(HardwareSoftwareAdmin, self).get_fieldsets(request, obj))
        if request.user.projectmember.is_nira_admin or request.user.is_superuser:
            fieldsets[0][1]['fields'].append('origin')
            fieldsets[0][1]['fields'].append('category')
            fieldsets[0][1]['fields'].append('institution')
        return fieldsets

admin.site.register(HardwareSoftware, HardwareSoftwareAdmin)

class ServiceAdmin(SuperOrder):
    fieldsets = [
        (None, {
            'fields': ['status']
        }),
        (_('Service Info'), {
            'fields': ('type',)
        }),
        (_('Purpose'), {
            'fields': ('justification',)
        }),
    ]

    list_display = ('order_number', 'status', 'requester', 'type', 'order_date')

    list_display_links = ('order_number', 'status', 'type', 'order_date')

    form = ServiceAdminForm

    # If superuser or NIRA admin, show origin field.
    def get_fieldsets(self, request, obj=None):
        fieldsets = copy.deepcopy(super(ServiceAdmin, self).get_fieldsets(request, obj))
        if request.user.projectmember.is_nira_admin or request.user.is_superuser:
            fieldsets[0][1]['fields'].append('origin')
        return fieldsets

admin.site.register(Service, ServiceAdmin)

class TicketAdmin(SuperOrder):
    fieldsets = [
        (None, {
            'fields': ['status']
        }),
        (_('Ticket Info'), {
            'fields': ('type_transportation', 'type', 'origin', 'destination', 'outbound_date',
                       'outbound_date_preference', 'inbound_date', 'inbound_date_preference', 'note')
        }),
        (_('Purpose of the trip'), {
            'fields': ('justification',)
        }),
    ]

    list_display = ('order_number', 'status', 'requester', 'origin', 'destination', 'outbound_date', 'inbound_date',
                    'order_date')

    list_display_links = ('order_number', 'status', 'origin', 'destination', 'outbound_date', 'inbound_date',
                          'order_date')

    form = TicketAdminForm

admin.site.register(Ticket, TicketAdmin)

class DailyStipendAdmin(SuperOrder):
    fieldsets = [
        (None, {
            'fields': ['status']
        }),
        (_('Daily stipend Info'), {
            'fields': ('origin', 'destination', 'departure', 'arrival')
        }),
        (_('Purpose'), {
            'fields': ('justification',)
        }),
    ]

    list_display = ('order_number', 'status', 'requester', 'origin', 'destination', 'order_date')

    list_display_links = ('order_number', 'status', 'origin', 'destination', 'order_date')

    # If superuser or NIRA admin, show receiver and mission fields.
    def get_fieldsets(self, request, obj=None):
        fieldsets = copy.deepcopy(super(DailyStipendAdmin, self).get_fieldsets(request, obj))
        if request.user.projectmember.is_nira_admin or request.user.is_superuser:
            fieldsets[0][1]['fields'].append('receiver')
            fieldsets[0][1]['fields'].append('mission')
        return fieldsets

admin.site.register(DailyStipend, DailyStipendAdmin)

class ReimbursementAdmin(SuperOrder):
    fieldsets = [
        (None, {
            'fields': ['status']
        }),
        (_('Reimbursement Info'), {
            'fields': ('why',)
        }),
        (_('Purpose'), {
            'fields': ('justification',)
        }),
    ]

    list_display = ('order_number', 'status', 'requester', 'why', 'order_date')

    list_display_links = ('order_number', 'status', 'why', 'order_date')

admin.site.register(Reimbursement, ReimbursementAdmin)<|endoftext|>


<commit_before>#import copper
import numpy as np
import pandas as pd
import requests
from bs4 import BeautifulSoup
from datetime import datetime, date
#copper.project.path = '../../'

year = 2010
#teams = copper.read_csv('teams.csv')
teams = pd.read_csv('/Users/daniel.george/Desktop/Github/espn-analytics/nfl/teams.csv')
BASE_URL = 'http://espn.go.com/nfl/team/schedule/_/name/{0}/year/{1}/{2}/seasontype/2'

# @Author Daniel George
# @Works Cited: Daniel Rodriguez, https://github.com/danielfrg/nba

match_id = []
dates = []
home_team = []
home_team_score = []
visit_team = []
visit_team_score = []

for index, row in teams.iterrows():
    _team, url = row['team'], row['url']
    r = requests.get(BASE_URL.format(row['prefix_1'], year, row['prefix_2']))
    #table = BeautifulSoup(r.text, "html.parser").table
    #new attempt
    soup = BeautifulSoup(r.text, "html.parser")
    table = soup.find('table', {'class':'tablehead'})

#     table_body = table.find('tbody')
#
#     print table_body

    for row in table.find_all('tr')[1:]: # Remove header
        columns = row.find_all('td')
        try:
            _home = True if columns[2].li.text == 'vs' else False
            _other_team = columns[2].find_all('a')[1].text
            _score = columns[3].a.text.split(' ')[0].split('-')
            _won = True if columns[3].span.text == 'W' else False

            match_id.append(columns[3].a['href'].split('?gameId=')[1])
            home_team.append(_team if _home else _other_team)
            visit_team.append(_team if not _home else _other_team)
            d = datetime.strptime(columns[1].text, '%a, %b %d')
            dates.append(date(year, d.month, d.day))

            if _home:
                if _won:
                    home_team_score.append(_score[0])
                    visit_team_score.append(_score[1])
                else:
                    home_team_score.append(_score[1])
                    visit_team_score.append(_score[0])
            else:
                if _won:
                    home_team_score.append(_score[1])
                    visit_team_score.append(_score[0])
                else:
                    home_team_score.append(_score[0])
                    visit_team_score.append(_score[1])
        except Exception as e:
            pass # Not all columns row are a match, is OK
            # print(e)

dic = {'id': match_id, 'date': dates, 'home_team': home_team, 'visit_team': visit_team,
        'home_team_score': home_team_score, 'visit_team_score': visit_team_score}

games = pd.DataFrame(dic).drop_duplicates(subset='id').set_index('id')
#print(games)
#copper.save(games, 'games')
games.to_csv('games.csv')<commit_msg>added comments<commit_after>import numpy as np
import pandas as pd
import requests
from bs4 import BeautifulSoup
from datetime import datetime, date

# @Author Daniel George
# @Works Cited: Daniel Rodriguez, https://github.com/danielfrg/nba

# Replace the "year" variable with the year
# you want to pull.
year = 2010

# Insert the path to the "teams.csv" file
# created by "get_teams.py".
teams = pd.read_csv('/path/to/teams.csv')
BASE_URL = 'http://espn.go.com/nfl/team/schedule/_/name/{0}/year/{1}/{2}/seasontype/2'

match_id = []
dates = []
home_team = []
home_team_score = []
visit_team = []
visit_team_score = []

for index, row in teams.iterrows():
    _team, url = row['team'], row['url']
    r = requests.get(BASE_URL.format(row['prefix_1'], year, row['prefix_2']))
    soup = BeautifulSoup(r.text, "html.parser")
    table = soup.find('table', {'class':'tablehead'})

    for row in table.find_all('tr')[1:]: # Remove header
        columns = row.find_all('td')
        try:
            _home = True if columns[2].li.text == 'vs' else False
            _other_team = columns[2].find_all('a')[1].text
            _score = columns[3].a.text.split(' ')[0].split('-')
            _won = True if columns[3].span.text == 'W' else False

            match_id.append(columns[3].a['href'].split('?gameId=')[1])
            home_team.append(_team if _home else _other_team)
            visit_team.append(_team if not _home else _other_team)
            d = datetime.strptime(columns[1].text, '%a, %b %d')
            dates.append(date(year, d.month, d.day))

            if _home:
                if _won:
                    home_team_score.append(_score[0])
                    visit_team_score.append(_score[1])
                else:
                    home_team_score.append(_score[1])
                    visit_team_score.append(_score[0])
            else:
                if _won:
                    home_team_score.append(_score[1])
                    visit_team_score.append(_score[0])
                else:
                    home_team_score.append(_score[0])
                    visit_team_score.append(_score[1])
        except Exception as e:
            pass # Not all columns row are a match, is OK
            # print(e)

dic = {'id': match_id, 'date': dates, 'home_team': home_team, 'visit_team': visit_team,
        'home_team_score': home_team_score, 'visit_team_score': visit_team_score}

games = pd.DataFrame(dic).drop_duplicates(subset='id').set_index('id')
games.to_csv('games.csv')<|endoftext|>