<commit_before>import fitsio
import esutil as eu
from esutil.htm import Matcher
import numpy as np
import itertools
from catalog import Catalog, Entry
import healpy as hp
import os

class Galaxy(Entry):

    """ docstring """

    pass

class GalaxyCatalog(Catalog):

    """ docstring """

    entry_class = Galaxy

    def __init__(self, *arrays, **kwargs):
        super(GalaxyCatalog, self).__init__(*arrays)
        self._htm_matcher = None
        self.depth = 10 if 'depth' not in kwargs else kwargs['depth']

    @classmethod
    def from_galfile(cls, filename, nside=None, hpix=None, border=0.0):
        """ docstring """
        if hpix is not None and nside is None:
            raise ValueError("If hpix is specified, must also specify nside")
        if border < 0.0:
            raise ValueError("Border must be >= 0.0.")
        # ensure that nside is valid, and hpix is within range (if necessary)
        if nside is not None:
            if not hp.isnsideok(nside):
                raise ValueError("Nside not valid")
            if hpix is not None:
                if hpix < 0 or hpix >= hp.nside2npix(nside):
                    raise ValueError("hpix out of range.")
        # check that the file is there and the right format
        # this will raise an exception if it's not there.
        hdr = fitsio.read_header(filename, ext=1)
        if 'PIXELS' not in hdr:
            return super(GalaxyCatalog, self).from_fits_file(filename)
        pixelated = hdr['PIXELS']
        # this is to keep us from trying to use old IDL galfiles
        if 'FITS' not in hdr:
            raise ValueError("Input galfile must describe fits files.")
        fitsformat = hdr['FITS']
        # now we can read in the galaxy table summary file...
        tab = fitsio.read(filename, ext=1)
        nside_tab = tab['NSIDE']
        if (nside > nside_tab):
            raise ValueError("""Requested nside (%d) must not be larger than
                                    table nside (%d).""" % (nside, nside_tab))
        # which files do we want to read?
        path = os.path.dirname(os.path.abspath(filename))
        if hpix is None:
            # all of them!
            indices = np.arange(tab[0]['FILENAMES'].size)
        else:
            # first we need all the pixels that are contained in the big pixel
            theta, phi = hp.pix2ang(nside_tab, tab[0]['HPIX'])
            ipring_big = hp.ang2pix(nside, theta, phi)
            indices = np.where(ipring_big == hpix)
            if border > 0.0:
                # now we need to find the extra boundary...
                boundaries = hp.boundaries(nside, hpix, step=nside_tab/nside)
                inhpix = tab[0]['HPIX'][indices]
                for i in xrange(boundaries.shape[1]):
                    pixint = hp.query_disc(nside_tab, boundaries[:,i],
                                    border*np.pi/180., inclusive=True, fact=8)
                    inhpix = np.append(inhpix, pixint)
                inhpix = np.unique(inhpix)
                _, indices = eu.numpy_util.match(inhpix, tab[0]['HPIX'])
        # create the catalog array to read into
        elt = fitsio.read('%s/%s' % (path, tab[0]['FILENAMES'][indices[0]]),
                                                                ext=1, rows=0)
        cat = np.zeros(np.sum(tab[0]['NGALS'][indices]), dtype=elt.dtype)
        # read the files
        ctr = 0
        for index in indices:
            cat[ctr : ctr+tab[0]['NGALS'][index]] =
                    fitsio.read('%s/%s' % (path, tab[0]['FILENAMES'][index]),
                                                                        ext=1)
            ctr += tab[0]['NGALS'][index]
        # In the IDL version this is trimmed to the precise boundary requested.
        # that's easy in simplepix.  Not sure how to do in healpix.
        return cls(cat)

    def match(self, galaxy, radius):
        if self._htm_matcher is None:
            self._htm_matcher = Matcher(self.depth, self.ra, self.dec)
        _, indices, dists = self._htm_matcher.match(galaxy.ra, galaxy.dec,
                                                        radius, maxmatch=0)
        return indices, dists

<commit_msg>consolidate line that was causing strange error<commit_after>import fitsio
import esutil as eu
from esutil.htm import Matcher
import numpy as np
import itertools
from catalog import Catalog, Entry
import healpy as hp
import os

class Galaxy(Entry):

    """ docstring """

    pass

class GalaxyCatalog(Catalog):

    """ docstring """

    entry_class = Galaxy

    def __init__(self, *arrays, **kwargs):
        super(GalaxyCatalog, self).__init__(*arrays)
        self._htm_matcher = None
        self.depth = 10 if 'depth' not in kwargs else kwargs['depth']

    @classmethod
    def from_galfile(cls, filename, nside=None, hpix=None, border=0.0):
        """ docstring """
        if hpix is not None and nside is None:
            raise ValueError("If hpix is specified, must also specify nside")
        if border < 0.0:
            raise ValueError("Border must be >= 0.0.")
        # ensure that nside is valid, and hpix is within range (if necessary)
        if nside is not None:
            if not hp.isnsideok(nside):
                raise ValueError("Nside not valid")
            if hpix is not None:
                if hpix < 0 or hpix >= hp.nside2npix(nside):
                    raise ValueError("hpix out of range.")
        # check that the file is there and the right format
        # this will raise an exception if it's not there.
        hdr = fitsio.read_header(filename, ext=1)
        if 'PIXELS' not in hdr:
            return super(GalaxyCatalog, self).from_fits_file(filename)
        pixelated = hdr['PIXELS']
        # this is to keep us from trying to use old IDL galfiles
        if 'FITS' not in hdr:
            raise ValueError("Input galfile must describe fits files.")
        fitsformat = hdr['FITS']
        # now we can read in the galaxy table summary file...
        tab = fitsio.read(filename, ext=1)
        nside_tab = tab['NSIDE']
        if (nside > nside_tab):
            raise ValueError("""Requested nside (%d) must not be larger than
                                    table nside (%d).""" % (nside, nside_tab))
        # which files do we want to read?
        path = os.path.dirname(os.path.abspath(filename))
        if hpix is None:
            # all of them!
            indices = np.arange(tab[0]['FILENAMES'].size)
        else:
            # first we need all the pixels that are contained in the big pixel
            theta, phi = hp.pix2ang(nside_tab, tab[0]['HPIX'])
            ipring_big = hp.ang2pix(nside, theta, phi)
            indices = np.where(ipring_big == hpix)
            if border > 0.0:
                # now we need to find the extra boundary...
                boundaries = hp.boundaries(nside, hpix, step=nside_tab/nside)
                inhpix = tab[0]['HPIX'][indices]
                for i in xrange(boundaries.shape[1]):
                    pixint = hp.query_disc(nside_tab, boundaries[:,i],
                                    border*np.pi/180., inclusive=True, fact=8)
                    inhpix = np.append(inhpix, pixint)
                inhpix = np.unique(inhpix)
                _, indices = eu.numpy_util.match(inhpix, tab[0]['HPIX'])
        # create the catalog array to read into
        elt = fitsio.read('%s/%s' % (path, tab[0]['FILENAMES'][indices[0]]),
                                                                ext=1, rows=0)
        cat = np.zeros(np.sum(tab[0]['NGALS'][indices]), dtype=elt.dtype)
        # read the files
        ctr = 0
        for index in indices:
            cat[ctr : ctr+tab[0]['NGALS'][index]] = fitsio.read('%s/%s' % (path, tab[0]['FILENAMES'][index]),ext=1)
            ctr += tab[0]['NGALS'][index]
        # In the IDL version this is trimmed to the precise boundary requested.
        # that's easy in simplepix.  Not sure how to do in healpix.
        return cls(cat)

    def match(self, galaxy, radius):
        if self._htm_matcher is None:
            self._htm_matcher = Matcher(self.depth, self.ra, self.dec)
        _, indices, dists = self._htm_matcher.match(galaxy.ra, galaxy.dec,
                                                        radius, maxmatch=0)
        return indices, dists

<|endoftext|>


<commit_before>"""
.. See the NOTICE file distributed with this work for additional information
   regarding copyright ownership.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
"""

from __future__ import print_function

import os
import shlex
import subprocess
import sys
import re
import tarfile

from utils import logger

try:
    if hasattr(sys, '_run_from_cmdl') is True:
        raise ImportError
    from pycompss.api.parameter import FILE_IN, FILE_OUT, IN, OUT
    from pycompss.api.task import task
    from pycompss.api.api import compss_wait_on
except ImportError:
    logger.warn("[Warning] Cannot import \"pycompss\" API packages.")
    logger.warn("          Using mock decorators.")

    from utils.dummy_pycompss import FILE_IN, FILE_OUT, IN, OUT # pylint: disable=ungrouped-imports
    from utils.dummy_pycompss import task # pylint: disable=ungrouped-imports
    from utils.dummy_pycompss import compss_wait_on # pylint: disable=ungrouped-imports

from basic_modules.tool import Tool
from basic_modules.metadata import Metadata

from tool.fastqreader import fastqreader

# ------------------------------------------------------------------------------

class fastq_splitter(Tool):
    """
    Script for splitting up FASTQ files into manageable chunks
    """

    def __init__(self, configuration=None):
        """
        Initialise the tool with its configuration.

        Parameters
        ----------
        configuration : dict
            a dictionary containing parameters that define how the operation
            should be carried out, which are specific to each Tool.
        """
        logger.info("FASTQ Splitter")
        Tool.__init__(self)

        if configuration is None:
            configuration = {}

        self.configuration.update(configuration)

    @task(
        in_file1=FILE_IN, tag=IN,
        out_file=FILE_OUT, files_out=OUT,
        returns=list)
    def single_splitter(self, in_file1, out_file, tag='tmp'):  # pylint disable=no-self-use
        """
        Function to divide the FastQ files into separate sub files of 1000000
        sequences so that the aligner can run in parallel.

        Parameters
        ----------
        in_file1 : str
            Location of first FASTQ file
        tag : str
            DEFAULT = tmp
            Tag used to identify the files. Useful if this is getting run
            manually on a single machine multiple times to prevent collisions of
            file names

        Returns
        -------
        Returns: Returns a list of the files that have been generated.
                 Each sub list containing the two paired end files for that
                 subset.
        paired_files : list
            List of lists of pair end files. Each sub list containing the two
            paired end files for that subset.
        """

        fqr = fastqreader()
        fqr.openFastQ(in_file1)
        fqr.createOutputFiles(tag)

        record1 = fqr.next(1)

        count_r3 = 0

        file_loc_1 = fqr.fastq1.split("/")
        file_loc_1[-1] = file_loc_1[-1].replace(
            ".fastq",
            "." + str(fqr.output_tag) + "_" + str(fqr.output_file_count) + ".fastq")
        file_loc_1.insert(-1, tag)

        files_out = [[file_loc_1[-1]]]

        while fqr.eof(1) is False:
            fqr.writeOutput(record1, 1)
            record1 = fqr.next(1)
            count_r3 += 1

            if count_r3 % 1000000 == 0:
                fqr.incrementOutputFiles()
                file_loc_1 = fqr.fastq1.split("/")
                new_suffix = "." + str(fqr.output_tag) + "_" + str(fqr.output_file_count) + ".fastq"
                file_loc_1[-1] = re.sub('.fastq$', new_suffix, file_loc_1[-1])
                file_loc_1.insert(-1, tag)

                files_out.append([file_loc_1[-1]])

        fqr.closeFastQ()
        fqr.closeOutputFiles()

        output_file_pregz = out_file.replace('.tar.gz', '.tar')

        if os.path.isfile(out_file):
            os.remove(out_file)
        tar = tarfile.open(output_file_pregz, "w")
        tar.add("/".join(file_loc_1[:-1]), arcname='tmp')
        tar.close()

        command_line = 'pigz ' + output_file_pregz
        args = shlex.split(command_line)
        process = subprocess.Popen(args)
        process.wait()

        return files_out

    @task(
        in_file1=FILE_IN, in_file2=FILE_IN, tag=IN,
        out_file=FILE_OUT, files_out=OUT,
        returns=list)
    def paired_splitter(self, in_file1, in_file2, out_file, tag='tmp'):  # pylint disable=no-self-use
        """
        Function to divide the FastQ files into separte sub files of 1000000
        sequences so that the aligner can run in parallel.

        Parameters
        ----------
        in_file1 : str
            Location of first paired end FASTQ file
        in_file2 : str
            Location of second paired end FASTQ file
        tag : str
            DEFAULT = tmp
            Tag used to identify the files. Useful if this is getting run
            manually on a single machine multiple times to prevent collisions of
            file names

        Returns
        -------
        Returns: Returns a list of lists of the files that have been generated.
                 Each sub list containing the two paired end files for that
                 subset.
        paired_files : list
            List of lists of pair end files. Each sub list containing the two
            paired end files for that subset.
        """

        fqr = fastqreader()
        fqr.openFastQ(in_file1, in_file2)
        fqr.createOutputFiles(tag)

        record1 = fqr.next(1)
        record2 = fqr.next(2)

        count_r1 = 0
        count_r2 = 0
        count_r3 = 0

        file_loc_1 = fqr.fastq1.split("/")
        new_suffix = "." + str(fqr.output_tag) + "_" + str(fqr.output_file_count) + ".fastq"
        file_loc_1[-1] = re.sub('.fastq$', new_suffix, file_loc_1[-1])
        file_loc_1.insert(-1, tag)

        file_loc_2 = fqr.fastq2.split("/")
        new_suffix = "." + str(fqr.output_tag) + "_" + str(fqr.output_file_count) + ".fastq"
        file_loc_2[-1] = re.sub('.fastq$', new_suffix, file_loc_2[-1])

        file_loc_2.insert(-1, tag)
        files_out = [[file_loc_1[-1], file_loc_2[-1]]]

        while fqr.eof(1) is False and fqr.eof(2) is False:
            r1_id = record1["id"].split(" ")
            r2_id = record2["id"].split(" ")

            if r1_id[0] == r2_id[0]:
                fqr.writeOutput(record1, 1)
                fqr.writeOutput(record2, 2)

                record1 = fqr.next(1)
                record2 = fqr.next(2)

                count_r1 += 1
                count_r2 += 1
                count_r3 += 1
            elif r1_id[0] < r2_id[0]:
                record1 = fqr.next(1)
                count_r1 += 1
            else:
                record2 = fqr.next(2)
                count_r2 += 1

            if count_r3 % 1000000 == 0:
                fqr.incrementOutputFiles()
                file_loc_1 = fqr.fastq1.split("/")
                new_suffix = "." + str(fqr.output_tag) + "_" + str(fqr.output_file_count) + ".fastq"
                file_loc_1[-1] = re.sub('.fastq$', new_suffix, file_loc_1[-1])
                file_loc_1.insert(-1, "tmp")

                file_loc_2 = fqr.fastq2.split("/")
                new_suffix = "." + str(fqr.output_tag) + "_" + str(fqr.output_file_count) + ".fastq"
                file_loc_2[-1] = re.sub('.fastq$', new_suffix, file_loc_2[-1])
                file_loc_2.insert(-1, "tmp")

                files_out.append([file_loc_1[-1], file_loc_2[-1]])

        fqr.closeFastQ()
        fqr.closeOutputFiles()

        output_file_pregz = out_file.replace('.tar.gz', '.tar')

        if os.path.isfile(out_file):
            os.remove(out_file)
        tar = tarfile.open(output_file_pregz, "w")
        tar.add("/".join(file_loc_1[:-1]), arcname='tmp')
        tar.close()

        command_line = 'pigz ' + output_file_pregz
        args = shlex.split(command_line)
        process = subprocess.Popen(args)
        process.wait()

        return files_out

    def run(self, input_files, input_metadata, output_files):
        """
        The main function to run the splitting of FASTQ files (single or paired)
        so that they can aligned in a distributed manner

        Parameters
        ----------
        input_files : dict
            List of input fastq file locations
        metadata : dict
        output_files : dict

        Returns
        -------
        output_file : str
            Location of compressed (.tar.gz) of the split FASTQ files
        output_names : list
            List of file names in the compressed file

        """

        sources = [input_files["fastq1"]]

        if "fastq2" in input_files:
            sources.append(input_files["fastq2"])
            results = self.paired_splitter(
                input_files["fastq1"], input_files["fastq2"],
                input_files["fastq1"] + ".tar.gz"
            )
        else:
            results = self.single_splitter(
                input_files["fastq1"],
                input_files["fastq1"] + ".tar.gz",
            )

        results = compss_wait_on(results)

        logger.info("FASTQ SPLITTER:", results)

        fastq_tar_meta = Metadata(
            data_type=input_metadata["fastq1"].data_type,
            file_type="TAR",
            file_path=output_files["output"],
            sources=sources,
            taxon_id=input_metadata["fastq1"].taxon_id,
            meta_data={
                "tool": "fastq_splitter"
            }
        )

        return (
            {"output": input_files["fastq1"] + ".tar.gz"},
            {"output": fastq_tar_meta}
        )

# ------------------------------------------------------------------------------
<commit_msg>Added a parameter to control the chuncking size for the fastq splitting<commit_after>"""
.. See the NOTICE file distributed with this work for additional information
   regarding copyright ownership.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
"""

from __future__ import print_function

import os
import shlex
import subprocess
import sys
import re
import tarfile

from utils import logger

try:
    if hasattr(sys, '_run_from_cmdl') is True:
        raise ImportError
    from pycompss.api.parameter import FILE_IN, FILE_OUT, IN, OUT
    from pycompss.api.task import task
    from pycompss.api.api import compss_wait_on
except ImportError:
    logger.warn("[Warning] Cannot import \"pycompss\" API packages.")
    logger.warn("          Using mock decorators.")

    from utils.dummy_pycompss import FILE_IN, FILE_OUT, IN, OUT # pylint: disable=ungrouped-imports
    from utils.dummy_pycompss import task # pylint: disable=ungrouped-imports
    from utils.dummy_pycompss import compss_wait_on # pylint: disable=ungrouped-imports

from basic_modules.tool import Tool
from basic_modules.metadata import Metadata

from tool.fastqreader import fastqreader

# ------------------------------------------------------------------------------

class fastq_splitter(Tool):
    """
    Script for splitting up FASTQ files into manageable chunks
    """

    def __init__(self, configuration=None):
        """
        Initialise the tool with its configuration.

        Parameters
        ----------
        configuration : dict
            a dictionary containing parameters that define how the operation
            should be carried out, which are specific to each Tool.
        """
        logger.info("FASTQ Splitter")
        Tool.__init__(self)

        if configuration is None:
            configuration = {}

        self.configuration.update(configuration)

    @task(
        in_file1=FILE_IN, tag=IN,
        out_file=FILE_OUT, files_out=OUT,
        returns=list)
    def single_splitter(self, in_file1, out_file, tag='tmp'):  # pylint disable=no-self-use
        """
        Function to divide the FastQ files into separate sub files of 1000000
        sequences so that the aligner can run in parallel.

        Parameters
        ----------
        in_file1 : str
            Location of first FASTQ file
        tag : str
            DEFAULT = tmp
            Tag used to identify the files. Useful if this is getting run
            manually on a single machine multiple times to prevent collisions of
            file names

        Returns
        -------
        Returns: Returns a list of the files that have been generated.
                 Each sub list containing the two paired end files for that
                 subset.
        paired_files : list
            List of lists of pair end files. Each sub list containing the two
            paired end files for that subset.
        """

        fqr = fastqreader()
        fqr.openFastQ(in_file1)
        fqr.createOutputFiles(tag)

        fastq_chunk_size = 1000000
        if "fastq_chunk_size" in self.configuration:
            fastq_chunk_size = self.configuration["fastq_chunk_size"]

        record1 = fqr.next(1)

        count_r3 = 0

        file_loc_1 = fqr.fastq1.split("/")
        file_loc_1[-1] = file_loc_1[-1].replace(
            ".fastq",
            "." + str(fqr.output_tag) + "_" + str(fqr.output_file_count) + ".fastq")
        file_loc_1.insert(-1, tag)

        files_out = [[file_loc_1[-1]]]

        while fqr.eof(1) is False:
            fqr.writeOutput(record1, 1)
            record1 = fqr.next(1)
            count_r3 += 1

            if count_r3 % fastq_chunk_size == 0:
                fqr.incrementOutputFiles()
                file_loc_1 = fqr.fastq1.split("/")
                new_suffix = "." + str(fqr.output_tag) + "_" + str(fqr.output_file_count) + ".fastq"
                file_loc_1[-1] = re.sub('.fastq$', new_suffix, file_loc_1[-1])
                file_loc_1.insert(-1, tag)

                files_out.append([file_loc_1[-1]])

        fqr.closeFastQ()
        fqr.closeOutputFiles()

        output_file_pregz = out_file.replace('.tar.gz', '.tar')

        if os.path.isfile(out_file):
            os.remove(out_file)
        tar = tarfile.open(output_file_pregz, "w")
        tar.add("/".join(file_loc_1[:-1]), arcname='tmp')
        tar.close()

        command_line = 'pigz ' + output_file_pregz
        args = shlex.split(command_line)
        process = subprocess.Popen(args)
        process.wait()

        return files_out

    @task(
        in_file1=FILE_IN, in_file2=FILE_IN, tag=IN,
        out_file=FILE_OUT, files_out=OUT,
        returns=list)
    def paired_splitter(self, in_file1, in_file2, out_file, tag='tmp'):  # pylint disable=no-self-use
        """
        Function to divide the FastQ files into separte sub files of 1000000
        sequences so that the aligner can run in parallel.

        Parameters
        ----------
        in_file1 : str
            Location of first paired end FASTQ file
        in_file2 : str
            Location of second paired end FASTQ file
        tag : str
            DEFAULT = tmp
            Tag used to identify the files. Useful if this is getting run
            manually on a single machine multiple times to prevent collisions of
            file names

        Returns
        -------
        Returns: Returns a list of lists of the files that have been generated.
                 Each sub list containing the two paired end files for that
                 subset.
        paired_files : list
            List of lists of pair end files. Each sub list containing the two
            paired end files for that subset.
        """

        fqr = fastqreader()
        fqr.openFastQ(in_file1, in_file2)
        fqr.createOutputFiles(tag)

        fastq_chunk_size = 1000000
        if "fastq_chunk_size" in self.configuration:
            fastq_chunk_size = self.configuration["fastq_chunk_size"]

        record1 = fqr.next(1)
        record2 = fqr.next(2)

        count_r1 = 0
        count_r2 = 0
        count_r3 = 0

        file_loc_1 = fqr.fastq1.split("/")
        new_suffix = "." + str(fqr.output_tag) + "_" + str(fqr.output_file_count) + ".fastq"
        file_loc_1[-1] = re.sub('.fastq$', new_suffix, file_loc_1[-1])
        file_loc_1.insert(-1, tag)

        file_loc_2 = fqr.fastq2.split("/")
        new_suffix = "." + str(fqr.output_tag) + "_" + str(fqr.output_file_count) + ".fastq"
        file_loc_2[-1] = re.sub('.fastq$', new_suffix, file_loc_2[-1])

        file_loc_2.insert(-1, tag)
        files_out = [[file_loc_1[-1], file_loc_2[-1]]]

        while fqr.eof(1) is False and fqr.eof(2) is False:
            r1_id = record1["id"].split(" ")
            r2_id = record2["id"].split(" ")

            if r1_id[0] == r2_id[0]:
                fqr.writeOutput(record1, 1)
                fqr.writeOutput(record2, 2)

                record1 = fqr.next(1)
                record2 = fqr.next(2)

                count_r1 += 1
                count_r2 += 1
                count_r3 += 1
            elif r1_id[0] < r2_id[0]:
                record1 = fqr.next(1)
                count_r1 += 1
            else:
                record2 = fqr.next(2)
                count_r2 += 1

            if count_r3 % fastq_chunk_size == 0:
                fqr.incrementOutputFiles()
                file_loc_1 = fqr.fastq1.split("/")
                new_suffix = "." + str(fqr.output_tag) + "_" + str(fqr.output_file_count) + ".fastq"
                file_loc_1[-1] = re.sub('.fastq$', new_suffix, file_loc_1[-1])
                file_loc_1.insert(-1, "tmp")

                file_loc_2 = fqr.fastq2.split("/")
                new_suffix = "." + str(fqr.output_tag) + "_" + str(fqr.output_file_count) + ".fastq"
                file_loc_2[-1] = re.sub('.fastq$', new_suffix, file_loc_2[-1])
                file_loc_2.insert(-1, "tmp")

                files_out.append([file_loc_1[-1], file_loc_2[-1]])

        fqr.closeFastQ()
        fqr.closeOutputFiles()

        output_file_pregz = out_file.replace('.tar.gz', '.tar')

        if os.path.isfile(out_file):
            os.remove(out_file)
        tar = tarfile.open(output_file_pregz, "w")
        tar.add("/".join(file_loc_1[:-1]), arcname='tmp')
        tar.close()

        command_line = 'pigz ' + output_file_pregz
        args = shlex.split(command_line)
        process = subprocess.Popen(args)
        process.wait()

        return files_out

    def run(self, input_files, input_metadata, output_files):
        """
        The main function to run the splitting of FASTQ files (single or paired)
        so that they can aligned in a distributed manner

        Parameters
        ----------
        input_files : dict
            List of input fastq file locations
        metadata : dict
        output_files : dict

        Returns
        -------
        output_file : str
            Location of compressed (.tar.gz) of the split FASTQ files
        output_names : list
            List of file names in the compressed file

        """

        sources = [input_files["fastq1"]]

        if "fastq2" in input_files:
            sources.append(input_files["fastq2"])
            results = self.paired_splitter(
                input_files["fastq1"], input_files["fastq2"],
                input_files["fastq1"] + ".tar.gz"
            )
        else:
            results = self.single_splitter(
                input_files["fastq1"],
                input_files["fastq1"] + ".tar.gz",
            )

        results = compss_wait_on(results)

        logger.info("FASTQ SPLITTER:", results)

        fastq_tar_meta = Metadata(
            data_type=input_metadata["fastq1"].data_type,
            file_type="TAR",
            file_path=output_files["output"],
            sources=sources,
            taxon_id=input_metadata["fastq1"].taxon_id,
            meta_data={
                "tool": "fastq_splitter"
            }
        )

        return (
            {"output": input_files["fastq1"] + ".tar.gz"},
            {"output": fastq_tar_meta}
        )

# ------------------------------------------------------------------------------
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import os.path

from nimp.utilities.perforce  import *
from nimp.utilities.processes import *
from nimp.utilities.file_set  import *
import shutil

def wwise_cache_platform(platform):
    return {
        "pc"            : "Windows",
        "pcconsole"     : "Windows",
        "win32"         : "Windows",
        "win64"         : "Windows",
        "windows"       : "Windows",
        "ps3"           : "Ps3",
        "xbox 360"      : "Xbox360",
        "xbox360"       : "Xbox360",
        "x360"          : "Xbox360",
        "dingo"         : "XboxOne",
        "xboxone"       : "XboxOne",
        "orbis"         : "PS4",
        "ps4"           : "ps4"}[platform.lower()]

def wwise_banks_platform(platform):
    return {
        "pc"            : "PC",
        "pcconsole"     : "PC",
        "win32"         : "PC",
        "win64"         : "PC",
        "windows"       : "PC",
        "ps3"           : "Ps3",
        "xbox 360"      : "X360",
        "xbox360"       : "X360",
        "x360"          : "X360",
        "dingo"         : "X360",
        "xboxone"       : "X360",
        "orbis"         : "PS4",
        "ps4"           : "ps4"}[platform.lower()]

def wwise_cmd_line_platform(platform):
    return {
        "pc"            : "Windows",
        "pcconsole"     : "Windows",
        "win32"         : "Windows",
        "win64"         : "Windows",
        "windows"       : "Windows",
        "ps3"           : "Ps3",
        "xbox 360"      : "XBox360",
        "xbox360"       : "XBox360",
        "x360"          : "XBox360",
        "dingo"         : "XBox360",
        "xboxone"       : "XBox360",
        "orbis"         : "PS4",
        "ps4"           : "ps4"}[platform.lower()]

def build_wwise_banks(context):
    platform         = context.platform
    wwise_banks_path = context.wwise_banks_path
    wwise_banks_path = os.path.join(wwise_banks_path, wwise_banks_platform(platform))
    cl_name          = "[CIS] Updated {0} Wwise Banks".format(platform)
    wwise_cli_path   = os.path.join(os.getenv('WWISEROOT'), "Authoring\\x64\\Release\\bin\\WWiseCLI.exe")
    wwise_command    = [wwise_cli_path,  os.path.abspath(context.wwise_project), "-GenerateSoundBanks", "-Platform", wwise_cmd_line_platform(platform)]

    log_notification("Updating local Wwise cache...")
    _copy_wwise_cache(context, context.shared_cache_path, context.local_cache_path)

    result      = True
    with p4_transaction(cl_name, submit_on_success = context.checkin) as trans:
        log_notification("Checking out bank files...")
        add_to_p4(context, trans, wwise_banks_path)
        if call_process(".", wwise_command) == 1:
            result = False
            log_error("Error while running WwiseCli...")
            trans.abort()
        else:
            log_notification("Adding bank files to perforce...")
            add_to_p4(context, trans, wwise_banks_path)
            log_notification("Updating shared Wwise cache...")
            _copy_wwise_cache(context, context.local_cache_path, context.shared_cache_path)

    return result

def _copy_wwise_cache(context, source, destination):
    wwise_cache = copy_files(context).frm(source).newer().to(destination)
    wwise_cache = wwise_cache.format(platform = wwise_cache_platform(context.platform))
    log_notification("Listing cache files...")
    wwise_cache = wwise_cache.recursive().add('*')
    log_notification("Copying needed files...")
    wwise_cache.process(copy_mkdest_dir)<commit_msg>Fixed wwise platform name<commit_after># -*- coding: utf-8 -*-

import os.path

from nimp.utilities.perforce  import *
from nimp.utilities.processes import *
from nimp.utilities.file_set  import *
import shutil

def wwise_cache_platform(platform):
    return {
        "pc"            : "Windows",
        "pcconsole"     : "Windows",
        "win32"         : "Windows",
        "win64"         : "Windows",
        "windows"       : "Windows",
        "ps3"           : "Ps3",
        "xbox 360"      : "XBox360",
        "xbox360"       : "XBox360",
        "x360"          : "XBox360",
        "dingo"         : "XboxOne",
        "xboxone"       : "XboxOne",
        "orbis"         : "PS4",
        "ps4"           : "ps4"}[platform.lower()]

def wwise_banks_platform(platform):
    return {
        "pc"            : "PC",
        "pcconsole"     : "PC",
        "win32"         : "PC",
        "win64"         : "PC",
        "windows"       : "PC",
        "ps3"           : "Ps3",
        "xbox 360"      : "X360",
        "xbox360"       : "X360",
        "x360"          : "X360",
        "dingo"         : "X360",
        "xboxone"       : "X360",
        "orbis"         : "PS4",
        "ps4"           : "ps4"}[platform.lower()]

def wwise_cmd_line_platform(platform):
    return {
        "pc"            : "Windows",
        "pcconsole"     : "Windows",
        "win32"         : "Windows",
        "win64"         : "Windows",
        "windows"       : "Windows",
        "ps3"           : "Ps3",
        "xbox 360"      : "XBox360",
        "xbox360"       : "XBox360",
        "x360"          : "XBox360",
        "dingo"         : "XboxOne",
        "xboxone"       : "XboxOne",
        "orbis"         : "PS4",
        "ps4"           : "ps4"}[platform.lower()]

def build_wwise_banks(context):
    platform         = context.platform
    wwise_banks_path = context.wwise_banks_path
    wwise_banks_path = os.path.join(wwise_banks_path, wwise_banks_platform(platform))
    cl_name          = "[CIS] Updated {0} Wwise Banks".format(platform)
    wwise_cli_path   = os.path.join(os.getenv('WWISEROOT'), "Authoring\\x64\\Release\\bin\\WWiseCLI.exe")
    wwise_command    = [wwise_cli_path,  os.path.abspath(context.wwise_project), "-GenerateSoundBanks", "-Platform", wwise_cmd_line_platform(platform)]

    log_notification("Updating local Wwise cache...")
    _copy_wwise_cache(context, context.shared_cache_path, context.local_cache_path)

    result      = True
    with p4_transaction(cl_name, submit_on_success = context.checkin) as trans:
        log_notification("Checking out bank files...")
        add_to_p4(context, trans, wwise_banks_path)
        if call_process(".", wwise_command) == 1:
            result = False
            log_error("Error while running WwiseCli...")
            trans.abort()
        else:
            log_notification("Adding bank files to perforce...")
            add_to_p4(context, trans, wwise_banks_path)
            log_notification("Updating shared Wwise cache...")
            _copy_wwise_cache(context, context.local_cache_path, context.shared_cache_path)

    return result

def _copy_wwise_cache(context, source, destination):
    wwise_cache = copy_files(context).frm(source).newer().to(destination)
    wwise_cache = wwise_cache.format(platform = wwise_cache_platform(context.platform))
    log_notification("Listing cache files...")
    wwise_cache = wwise_cache.recursive().add('*')
    log_notification("Copying needed files...")
    wwise_cache.process(copy_mkdest_dir)<|endoftext|>


<commit_before>"""py.test configuration"""
import os
from pathlib import Path
import numpy
import pytest

from .utils.bids import collect_data

test_data_env = os.getenv('TEST_DATA_HOME',
                          str(Path.home() / '.cache' / 'stanford-crn'))
data_dir = Path(test_data_env) / 'BIDS-examples-1-enh-ds054'

@pytest.fixture(autouse=True)
def add_np(doctest_namespace):
    doctest_namespace['np'] = numpy
    doctest_namespace['os'] = os
    doctest_namespace['Path'] = Path
    doctest_namespace['datadir'] = data_dir
    doctest_namespace['bids_collect_data'] = collect_data

@pytest.fixture
def testdata_dir():
    return data_dir
<commit_msg>DOCTEST: Make nifti_fname available to doctests<commit_after>"""py.test configuration"""
import os
from pathlib import Path
import numpy as np
import nibabel as nb
import pytest
import tempfile

from .utils.bids import collect_data

test_data_env = os.getenv('TEST_DATA_HOME',
                          str(Path.home() / '.cache' / 'stanford-crn'))
data_dir = Path(test_data_env) / 'BIDS-examples-1-enh-ds054'

@pytest.fixture(autouse=True)
def add_np(doctest_namespace):
    doctest_namespace['np'] = np
    doctest_namespace['os'] = os
    doctest_namespace['Path'] = Path
    doctest_namespace['datadir'] = data_dir
    doctest_namespace['bids_collect_data'] = collect_data

    tmpdir = tempfile.TemporaryDirectory()
    nifti_fname = str(Path(tmpdir.name) / 'test.nii.gz')
    nb.Nifti1Image(np.random.random((5, 5)).astype('f4'), np.eye(4)).to_filename(nifti_fname)
    doctest_namespace['nifti_fname'] = nifti_fname
    yield
    tmpdir.cleanup()

@pytest.fixture
def testdata_dir():
    return data_dir
<|endoftext|>


<commit_before>import argparse
import calendar
from datetime import datetime
import glob
import os
import shutil
import subprocess
import sys
import gzip
import unifyQueryTypes
from utility import utility
import config

months = {'january': [1, 31],
          'february': [2, 28],
          'march': [3, 31],
          'april': [4, 30],
          'may': [5, 31],
          'june': [6, 30],
          'july': [7, 30],
          'august': [8, 31],
          'september': [9, 30],
          'october': [10, 31],
          'november': [11, 30],
          'december': [12, 31]}

parser = argparse.ArgumentParser("This script extracts the raw log data (if "
                                 + "it was not already done), processes them"
                                 + " using the java application and unifies "
                                 + "the query types.")
parser.add_argument("--ignoreLock", "-i", help="Ignore locked file and "
                    + "execute anyways", action="store_true")
parser.add_argument("--threads", "-t", default=5, type=int, help="The number "
                    + "of threads to run the java program with (default 5).")
parser.add_argument("--logging", "-l", help="Enables file logging.",
                    action="store_true")
parser.add_argument("--noBotMetrics", "-b", help="Disables metric calculation"
                    + " for bot queries.", action="store_true")
parser.add_argument("--noDynamicQueryTypes", "-d", help="Disables dynamic "
                    + "generation of query types.", action="store_true")
parser.add_argument("--noGzipOutput", "-g", help="Disables gzipping of the "
                    + "output files.", action="store_true")
parser.add_argument("--noExampleQueriesOutput", "-e", help="Disables the "
                    + "matching of example queries.", action="store_true")
parser.add_argument("--referenceDirectory", "-r",
                    default=config.queryReferenceDirectory,
                    type=str,
                    help="The directory with the reference query types.")
parser.add_argument("--fdupesExecutable", "-f",
                    default=config.fdupesExecutable, type=str,
                    help="The location of the fdupes executable.")
parser.add_argument("--monthsFolder", "-m", default=config.monthsFolder,
                    type=str,
                    help="The folder in which the months directory are "
                    + "residing.")
parser.add_argument("--year", "-y", default=datetime.now().year, type=int,
                    help="The year to be processed (default current year).")
parser.add_argument("months", type=str, help="The months to be processed")

# These are the field we extract from wmf.wdqs_extract that form the raw
# log data. They are not configurable via argument because the java program
# does not detect headers and thus depends on this specific order.
fields = ["uri_query", "uri_path", "user_agent", "ts", "agent_type",
          "hour", "http_status"]

header = ""
for field in fields:
    header += field + "\t"
    header = header[:-1] + "\n"

if (len(sys.argv[1:]) == 0):
    parser.print_help()
    parser.exit()

args = parser.parse_args()

if calendar.isleap(args.year):
    months['february'][1] = 29

for monthName in args.months.split(","):
    if os.path.isfile(utility.addMissingSlash(args.monthsFolder)
                      + utility.addMissingSlash(monthName) + "locked") \
       and not args.ignoreLock:
        print "ERROR: The month " + monthName + " is being edited at the "
        + "moment. Use -i if you want to force the execution of this script."
        sys.exit()

    month = os.path.abspath(utility.addMissingSlash(args.monthsFolder)
                            + utility.addMissingSlash(monthName))

    processedLogDataDirectory = month + "processedLogData/"
    rawLogDataDirectory = month + "rawLogData/"
    tempDirectory = rawLogDataDirectory + "temp/"

    # If the month directory does not exist it is being created along with
    # the directories for raw and processed log data.
    if not os.path.exists(month):
        print("Starting data extraction from wmf.wdqs_extract for "
              + monthName + ".")

        os.makedirs(month)
        os.makedirs(processedLogDataDirectory)
        os.makedirs(rawLogDataDirectory)

        arguments = ['hive', '-e']

        # For each day we send a command to hive that extracts all entries for
        # this day (in the given month and year) and writes them to temporary
        # files.
        for day in xrange(1, months[monthName][1] + 1):
            os.makedirs(tempDirectory)
            hive_call = 'insert overwrite local directory \'' + tempDirectory \
                    + '\' row format delimited fields terminated " \
                    + "by \'\\t\' select '

            # We add all the fields to the request
            for field in fields:
                hive_call += field + " "

            hive_call += ' from wmf.wdqs_extract where uri_query<>"" ' \
                    + 'and year=\'' + str(args.year) +  '\' and month=\'' \
                    + str(months[monthName][0]) + '\' and day=\'' + str(day) + '\''

            arguments.append(hive_call)
            if subprocess.call(arguments) != 0:
                print("ERROR: Raw data for month " + monthName + " does not "
                      + "exist but could not be extracted using hive.")
                sys.exit(1)

            # The content of the temporary files is then copied to the actual
            # raw log data file (with added headers)
            with gzip.open(rawLogDataDirectory + "QueryCnt"
                           + day + ".tsv.gz", "wb") as dayfile:
                dayfile.write(header)

                for filename in glob.glob('*'):
                    with open(filename) as temp:
                        for line in temp:
                            dayfile.write(line)

            shutil.rmtree(tempDirectory)

    # We build the call to execute the java application with the location of
    # the files, the number of threads to use and any optional arguments needed

    mavenCall = ['mvn', 'exec:java']

    mavenArguments = '-Dexec.args=-w ' + month + ' -n ' + str(args.threads)

    if args.logging:
        mavenArguments += " -l"
        if args.noBotMetrics:
            mavenArguments += " -b"
            if args.noDynamicQueryTypes:
                mavenArguments += " -d"
                if args.noGzipOutput:
                    mavenArguments += " -g"
                    if args.noExampleQueriesOutput:
                        mavenArguments += " -e"

    mavenCall.append(mavenArguments)

    owd = os.getcwd()
    os.chdir("..")

    print "Starting data processing using QueryAnalysis for " + monthName + "."

    if subprocess.call(['mvn', 'clean', 'package']) != 0:
        print "ERROR: Could not package the java application."
        sys.exit(1)

    if subprocess.call(mavenCall) != 0:
        print("ERROR: Could not execute the java application. Check the logs "
              + "for details or rerun this script with -l to generate logs.")
        sys.exit(1)

    os.chdir(owd)

    # Finally we call the query type unification
    # (see python unifyQueryTypes.py for details)

    print "Starting to unify the query types for " + monthName + "."

    unifyQueryTypes.unifyQueryTypes(monthName,
                                    args.monthsFolder,
                                    args.referenceDirectory,
                                    args.fdupesExecutable)
<commit_msg>FIX: QueryAnalysis not creating subdirectories<commit_after>import argparse
import calendar
from datetime import datetime
import glob
import os
import shutil
import subprocess
import sys
import gzip
import unifyQueryTypes
from utility import utility
import config

months = {'january': [1, 31],
          'february': [2, 28],
          'march': [3, 31],
          'april': [4, 30],
          'may': [5, 31],
          'june': [6, 30],
          'july': [7, 30],
          'august': [8, 31],
          'september': [9, 30],
          'october': [10, 31],
          'november': [11, 30],
          'december': [12, 31]}

parser = argparse.ArgumentParser("This script extracts the raw log data (if "
                                 + "it was not already done), processes them"
                                 + " using the java application and unifies "
                                 + "the query types.")
parser.add_argument("--ignoreLock", "-i", help="Ignore locked file and "
                    + "execute anyways", action="store_true")
parser.add_argument("--threads", "-t", default=5, type=int, help="The number "
                    + "of threads to run the java program with (default 5).")
parser.add_argument("--logging", "-l", help="Enables file logging.",
                    action="store_true")
parser.add_argument("--noBotMetrics", "-b", help="Disables metric calculation"
                    + " for bot queries.", action="store_true")
parser.add_argument("--noDynamicQueryTypes", "-d", help="Disables dynamic "
                    + "generation of query types.", action="store_true")
parser.add_argument("--noGzipOutput", "-g", help="Disables gzipping of the "
                    + "output files.", action="store_true")
parser.add_argument("--noExampleQueriesOutput", "-e", help="Disables the "
                    + "matching of example queries.", action="store_true")
parser.add_argument("--referenceDirectory", "-r",
                    default=config.queryReferenceDirectory,
                    type=str,
                    help="The directory with the reference query types.")
parser.add_argument("--fdupesExecutable", "-f",
                    default=config.fdupesExecutable, type=str,
                    help="The location of the fdupes executable.")
parser.add_argument("--monthsFolder", "-m", default=config.monthsFolder,
                    type=str,
                    help="The folder in which the months directory are "
                    + "residing.")
parser.add_argument("--year", "-y", default=datetime.now().year, type=int,
                    help="The year to be processed (default current year).")
parser.add_argument("months", type=str, help="The months to be processed")

# These are the field we extract from wmf.wdqs_extract that form the raw
# log data. They are not configurable via argument because the java program
# does not detect headers and thus depends on this specific order.
fields = ["uri_query", "uri_path", "user_agent", "ts", "agent_type",
          "hour", "http_status"]

header = ""
for field in fields:
    header += field + "\t"
    header = header[:-1] + "\n"

if (len(sys.argv[1:]) == 0):
    parser.print_help()
    parser.exit()

args = parser.parse_args()

if calendar.isleap(args.year):
    months['february'][1] = 29

for monthName in args.months.split(","):
    if os.path.isfile(utility.addMissingSlash(args.monthsFolder)
                      + utility.addMissingSlash(monthName) + "locked") \
       and not args.ignoreLock:
        print "ERROR: The month " + monthName + " is being edited at the " + "moment. Use -i if you want to force the execution of this script."
        sys.exit()

    month = utility.addMissingSlash(os.path.abspath(utility.addMissingSlash(args.monthsFolder)
                            + utility.addMissingSlash(monthName)))

    processedLogDataDirectory = month + "processedLogData/"
    rawLogDataDirectory = month + "rawLogData/"
    tempDirectory = rawLogDataDirectory + "temp/"

    # If the month directory does not exist it is being created along with
    # the directories for raw and processed log data.
    if not os.path.exists(month):
        print("Starting data extraction from wmf.wdqs_extract for "
              + monthName + ".")

        os.makedirs(month)
        os.makedirs(processedLogDataDirectory)
        os.makedirs(rawLogDataDirectory)

        arguments = ['hive', '-e']

        # For each day we send a command to hive that extracts all entries for
        # this day (in the given month and year) and writes them to temporary
        # files.
        for day in xrange(1, months[monthName][1] + 1):
            os.makedirs(tempDirectory)
            hive_call = 'insert overwrite local directory \'' + tempDirectory \
                    + '\' row format delimited fields terminated " \
                    + "by \'\\t\' select '

            # We add all the fields to the request
            for field in fields:
                hive_call += field + " "

            hive_call += ' from wmf.wdqs_extract where uri_query<>"" ' \
                    + 'and year=\'' + str(args.year) +  '\' and month=\'' \
                    + str(months[monthName][0]) + '\' and day=\'' + str(day) + '\''

            arguments.append(hive_call)
            if subprocess.call(arguments) != 0:
                print("ERROR: Raw data for month " + monthName + " does not "
                      + "exist but could not be extracted using hive.")
                sys.exit(1)

            # The content of the temporary files is then copied to the actual
            # raw log data file (with added headers)
            with gzip.open(rawLogDataDirectory + "QueryCnt"
                           + day + ".tsv.gz", "wb") as dayfile:
                dayfile.write(header)

                for filename in glob.glob('*'):
                    with open(filename) as temp:
                        for line in temp:
                            dayfile.write(line)

            shutil.rmtree(tempDirectory)

    # We build the call to execute the java application with the location of
    # the files, the number of threads to use and any optional arguments needed

    mavenCall = ['mvn', 'exec:java']

    mavenArguments = '-Dexec.args=-w ' + month + ' -n ' + str(args.threads)

    if args.logging:
        mavenArguments += " -l"
        if args.noBotMetrics:
            mavenArguments += " -b"
            if args.noDynamicQueryTypes:
                mavenArguments += " -d"
                if args.noGzipOutput:
                    mavenArguments += " -g"
                    if args.noExampleQueriesOutput:
                        mavenArguments += " -e"

    mavenCall.append(mavenArguments)

    owd = os.getcwd()
    os.chdir("..")

    print "Starting data processing using QueryAnalysis for " + monthName + "."

    if subprocess.call(['mvn', 'clean', 'package']) != 0:
        print "ERROR: Could not package the java application."
        sys.exit(1)

    if subprocess.call(mavenCall) != 0:
        print("ERROR: Could not execute the java application. Check the logs "
              + "for details or rerun this script with -l to generate logs.")
        sys.exit(1)

    os.chdir(owd)

    # Finally we call the query type unification
    # (see python unifyQueryTypes.py for details)

    print "Starting to unify the query types for " + monthName + "."

    unifyQueryTypes.unifyQueryTypes(monthName,
                                    args.monthsFolder,
                                    args.referenceDirectory,
                                    args.fdupesExecutable)
<|endoftext|>


<commit_before>#!/usr/bin/python
# Copyright (c) 2010 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Snapshot Build Bisect Tool

This script bisects a snapshot archive using binary search. It starts at
a bad revision (it will try to guess HEAD) and asks for a last known-good
revision. It will then binary search across this revision range by downloading,
unzipping, and opening Chromium for you. After testing the specific revision,
it will ask you whether it is good or bad before continuing the search.
"""

# Base URL to download snapshots from.
BUILD_BASE_URL = 'http://build.chromium.org/f/chromium/snapshots/'

# The type (platform) of the build archive. This is what's passed in to the
# '-a/--archive' option.
BUILD_ARCHIVE_TYPE = ''

# The selected archive to bisect.
BUILD_ARCHIVE_DIR = ''

# The location of the builds.
BUILD_ARCHIVE_URL = '/%d/'

# Name of the build archive.
BUILD_ZIP_NAME = ''

# Directory name inside the archive.
BUILD_DIR_NAME = ''

# Name of the executable.
BUILD_EXE_NAME = ''

# URL to the ViewVC commit page.
BUILD_VIEWVC_URL = 'http://src.chromium.org/viewvc/chrome?view=rev&revision=%d'

# Changelogs URL
CHANGELOG_URL = 'http://build.chromium.org/f/chromium/' \
                'perf/dashboard/ui/changelog.html?url=/trunk/src&range=%d:%d'

###############################################################################

import math
import optparse
import os
import pipes
import re
import shutil
import sys
import tempfile
import urllib
import zipfile

def UnzipFilenameToDir(filename, dir):
  """Unzip |filename| to directory |dir|."""
  zf = zipfile.ZipFile(filename)
  # Make base.
  pushd = os.getcwd()
  try:
    if not os.path.isdir(dir):
      os.mkdir(dir)
    os.chdir(dir)
    # Extract files.
    for info in zf.infolist():
      name = info.filename
      if name.endswith('/'):  # dir
        if not os.path.isdir(name):
          os.makedirs(name)
      else:  # file
        dir = os.path.dirname(name)
        if not os.path.isdir(dir):
          os.makedirs(dir)
        out = open(name, 'wb')
        out.write(zf.read(name))
        out.close()
      # Set permissions. Permission info in external_attr is shifted 16 bits.
      os.chmod(name, info.external_attr >> 16L)
    os.chdir(pushd)
  except Exception, e:
    print >>sys.stderr, e
    sys.exit(1)

def SetArchiveVars(archive):
  """Set a bunch of global variables appropriate for the specified archive."""
  global BUILD_ARCHIVE_TYPE
  global BUILD_ARCHIVE_DIR
  global BUILD_ZIP_NAME
  global BUILD_DIR_NAME
  global BUILD_EXE_NAME
  global BUILD_BASE_URL

  BUILD_ARCHIVE_TYPE = archive
  BUILD_ARCHIVE_DIR = 'chromium-rel-' + BUILD_ARCHIVE_TYPE

  if BUILD_ARCHIVE_TYPE in ('linux', 'linux-64', 'linux-chromiumos'):
    BUILD_ZIP_NAME = 'chrome-linux.zip'
    BUILD_DIR_NAME = 'chrome-linux'
    BUILD_EXE_NAME = 'chrome'
  elif BUILD_ARCHIVE_TYPE in ('mac'):
    BUILD_ZIP_NAME = 'chrome-mac.zip'
    BUILD_DIR_NAME = 'chrome-mac'
    BUILD_EXE_NAME = 'Chromium.app/Contents/MacOS/Chromium'
  elif BUILD_ARCHIVE_TYPE in ('xp'):
    BUILD_ZIP_NAME = 'chrome-win32.zip'
    BUILD_DIR_NAME = 'chrome-win32'
    BUILD_EXE_NAME = 'chrome.exe'

  BUILD_BASE_URL += BUILD_ARCHIVE_DIR

def ParseDirectoryIndex(url):
  """Parses the HTML directory listing into a list of revision numbers."""
  handle = urllib.urlopen(url)
  dirindex = handle.read()
  handle.close()
  return re.findall(r'\1/', dirindex)

def GetRevList(good, bad):
  """Gets the list of revision numbers between |good| and |bad|."""
  # Download the main revlist.
  revlist = ParseDirectoryIndex(BUILD_BASE_URL)
  revlist = map(int, revlist)
  revlist = filter(lambda r: range(good, bad).__contains__(int(r)), revlist)
  revlist.sort()
  return revlist

def TryRevision(rev, profile, args):
  """Downloads revision |rev|, unzips it, and opens it for the user to test.
  |profile| is the profile to use."""
  # Do this in a temp dir so we don't collide with user files.
  cwd = os.getcwd()
  tempdir = tempfile.mkdtemp(prefix='bisect_tmp')
  os.chdir(tempdir)

  # Download the file.
  download_url = BUILD_BASE_URL + (BUILD_ARCHIVE_URL % rev) + BUILD_ZIP_NAME
  def _Reporthook(blocknum, blocksize, totalsize):
    size = blocknum * blocksize
    if totalsize == -1:  # Total size not known.
      progress = "Received %d bytes" % size
    else:
      size = min(totalsize, size)
      progress = "Received %d of %d bytes, %.2f%%" % (
          size, totalsize, 100.0 * size / totalsize)
    # Send a \r to let all progress messages use just one line of output.
    sys.stdout.write("\r" + progress)
    sys.stdout.flush()
  try:
    print 'Fetching ' + download_url
    urllib.urlretrieve(download_url, BUILD_ZIP_NAME, _Reporthook)
    print
  except Exception, e:
    print('Could not retrieve the download. Sorry.')
    sys.exit(-1)

  # Unzip the file.
  print 'Unzipping ...'
  UnzipFilenameToDir(BUILD_ZIP_NAME, os.curdir)

  # Tell the system to open the app.
  args = ['--user-data-dir=%s' % profile] + args
  flags = ' '.join(map(pipes.quote, args))
  exe = os.path.join(os.getcwd(), BUILD_DIR_NAME, BUILD_EXE_NAME)
  cmd = '%s %s' % (exe, flags)
  print 'Running %s' % cmd
  os.system(cmd)

  os.chdir(cwd)
  print 'Cleaning temp dir ...'
  try:
    shutil.rmtree(tempdir, True)
  except Exception, e:
    pass

def AskIsGoodBuild(rev):
  """Ask the user whether build |rev| is good or bad."""
  # Loop until we get a response that we can parse.
  while True:
    response = raw_input('\nBuild %d is [(g)ood/(b)ad]: ' % int(rev))
    if response and response in ('g', 'b'):
      return response == 'g'

def main():
  usage = ('%prog [options] [-- chromium-options]\n'
           'Perform binary search on the snapshot builds.\n'
           '\n'
           'Tip: add "-- --no-first-run" to bypass the first run prompts.')
  parser = optparse.OptionParser(usage=usage)
  # Strangely, the default help output doesn't include the choice list.
  choices = ['mac', 'xp', 'linux', 'linux-64', 'linux-chromiumos']
  parser.add_option('-a', '--archive',
                    choices = choices,
                    help = 'The buildbot archive to bisect [%s].' %
                           '|'.join(choices))
  parser.add_option('-b', '--bad', type = 'int',
                    help = 'The bad revision to bisect to.')
  parser.add_option('-g', '--good', type = 'int',
                    help = 'The last known good revision to bisect from.')
  parser.add_option('-p', '--profile', '--user-data-dir', type = 'str',
                    help = 'Profile to use; this will not reset every run. ' +
                    'Defaults to a clean profile.')
  (opts, args) = parser.parse_args()

  if opts.archive is None:
    print 'Error: missing required parameter: --archive'
    print
    parser.print_help()
    return 1

  if opts.bad and opts.good and (opts.good > opts.bad):
    print ('The good revision (%d) must precede the bad revision (%d).\n' %
           (opts.good, opts.bad))
    parser.print_help()
    return 1

  SetArchiveVars(opts.archive)

  # Pick a starting point, try to get HEAD for this.
  if opts.bad:
    bad_rev = opts.bad
  else:
    bad_rev = 0
    try:
      # Location of the latest build revision number
      BUILD_LATEST_URL = '%s/LATEST' % (BUILD_BASE_URL)
      nh = urllib.urlopen(BUILD_LATEST_URL)
      latest = int(nh.read())
      nh.close()
      bad_rev = raw_input('Bad revision [HEAD:%d]: ' % latest)
      if (bad_rev == ''):
        bad_rev = latest
      bad_rev = int(bad_rev)
    except Exception, e:
      print('Could not determine latest revision. This could be bad...')
      bad_rev = int(raw_input('Bad revision: '))

  # Find out when we were good.
  if opts.good:
    good_rev = opts.good
  else:
    good_rev = 0
    try:
      good_rev = int(raw_input('Last known good [0]: '))
    except Exception, e:
      pass

  # Get a list of revisions to bisect across.
  revlist = GetRevList(good_rev, bad_rev)
  if len(revlist) < 2:  # Don't have enough builds to bisect
    print 'We don\'t have enough builds to bisect. revlist: %s' % revlist
    sys.exit(1)

  # If we don't have a |good_rev|, set it to be the first revision possible.
  if good_rev == 0:
    good_rev = revlist[0]

  # These are indexes of |revlist|.
  good = 0
  bad = len(revlist) - 1
  last_known_good_rev = revlist[good]

  # Binary search time!
  while good < bad:
    candidates = revlist[good:bad]
    num_poss = len(candidates)
    if num_poss > 10:
      print('%d candidates. %d tries left.' %
          (num_poss, round(math.log(num_poss, 2))))
    else:
      print('Candidates: %s' % revlist[good:bad])

    # Cut the problem in half...
    test = int((bad - good) / 2) + good
    test_rev = revlist[test]

    # Let the user give this rev a spin (in her own profile, if she wants).
    profile = opts.profile
    if not profile:
      profile = 'profile'  # In a temp dir.
    TryRevision(test_rev, profile, args)
    if AskIsGoodBuild(test_rev):
      last_known_good_rev = revlist[good]
      good = test + 1
    else:
      bad = test

  # We're done. Let the user know the results in an official manner.
  print('You are probably looking for build %d.' % revlist[bad])
  print('CHANGELOG URL:')
  print(CHANGELOG_URL % (last_known_good_rev, revlist[bad]))
  print('Built at revision:')
  print(BUILD_VIEWVC_URL % revlist[bad])

if __name__ == '__main__':
  sys.exit(main())
<commit_msg>bisect-builds.py: Use the continuous archive, rather than the snapshots.<commit_after>#!/usr/bin/python
# Copyright (c) 2011 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Snapshot Build Bisect Tool

This script bisects a snapshot archive using binary search. It starts at
a bad revision (it will try to guess HEAD) and asks for a last known-good
revision. It will then binary search across this revision range by downloading,
unzipping, and opening Chromium for you. After testing the specific revision,
it will ask you whether it is good or bad before continuing the search.
"""

# Base URL to download snapshots from.
BUILD_BASE_URL = 'http://build.chromium.org/f/chromium/continuous/'

# The index file that lists all the builds. This lives in BUILD_BASE_URL.
BUILD_INDEX_FILE = 'all_builds.txt'

# The type (platform) of the build archive. This is what's passed in to the
# '-a/--archive' option.
BUILD_ARCHIVE_TYPE = ''

# The location of the builds. Format this with a (date, revision) tuple, which
# can be obtained through ParseIndexLine().
BUILD_ARCHIVE_URL = '/%s/%d/'

# Name of the build archive.
BUILD_ZIP_NAME = ''

# Directory name inside the archive.
BUILD_DIR_NAME = ''

# Name of the executable.
BUILD_EXE_NAME = ''

# URL to the ViewVC commit page.
BUILD_VIEWVC_URL = 'http://src.chromium.org/viewvc/chrome?view=rev&revision=%d'

# Changelogs URL
CHANGELOG_URL = 'http://build.chromium.org/f/chromium/' \
                'perf/dashboard/ui/changelog.html?url=/trunk/src&range=%d:%d'

###############################################################################

import math
import optparse
import os
import pipes
import re
import shutil
import sys
import tempfile
import urllib
import zipfile

def UnzipFilenameToDir(filename, dir):
  """Unzip |filename| to directory |dir|."""
  zf = zipfile.ZipFile(filename)
  # Make base.
  pushd = os.getcwd()
  try:
    if not os.path.isdir(dir):
      os.mkdir(dir)
    os.chdir(dir)
    # Extract files.
    for info in zf.infolist():
      name = info.filename
      if name.endswith('/'):  # dir
        if not os.path.isdir(name):
          os.makedirs(name)
      else:  # file
        dir = os.path.dirname(name)
        if not os.path.isdir(dir):
          os.makedirs(dir)
        out = open(name, 'wb')
        out.write(zf.read(name))
        out.close()
      # Set permissions. Permission info in external_attr is shifted 16 bits.
      os.chmod(name, info.external_attr >> 16L)
    os.chdir(pushd)
  except Exception, e:
    print >>sys.stderr, e
    sys.exit(1)

def SetArchiveVars(archive):
  """Set a bunch of global variables appropriate for the specified archive."""
  global BUILD_ARCHIVE_TYPE
  global BUILD_ZIP_NAME
  global BUILD_DIR_NAME
  global BUILD_EXE_NAME
  global BUILD_BASE_URL

  BUILD_ARCHIVE_TYPE = archive

  if BUILD_ARCHIVE_TYPE in ('linux', 'linux64', 'linux-chromiumos'):
    BUILD_ZIP_NAME = 'chrome-linux.zip'
    BUILD_DIR_NAME = 'chrome-linux'
    BUILD_EXE_NAME = 'chrome'
  elif BUILD_ARCHIVE_TYPE in ('mac'):
    BUILD_ZIP_NAME = 'chrome-mac.zip'
    BUILD_DIR_NAME = 'chrome-mac'
    BUILD_EXE_NAME = 'Chromium.app/Contents/MacOS/Chromium'
  elif BUILD_ARCHIVE_TYPE in ('xp'):
    BUILD_ZIP_NAME = 'chrome-win32.zip'
    BUILD_DIR_NAME = 'chrome-win32'
    BUILD_EXE_NAME = 'chrome.exe'

def ParseDirectoryIndex(url):
  """Parses the all_builds.txt index file. The format of this file is:
    mac/2011-02-16/75130
    mac/2011-02-16/75218
    mac/2011-02-16/75226
    mac/2011-02-16/75234
    mac/2011-02-16/75184
  This function will return a list of DATE/REVISION strings for the platform
  specified by BUILD_ARCHIVE_TYPE.
  """
  handle = urllib.urlopen(url)
  dirindex = handle.readlines()
  handle.close()

  # Only return values for the specified platform. Include the trailing slash to
  # not confuse linux and linux64.
  archtype = BUILD_ARCHIVE_TYPE + '/'
  dirindex = filter(lambda l: l.startswith(archtype), dirindex)

  # Remove the newline separator and the platform token.
  dirindex = map(lambda l: l[len(archtype):].strip(), dirindex)
  dirindex.sort()
  return dirindex

def ParseIndexLine(iline):
  """Takes an index line returned by ParseDirectoryIndex() and returns a
  2-tuple of (date, revision). |date| is a string and |revision| is an int."""
  split = iline.split('/')
  assert(len(split) == 2)
  return (split[0], int(split[1]))

def GetRevision(iline):
    """Takes an index line, parses it, and returns the revision."""
    return ParseIndexLine(iline)[1]

def GetRevList(good, bad):
  """Gets the list of revision numbers between |good| and |bad|."""
  # Download the main revlist.
  revlist = ParseDirectoryIndex(BUILD_BASE_URL + BUILD_INDEX_FILE)
  revrange = range(good, bad)
  revlist = filter(lambda r: GetRevision(r) in revrange, revlist)
  revlist.sort()
  return revlist

def TryRevision(iline, profile, args):
  """Downloads revision from |iline|, unzips it, and opens it for the user to
  test. |profile| is the profile to use."""
  # Do this in a temp dir so we don't collide with user files.
  cwd = os.getcwd()
  tempdir = tempfile.mkdtemp(prefix='bisect_tmp')
  os.chdir(tempdir)

  # Download the file.
  download_url = BUILD_BASE_URL + BUILD_ARCHIVE_TYPE + \
      (BUILD_ARCHIVE_URL % ParseIndexLine(iline)) + BUILD_ZIP_NAME
  def _ReportHook(blocknum, blocksize, totalsize):
    size = blocknum * blocksize
    if totalsize == -1:  # Total size not known.
      progress = "Received %d bytes" % size
    else:
      size = min(totalsize, size)
      progress = "Received %d of %d bytes, %.2f%%" % (
          size, totalsize, 100.0 * size / totalsize)
    # Send a \r to let all progress messages use just one line of output.
    sys.stdout.write("\r" + progress)
    sys.stdout.flush()
  try:
    print 'Fetching ' + download_url
    urllib.urlretrieve(download_url, BUILD_ZIP_NAME, _ReportHook)
    print
  except Exception, e:
    print('Could not retrieve the download. Sorry.')
    sys.exit(-1)

  # Unzip the file.
  print 'Unzipping ...'
  UnzipFilenameToDir(BUILD_ZIP_NAME, os.curdir)

  # Tell the system to open the app.
  args = ['--user-data-dir=%s' % profile] + args
  flags = ' '.join(map(pipes.quote, args))
  exe = os.path.join(os.getcwd(), BUILD_DIR_NAME, BUILD_EXE_NAME)
  cmd = '%s %s' % (exe, flags)
  print 'Running %s' % cmd
  os.system(cmd)

  os.chdir(cwd)
  print 'Cleaning temp dir ...'
  try:
    shutil.rmtree(tempdir, True)
  except Exception, e:
    pass

def AskIsGoodBuild(iline):
  """Ask the user whether build from index line |iline| is good or bad."""
  # Loop until we get a response that we can parse.
  while True:
    response = raw_input('\nBuild %d is [(g)ood/(b)ad]: ' % GetRevision(iline))
    if response and response in ('g', 'b'):
      return response == 'g'

def main():
  usage = ('%prog [options] [-- chromium-options]\n'
           'Perform binary search on the snapshot builds.\n'
           '\n'
           'Tip: add "-- --no-first-run" to bypass the first run prompts.')
  parser = optparse.OptionParser(usage=usage)
  # Strangely, the default help output doesn't include the choice list.
  choices = ['mac', 'xp', 'linux', 'linux64']
            # linux-chromiumos lacks a continuous archive http://crbug.com/78158
  parser.add_option('-a', '--archive',
                    choices = choices,
                    help = 'The buildbot archive to bisect [%s].' %
                           '|'.join(choices))
  parser.add_option('-b', '--bad', type = 'int',
                    help = 'The bad revision to bisect to.')
  parser.add_option('-g', '--good', type = 'int',
                    help = 'The last known good revision to bisect from.')
  parser.add_option('-p', '--profile', '--user-data-dir', type = 'str',
                    help = 'Profile to use; this will not reset every run. ' +
                    'Defaults to a clean profile.')
  (opts, args) = parser.parse_args()

  if opts.archive is None:
    print 'Error: missing required parameter: --archive'
    print
    parser.print_help()
    return 1

  if opts.bad and opts.good and (opts.good > opts.bad):
    print ('The good revision (%d) must precede the bad revision (%d).\n' %
           (opts.good, opts.bad))
    parser.print_help()
    return 1

  SetArchiveVars(opts.archive)

  # Pick a starting point, try to get HEAD for this.
  if opts.bad:
    bad_rev = opts.bad
  else:
    bad_rev = 0
    try:
      # Location of the latest build revision number
      BUILD_LATEST_URL = '%s/LATEST/REVISION' % (BUILD_BASE_URL)
      nh = urllib.urlopen(BUILD_LATEST_URL)
      latest = int(nh.read())
      nh.close()
      bad_rev = raw_input('Bad revision [HEAD:%d]: ' % latest)
      if (bad_rev == ''):
        bad_rev = latest
      bad_rev = int(bad_rev)
    except Exception, e:
      print('Could not determine latest revision. This could be bad...')
      bad_rev = int(raw_input('Bad revision: '))

  # Find out when we were good.
  if opts.good:
    good_rev = opts.good
  else:
    good_rev = 0
    try:
      good_rev = int(raw_input('Last known good [0]: '))
    except Exception, e:
      pass

  # Get a list of revisions to bisect across.
  revlist = GetRevList(good_rev, bad_rev)
  if len(revlist) < 2:  # Don't have enough builds to bisect
    print 'We don\'t have enough builds to bisect. revlist: %s' % revlist
    sys.exit(1)

  # If we don't have a |good_rev|, set it to be the first revision possible.
  if good_rev == 0:
    good_rev = revlist[0]

  # These are indexes of |revlist|.
  good = 0
  bad = len(revlist) - 1
  last_known_good_rev = revlist[good]

  # Binary search time!
  while good < bad:
    candidates = revlist[good:bad]
    num_poss = len(candidates)
    if num_poss > 10:
      print('%d candidates. %d tries left.' %
          (num_poss, round(math.log(num_poss, 2))))
    else:
      print('Candidates: %s' % map(GetRevision, revlist[good:bad]))

    # Cut the problem in half...
    test = int((bad - good) / 2) + good
    test_rev = revlist[test]

    # Let the user give this rev a spin (in her own profile, if she wants).
    profile = opts.profile
    if not profile:
      profile = 'profile'  # In a temp dir.
    TryRevision(test_rev, profile, args)
    if AskIsGoodBuild(test_rev):
      last_known_good_rev = revlist[good]
      good = test + 1
    else:
      bad = test

  # We're done. Let the user know the results in an official manner.
  bad_revision = GetRevision(revlist[bad])
  print('You are probably looking for build %d.' % bad_revision)
  print('CHANGELOG URL:')
  print(CHANGELOG_URL % (GetRevision(last_known_good_rev), bad_revision))
  print('Built at revision:')
  print(BUILD_VIEWVC_URL % bad_revision)

if __name__ == '__main__':
  sys.exit(main())
<|endoftext|>


<commit_before>''' imports from installed packages '''
from django.core.management.base import BaseCommand, CommandError

from django_mongokit import get_database

try:
    from bson import ObjectId
except ImportError:  # old pymongo
    from pymongo.objectid import ObjectId

''' imports from application folders/files '''
from gnowsys_ndf.ndf.models import Node
###################################################################################################################################################################################
collection = get_database()[Node.collection_name]
gapp_GST = collection.Node.one({'_type': 'MetaType', 'name': 'GAPP'})

class Command(BaseCommand):

    help = "This script will create a Term app"

    def handle(self, *args, **options):

        term = collection.Node.one({'_type': 'GSystemType', 'name': 'Term', 'member_of': ObjectId(gapp_GST._id) })
        if not term:
            term = collection.GSystemType()
            term.name = u"Term"
            term.access_policy = u"PUBLIC"
            term.contributors.append(1)
            term.created_by = int(1)
            term.modified_by = int(1)
            term.status = u"PUBLISHED"
            term.member_of.append(gapp_GST._id)
            term.meta_type_set.append(gapp_GST)
            term.save()
            print "'Term' app successfully created ! !"
        else:
            print "'Term' app already available"<commit_msg>altname 'Topics' added in script for creating term app<commit_after>''' imports from installed packages '''
from django.core.management.base import BaseCommand, CommandError

from django_mongokit import get_database

try:
    from bson import ObjectId
except ImportError:  # old pymongo
    from pymongo.objectid import ObjectId

''' imports from application folders/files '''
from gnowsys_ndf.ndf.models import Node
###################################################################################################################################################################################
collection = get_database()[Node.collection_name]
gapp_GST = collection.Node.one({'_type': 'MetaType', 'name': 'GAPP'})

class Command(BaseCommand):

    help = "This script will create a Term app"

    def handle(self, *args, **options):

        term = collection.Node.one({'_type': 'GSystemType', 'name': 'Term', 'member_of': ObjectId(gapp_GST._id) })
        if not term:
            term = collection.GSystemType()
            term.name = u"Term"
            term.altnames = u"Topics"
            term.access_policy = u"PUBLIC"
            term.contributors.append(1)
            term.created_by = int(1)
            term.modified_by = int(1)
            term.status = u"PUBLISHED"
            term.member_of.append(gapp_GST._id)
            term.meta_type_set.append(gapp_GST)
            term.save()
            print "'Term' app successfully created ! !"
        else:
            print "'Term' app already available"<|endoftext|>


<commit_before>import PyOpenWorm as P
import traceback
import sqlite3

def print_evidence():
    try:
        conn = sqlite3.connect('db/celegans.db')
        cur = conn.cursor()
        cur.execute("SELECT DISTINCT a.Entity, b.Entity, Citations FROM tblrelationship, tblentity a, tblentity b where EnID1=a.id and EnID2=b.id and Citations!='' ")
        for r in cur.fetchall():
            print r
    except Exception:
        traceback.print_exc()
    finally:
        conn.close()

def upload_muscles():
    """ Upload muscles and the neurons that connect to them
    """
    try:
        conn = sqlite3.connect('db/celegans.db')
        cur = conn.cursor()
        w = P.Worm()
        cur.execute("SELECT DISTINCT a.Entity, b.Entity FROM tblrelationship, tblentity b, tblentity a where Relation = '1516' and EnID2=b.id and EnID1=a.id")
        for r in cur.fetchall():
            muscle_name = str(r[1])
            m = P.Muscle(muscle_name)
            w.muscle(m)
            n = P.Neuron(str(r[0]))
            m.innervatedBy(n)

        w.save()
        #second step, get the relationships between them and add them to the graph
    except Exception:
        traceback.print_exc()
    finally:
        conn.close()

def upload_lineage_and_descriptions():
    """ Upload lineage names and descriptions pulled from the WormAtlas master cell list

    Assumes that Neurons and Muscles have already been added
    """
    # XXX: This wants a way to insert cells, then later, to insert neurons from the same set
    # and have the later recoginzed as the former. Identifier matching limits us here. It would
    # be best to establish owl:sameAs links to the super class (Cell) from the subclass (Neuron)
    # at the sub-class insert and have a reasoner relate
    # the two sets of inserts.
    cells = dict()
    try:
        w = P.Worm()
        ev = P.Evidence(uri="http://www.wormatlas.org/celllist.htm")
        # insert neurons.
        # save
        f = open("C. elegans Cell List - WormAtlas.tsv", "r")
        h = open("C. elegans Cell List - WormBase.tsv", "r")

        # Skip headers
        next(h)
        next(h)
        next(h)
        next(f)
        names = dict()
        for x in f:
             j = [x.strip().strip("\"") for x in x.split("\t")]
             name=j[0]
             if name in names:
                 while (name in names):
                     names[name] += 1
                     name = name + "("+ str(names[name]) +")"
             else:
                 names[name] = 0

             c = P.Cell(name,j[1])
             c.description(j[2])
             w.cell(c)
             cells[name] = (j[1],j[2])
        # We bring in the neurons and muscles through their shared name
        # -- the name is the only thing that relates these sets of
        # objects

        ev.asserts(w)
        ev.save()
    except Exception, e:
        traceback.print_exc()
    return cells

def norn(x):
    """ return the next or None of an iterator """
    try:
        return next(x)
    except StopIteration:
        return None

def update_neurons_and_muscles_with_lineage_and_descriptions():
    v = P.values('neurons and muscles')
    #XXX: This could be expensive...the lineage name and description should be loaded with the cell though.
    cells = {next(x.name()) : (norn(x.lineageName()), norn(x.description())) for x in P.Cell().load() }
    def dtt(x):
        """ Do the thing """
        try:
            name = next(x.name())
            if cells[name][0] is not None:
                x.lineageName(cells[name][0])
            if cells[name][1] is not None:
                x.description(cells[name][1])
            v.value(x)
        except:
            traceback.print_exc()
    for x in P.Neuron().load():
        dtt(x)
    for x in P.Muscle().load():
        dtt(x)

    v.save()

def upload_neurons():
    try:
        conn = sqlite3.connect('db/celegans.db')
        cur = conn.cursor()
        w = P.Worm()
        n = P.Network()
        w.neuron_network(n)
        # insert neurons.
        # save
        cur.execute("SELECT DISTINCT a.Entity FROM tblrelationship, tblentity a, tblentity b where EnID1=a.id and Relation = '1515' and EnID2='1'")
        for r in cur.fetchall():
            neuron_name = str(r[0])
            n.neuron(P.Neuron(name=neuron_name))
        n.save()
        #second step, get the relationships between them and add them to the graph
    except Exception:
        traceback.print_exc()
    finally:
        conn.close()

def upload_receptors_and_innexins():
    try:
        conn = sqlite3.connect('db/celegans.db')
        cur = conn.cursor()
        w = P.Worm()
        n = P.Network()
        w.neuron_network(n)
        # insert neurons.
        # save
        # get the receptor (354) and innexin (355)
        cur.execute("""
        SELECT DISTINCT a.Entity, b.Entity
        FROM
        tblrelationship q,
        tblrelationship r,
        tblentity a,
        tblentity b
        where q.EnID1=a.id and q.Relation = '1515' and q.EnID2='1'
        and   r.EnID1=a.id and r.Relation = '354'  and r.EnID2=b.id
        """)
        for r in cur.fetchall():
            neuron_name = str(r[0])
            receptor = str(r[1])
            neur = P.Neuron(name=neuron_name)
            neur.receptor(receptor)
            n.neuron(neur)
        cur.execute("""
        SELECT DISTINCT a.Entity, b.Entity
        FROM
        tblrelationship q,
        tblrelationship r,
        tblentity a,
        tblentity b
        where q.EnID1=a.id and q.Relation = '1515' and q.EnID2='1'
        and   r.EnID1=a.id and r.Relation = '355'  and r.EnID2=b.id
        """)
        for r in cur.fetchall():
            neuron_name = str(r[0])
            innexin = str(r[1])
            neur = P.Neuron(name=neuron_name)
            neur.innexin(innexin)
            n.neuron(neur)
        n.save()
        #second step, get the relationships between them and add them to the graph
    except Exception:
        traceback.print_exc()
    finally:
        conn.close()

def upload_synapses():
    try:
        conn = sqlite3.connect('db/celegans.db')
        cur = conn.cursor()
        w = P.Worm()
        n = P.Network()
        w.neuron_network(n)
        #second step, get the relationships between them and add them to the graph
        cur.execute("SELECT DISTINCT a.Entity, b.Entity, Weight, Relation FROM tblrelationship, tblentity a, tblentity b where EnID1=a.id and EnID2=b.id and (Relation = '356' OR Relation = '357')")

        for r in cur.fetchall():
            #all items are numbers -- need to be converted to a string
            first = str(r[0])
            second = str(r[1])
            third = str(r[2])
            syntype = str(r[3])
            if syntype == '356':
                syntype = 'send'
            else:
                syntype = 'gapjunction'
            try:
                weight = int(third)
                # NMJs have negative weights. we only want the synaptic connections
                if weight < 0:
                    syntype = 'gapjunction'
                    weight = -1 * weight

            except:
                weight = None

            if weight:
                c = P.Connection(pre_cell=first, post_cell=second, number=weight, syntype=syntype)
                n.synapse(c)
        e = P.Evidence(author='markw@cs.utexas.edu')
        e.asserts(w)
        e.save()
    except Exception, e:
        traceback.print_exc()
    finally:
        conn.close()

def infer():
    from rdflib import Graph
    from FuXi.Rete.RuleStore import SetupRuleStore
    from FuXi.Rete.Util import generateTokenSet
    from FuXi.Horn.HornRules import HornFromN3

    try:
        semnet = Graph()
        w = P.Worm()
        semnet = w.get_semantic_net() #fetches the entire worm.db graph
        facts = semnet.serialize(format='n3') #formats the graph so FuXi can read it

        rule_store, rule_graph, network = SetupRuleStore(makeNetwork=True)
        closureDeltaGraph = Graph()
        network.inferredFacts = closureDeltaGraph

        #build a network of rules
        for rule in HornFromN3('fuxi/testrules.n3'):
            network.buildNetworkFromClause(rule)

        network.feedFactsToAdd(generateTokenSet(semnet)) #apply rules to original facts to infer new facts

        full_graph = semnet + closureDeltaGraph #combine original facts with inferred facts
        w['semantic_net'] = full_graph #set worm graph to this combined graph
        w.save() #write worm object to worm.db

        ###uncomment next 4 lines to print inferred facts to human-readable file (demo purposes)
        #inferred_facts = closureDeltaGraph.serialize(format='n3') #format inferred facts to notation 3
        #inferred = open('what_was_inferred.n3', 'w')
        #inferred.write(inferred_facts)
        #inferred.close()

    except Exception, e:
        traceback.print_exc()

if __name__ == '__main__':
    import sys
    logging = False
    if len(sys.argv) > 1 and sys.argv[1] == '-l':
        logging = True
    P.connect(configFile='readme.conf',do_logging=logging)
    try:
        print_evidence()
        upload_muscles()
        print ("uploaded muscles")
        upload_lineage_and_descriptions()
        print ("uploaded lineage and descriptions")
        upload_synapses()
        print ("uploaded synapses")
        upload_receptors_and_innexins()
        print ("uploaded receptors and innexins")
        update_neurons_and_muscles_with_lineage_and_descriptions()
        print ("updated muscles and neurons with cell data")
        infer()
        print ("filled in with inferred data")
    except:
        traceback.print_exc()
    finally:
        P.disconnect()

<commit_msg>Removed some dead-code in insert_worm.py<commit_after>import PyOpenWorm as P
import traceback
import sqlite3

def print_evidence():
    try:
        conn = sqlite3.connect('db/celegans.db')
        cur = conn.cursor()
        cur.execute("SELECT DISTINCT a.Entity, b.Entity, Citations FROM tblrelationship, tblentity a, tblentity b where EnID1=a.id and EnID2=b.id and Citations!='' ")
        for r in cur.fetchall():
            print r
    except Exception:
        traceback.print_exc()
    finally:
        conn.close()

def upload_muscles():
    """ Upload muscles and the neurons that connect to them
    """
    try:
        conn = sqlite3.connect('db/celegans.db')
        cur = conn.cursor()
        w = P.Worm()
        cur.execute("SELECT DISTINCT a.Entity, b.Entity FROM tblrelationship, tblentity b, tblentity a where Relation = '1516' and EnID2=b.id and EnID1=a.id")
        for r in cur.fetchall():
            muscle_name = str(r[1])
            m = P.Muscle(muscle_name)
            w.muscle(m)
            n = P.Neuron(str(r[0]))
            m.innervatedBy(n)

        w.save()
        #second step, get the relationships between them and add them to the graph
    except Exception:
        traceback.print_exc()
    finally:
        conn.close()

def upload_lineage_and_descriptions():
    """ Upload lineage names and descriptions pulled from the WormAtlas master cell list

    Assumes that Neurons and Muscles have already been added
    """
    # XXX: This wants a way to insert cells, then later, to insert neurons from the same set
    # and have the later recoginzed as the former. Identifier matching limits us here. It would
    # be best to establish owl:sameAs links to the super class (Cell) from the subclass (Neuron)
    # at the sub-class insert and have a reasoner relate
    # the two sets of inserts.
    cells = dict()
    try:
        w = P.Worm()
        ev = P.Evidence(uri="http://www.wormatlas.org/celllist.htm")
        # insert neurons.
        # save
        f = open("C. elegans Cell List - WormAtlas.tsv", "r")
        h = open("C. elegans Cell List - WormBase.tsv", "r")

        # Skip headers
        next(h)
        next(h)
        next(h)
        next(f)
        names = dict()
        for x in f:
             j = [x.strip().strip("\"") for x in x.split("\t")]
             name=j[0]
             if name in names:
                 while (name in names):
                     names[name] += 1
                     name = name + "("+ str(names[name]) +")"
             else:
                 names[name] = 0

             c = P.Cell(name,j[1])
             c.description(j[2])
             w.cell(c)
             cells[name] = (j[1],j[2])
        # We bring in the neurons and muscles through their shared name
        # -- the name is the only thing that relates these sets of
        # objects

        ev.asserts(w)
        ev.save()
    except Exception, e:
        traceback.print_exc()
    return cells

def norn(x):
    """ return the next or None of an iterator """
    try:
        return next(x)
    except StopIteration:
        return None

def update_neurons_and_muscles_with_lineage_and_descriptions():
    v = P.values('neurons and muscles')
    #XXX: This could be expensive...the lineage name and description should be loaded with the cell though.
    cells = {next(x.name()) : (norn(x.lineageName()), norn(x.description())) for x in P.Cell().load() }
    def dtt(x):
        """ Do the thing """
        try:
            name = next(x.name())
            if cells[name][0] is not None:
                x.lineageName(cells[name][0])
            if cells[name][1] is not None:
                x.description(cells[name][1])
            v.value(x)
        except:
            traceback.print_exc()
    for x in P.Neuron().load():
        dtt(x)
    for x in P.Muscle().load():
        dtt(x)

    v.save()

def upload_neurons():
    try:
        conn = sqlite3.connect('db/celegans.db')
        cur = conn.cursor()
        w = P.Worm()
        n = P.Network()
        w.neuron_network(n)
        # insert neurons.
        # save
        cur.execute("SELECT DISTINCT a.Entity FROM tblrelationship, tblentity a, tblentity b where EnID1=a.id and Relation = '1515' and EnID2='1'")
        for r in cur.fetchall():
            neuron_name = str(r[0])
            n.neuron(P.Neuron(name=neuron_name))
        n.save()
        #second step, get the relationships between them and add them to the graph
    except Exception:
        traceback.print_exc()
    finally:
        conn.close()

def upload_receptors_and_innexins():
    try:
        conn = sqlite3.connect('db/celegans.db')
        cur = conn.cursor()
        w = P.Worm()
        n = P.Network()
        w.neuron_network(n)
        # insert neurons.
        # save
        # get the receptor (354) and innexin (355)
        cur.execute("""
        SELECT DISTINCT a.Entity, b.Entity
        FROM
        tblrelationship q,
        tblrelationship r,
        tblentity a,
        tblentity b
        where q.EnID1=a.id and q.Relation = '1515' and q.EnID2='1'
        and   r.EnID1=a.id and r.Relation = '354'  and r.EnID2=b.id
        """)
        for r in cur.fetchall():
            neuron_name = str(r[0])
            receptor = str(r[1])
            neur = P.Neuron(name=neuron_name)
            neur.receptor(receptor)
            n.neuron(neur)
        cur.execute("""
        SELECT DISTINCT a.Entity, b.Entity
        FROM
        tblrelationship q,
        tblrelationship r,
        tblentity a,
        tblentity b
        where q.EnID1=a.id and q.Relation = '1515' and q.EnID2='1'
        and   r.EnID1=a.id and r.Relation = '355'  and r.EnID2=b.id
        """)
        for r in cur.fetchall():
            neuron_name = str(r[0])
            innexin = str(r[1])
            neur = P.Neuron(name=neuron_name)
            neur.innexin(innexin)
            n.neuron(neur)
        n.save()
        #second step, get the relationships between them and add them to the graph
    except Exception:
        traceback.print_exc()
    finally:
        conn.close()

def upload_synapses():
    try:
        conn = sqlite3.connect('db/celegans.db')
        cur = conn.cursor()
        w = P.Worm()
        n = P.Network()
        w.neuron_network(n)
        #second step, get the relationships between them and add them to the graph
        cur.execute("SELECT DISTINCT a.Entity, b.Entity, Weight, Relation FROM tblrelationship, tblentity a, tblentity b where EnID1=a.id and EnID2=b.id and (Relation = '356' OR Relation = '357')")

        for r in cur.fetchall():
            #all items are numbers -- need to be converted to a string
            first = str(r[0])
            second = str(r[1])
            third = str(r[2])
            syntype = str(r[3])
            if syntype == '356':
                syntype = 'send'
            else:
                syntype = 'gapjunction'
            try:
                weight = int(third)
                # NMJs have negative weights. we only want the synaptic connections
                if weight < 0:
                    syntype = 'gapjunction'
                    weight = -1 * weight

            except:
                weight = None

            if weight:
                c = P.Connection(pre_cell=first, post_cell=second, number=weight, syntype=syntype)
                n.synapse(c)
        e = P.Evidence(author='markw@cs.utexas.edu')
        e.asserts(w)
        e.save()
    except Exception, e:
        traceback.print_exc()
    finally:
        conn.close()

def infer():
    from rdflib import Graph
    from FuXi.Rete.RuleStore import SetupRuleStore
    from FuXi.Rete.Util import generateTokenSet
    from FuXi.Horn.HornRules import HornFromN3

    try:
        w = P.Worm()
        semnet = w.rdf #fetches the entire worm.db graph

        rule_store, rule_graph, network = SetupRuleStore(makeNetwork=True)
        closureDeltaGraph = Graph()
        network.inferredFacts = closureDeltaGraph

        #build a network of rules
        for rule in HornFromN3('fuxi/testrules.n3'):
            network.buildNetworkFromClause(rule)

        network.feedFactsToAdd(generateTokenSet(semnet)) # apply rules to original facts to infer new facts

        w.rdf = semnet + closureDeltaGraph # combine original facts with inferred facts

        ###uncomment next 4 lines to print inferred facts to human-readable file (demo purposes)
        #inferred_facts = closureDeltaGraph.serialize(format='n3') #format inferred facts to notation 3
        #inferred = open('what_was_inferred.n3', 'w')
        #inferred.write(inferred_facts)
        #inferred.close()

    except Exception, e:
        traceback.print_exc()

if __name__ == '__main__':
    import sys
    logging = False
    if len(sys.argv) > 1 and sys.argv[1] == '-l':
        logging = True
    P.connect(configFile='readme.conf',do_logging=logging)
    try:
        print_evidence()
        upload_muscles()
        print ("uploaded muscles")
        upload_lineage_and_descriptions()
        print ("uploaded lineage and descriptions")
        upload_synapses()
        print ("uploaded synapses")
        upload_receptors_and_innexins()
        print ("uploaded receptors and innexins")
        update_neurons_and_muscles_with_lineage_and_descriptions()
        print ("updated muscles and neurons with cell data")
        infer()
        print ("filled in with inferred data")
    except:
        traceback.print_exc()
    finally:
        P.disconnect()

<|endoftext|>