<commit_before>import numpy

class ChiSquared(object):
    """ This class calculates the chi squared comparing "data" to "montecarlo".

    The calculation is based on a spectrum containing observed events and one
    containing expected events. The paradigm assumed is that the observed
    events form the data spectrum and the expected events form the montecarlo
    spectrum. A couple of different methods for calculating chi squared are
    included, as well as the option to add constraints via a penalty term.

    Args:
      form (str, optional): specify form of chi squared calculation to use
      **kwargs: Keyword arguments

    .. note::

      Forms of chi squared include

        * "pearson"
        * "neyman"
        * "poisson_likelihood" (*default*)

    .. note::

      Keyword arguments include

        * penalty_terms (*dict*): specify (for each penalty term) values for:

          * "parameter_value" (*optional*)
          * "sigma"

    Attributes:
      _form (str): form of chi squared calculation to use
      _penalty_terms (dict): information about each penalty term
      _penalty_terms_set (bool): True if one or more penalty terms have been
        set
    """
    def __init__(self, form="poisson_likelihood", **kwargs):
        self._form = form
        if (kwargs.get("penalty_terms") is not None):
            self._penalty_terms = kwargs.get("penalty_terms")
            self._penalty_terms_set = True
        else:
            self._penalty_terms = None
            self._penalty_terms_set = False

    def get_chi_squared(self, observed, expected, **kwargs):
        """ Calculate the chi squared comparing observed to expected.

        Args:
          observed (:class:`numpy.array`): energy spectrum of observed events
          expected (:class:`numpy.array`): energy spectrum of expected events
          **kwargs: keyword argumets

        .. note::

          Keyword arguments include

            * penalty_terms (*dict*): specify (for each penalty term) values
              for:

              * "parameter_value"
              * "sigma" (*optional*)

        .. warning::

          A named penalty term defined here will overwrite one with the same
          name defined in the constructor.

        Returns:
          *float*. Value of chi squared calculated
        """
        # Set up penalty term
        if (kwargs.get("penalty_terms") is not None):
            if self._penalty_terms_set:
                for name, penalty_term in \
                        kwargs.get("penalty_terms").iteritems():
                    if (self._penalty_terms.get(name) is not None):
                        _penalty_term = self._penalty_terms.get(name)
                        # overwrite existing entries
                        if (penalty_term.get("parameter_value") is not None):
                            _penalty_term["parameter_value"] = \
                                penalty_term.get("parameter_value")
                        if (penalty_term.get("sigma") is not None):
                            _penalty_term["sigma"] = penalty_term.get("sigma")
                    else:  # create new entry
                        self._penalty_terms[name] = penalty_term
            else:  # no penalty term information currently set
                self._penalty_terms = kwargs.get("penalty_terms")
                self._penalty_terms_set = True

        # Calculate chi squared
        if (self._form == "pearson"):
            chi_squared = pearson_chi_squared(observed, expected)
        elif (self._form == "neyman"):
            chi_squared = neyman_chi_squared(observed, expected)
        else:  # (self._form == "poisson_likelihood")
            chi_squared = 2.0 * log_likelihood(observed, expected)

        # Add penalty term(s)
        if self._penalty_terms_set:
            for name, penalty_term in self._penalty_terms.iteritems():
                chi_squared += numpy.power(penalty_term.get("parameter_value")/
                                           penalty_term.get("sigma"), 2.0)
        return chi_squared

def check_bin_content(array):
    """ Checks bin content of a numpy array.

    Bin content must be > 0 to be used in chi squared calculations

    Args:
      array (:class:`numpy.array`, *float*): Array or value to check

    Returns:
      *bool*, *string*. Tuple containing result of check and an error message
    """
    if (numpy.sum(array <= 0.0) != 0):
        result = False
        zero_bins, = numpy.where(array <= 0.0)
        message = "array contains " + str(len(zero_bins)) + \
            " bins with content <= 0.0. First instance at bin " + \
            str(zero_bins[0]) + "."
    else:
        result = True
        message = "array - all bins > 0.0"
    return result, message

def pearson_chi_squared(observed, expected):
    """ Calculates Pearson's chi squared.

    .. note::

      Following the definition in `Baker and Cousins, 1984
      <http://www.sciencedirect.com/science/article/pii/0167508784900164>`_

    Args:
      observed (:class:`numpy.array`, *float*): Number of observed events
      expected (:class:`numpy.array`, *float*): Number of expected events

    Raises:
      ValueError: If either array contains a bin with content <= 0.0

    Returns:
      *float*. Calculated Pearson's chi squared
    """
    correct_bin_content, message = check_bin_content(observed)
    if not correct_bin_content:
        raise ValueError("chi_squared.pearson_chi_squared: observed " +
                         message)
    correct_bin_content, message = check_bin_content(expected)
    if not correct_bin_content:
        raise ValueError("chi_squared.pearson_chi_squared: expected " +
                         message)
    return numpy.sum((observed-expected)**2 / expected)

def neyman_chi_squared(observed, expected):
    """ Calculates Neyman's chi squared.

    .. note::

      Following the definition in `Baker and Cousins, 1984
      <http://www.sciencedirect.com/science/article/pii/0167508784900164>`_

    Args:
      observed (:class:`numpy.array`, *float*): Number of observed events.
      expected (:class:`numpy.array`, *float*): Number of expected events.

    Raises:
      ValueError: If either array contains a bin with content <= 0.0

    Returns:
      float. Calculated Neyman's chi squared
    """
    correct_bin_content, message = check_bin_content(observed)
    if not correct_bin_content:
        raise ValueError("chi_squared.pearson_chi_squared: observed " +
                         message)
    correct_bin_content, message = check_bin_content(expected)
    if not correct_bin_content:
        raise ValueError("chi_squared.pearson_chi_squared: expected " +
                         message)
    return numpy.sum((observed-expected)**2 / observed)

def log_likelihood(observed, expected):
    """ Calculates the (Baker-Cousins) log likelihood.

    .. note::

      For calculation of Poisson likelihood chi squared.

    .. note::

      Following the definition in `Baker and Cousins, 1984
      <http://www.sciencedirect.com/science/article/pii/0167508784900164>`_

    Args:
      observed (:class:`numpy.array`, *float*): Number of observed events
      expected (:class:`numpy.array`, *float*): Number of expected events

    Raises:
      ValueError: If either array contains a bin with content <= 0.0

    Returns:
      *float*. Calculated Neyman's chi squared
    """
    correct_bin_content, message = check_bin_content(observed)
    if not correct_bin_content:
        raise ValueError("chi_squared.pearson_chi_squared: observed " +
                         message)
    correct_bin_content, message = check_bin_content(expected)
    if not correct_bin_content:
        raise ValueError("chi_squared.pearson_chi_squared: expected " +
                         message)
    return numpy.sum(expected - observed +
                     observed*numpy.log(observed/expected))
<commit_msg>Final modifications based on review comments<commit_after>import numpy

class ChiSquared(object):
    """ This class calculates the chi squared comparing "data" to "montecarlo".

    The calculation is based on a spectrum containing observed events and one
    containing expected events. The paradigm assumed is that the observed
    events form the data spectrum and the expected events form the montecarlo
    spectrum. A couple of different methods for calculating chi squared are
    included, as well as the option to add constraints via a penalty term.

    Args:
      form (str, optional): specify form of chi squared calculation to use
      **kwargs: Keyword arguments

    .. note::

      Forms of chi squared include

        * "pearson"
        * "neyman"
        * "poisson_likelihood" (*default*)

    .. note::

      Keyword arguments include

        * penalty_terms (*dict*): specify (for each penalty term) values for:

          * "parameter_value" (*optional*)
          * "sigma"

    Attributes:
      _form (str): form of chi squared calculation to use
      _penalty_terms (dict): information about each penalty term
      _penalty_terms_set (bool): True if one or more penalty terms have been
        set
    """
    def __init__(self, form="poisson_likelihood", **kwargs):
        self._form = form
        if (kwargs.get("penalty_terms") is not None):
            self._penalty_terms = kwargs.get("penalty_terms")
            self._penalty_terms_set = True
        else:
            self._penalty_terms = None
            self._penalty_terms_set = False

    def get_chi_squared(self, observed, expected, **kwargs):
        """ Calculate the chi squared comparing observed to expected.

        Args:
          observed (:class:`numpy.array`): energy spectrum of observed events
          expected (:class:`numpy.array`): energy spectrum of expected events
          **kwargs: keyword argumets

        .. note::

          Keyword arguments include

            * penalty_terms (*dict*): specify (for each penalty term) values
              for:

              * "parameter_value"
              * "sigma" (*optional*)

        .. warning::

          A named penalty term defined here will overwrite one with the same
          name defined in the constructor.

        Returns:
          *float*. Value of chi squared calculated
        """
        # Set up penalty term
        if (kwargs.get("penalty_terms") is not None):
            if self._penalty_terms_set:
                for name, penalty_term in \
                        kwargs.get("penalty_terms").iteritems():
                    if (self._penalty_terms.get(name) is not None):
                        _penalty_term = self._penalty_terms.get(name)
                        # overwrite existing entries
                        if (penalty_term.get("parameter_value") is not None):
                            _penalty_term["parameter_value"] = \
                                penalty_term.get("parameter_value")
                        if (penalty_term.get("sigma") is not None):
                            _penalty_term["sigma"] = penalty_term.get("sigma")
                    else:  # create new entry
                        self._penalty_terms[name] = penalty_term
            else:  # no penalty term information currently set
                self._penalty_terms = kwargs.get("penalty_terms")
                self._penalty_terms_set = True

        # Calculate chi squared
        if (self._form == "pearson"):
            chi_squared = pearson_chi_squared(observed, expected)
        elif (self._form == "neyman"):
            chi_squared = neyman_chi_squared(observed, expected)
        else:  # (self._form == "poisson_likelihood")
            chi_squared = 2.0 * log_likelihood(observed, expected)

        # Add penalty term(s)
        if self._penalty_terms_set:
            for name, penalty_term in self._penalty_terms.iteritems():
                chi_squared += numpy.power(penalty_term.get("parameter_value")/
                                           penalty_term.get("sigma"), 2.0)
        return chi_squared

def check_bin_content(array):
    """ Checks bin content of a numpy array.

    Bin content must be > 0 to be used in chi squared calculations

    Args:
      array (:class:`numpy.array`, *float*): Array or value to check

    Returns:
      *bool*, *string*. Tuple containing result of check and an error message
    """
    if (numpy.sum(array <= 0.0) != 0):
        result = False
        zero_bins, = numpy.where(array <= 0.0)
        message = "array contains " + str(len(zero_bins)) + \
            " bins with content <= 0.0. First instance at bin " + \
            str(zero_bins[0]) + "."
    else:
        result = True
        message = "array - all bins > 0.0"
    return result, message

def pearson_chi_squared(observed, expected):
    """ Calculates Pearson's chi squared.

    .. note::

      Following the definition in `Baker and Cousins, 1984
      <http://www.sciencedirect.com/science/article/pii/0167508784900164>`_

    Args:
      observed (:class:`numpy.array`, *float*): Number of observed events
      expected (:class:`numpy.array`, *float*): Number of expected events

    Raises:
      ValueError: If either array contains a bin with content <= 0.0

    Returns:
      *float*. Calculated Pearson's chi squared
    """
    correct_bin_content, message = check_bin_content(observed)
    if not correct_bin_content:
        raise ValueError("chi_squared.pearson_chi_squared: observed " +
                         message)
    correct_bin_content, message = check_bin_content(expected)
    if not correct_bin_content:
        raise ValueError("chi_squared.pearson_chi_squared: expected " +
                         message)
    return numpy.sum((observed-expected)**2 / expected)

def neyman_chi_squared(observed, expected):
    """ Calculates Neyman's chi squared.

    .. note::

      Following the definition in `Baker and Cousins, 1984
      <http://www.sciencedirect.com/science/article/pii/0167508784900164>`_

    Args:
      observed (:class:`numpy.array`, *float*): Number of observed events.
      expected (:class:`numpy.array`, *float*): Number of expected events.

    Raises:
      ValueError: If either array contains a bin with content <= 0.0

    Returns:
      float. Calculated Neyman's chi squared
    """
    correct_bin_content, message = check_bin_content(observed)
    if not correct_bin_content:
        raise ValueError("chi_squared.pearson_chi_squared: observed " +
                         message)
    correct_bin_content, message = check_bin_content(expected)
    if not correct_bin_content:
        raise ValueError("chi_squared.pearson_chi_squared: expected " +
                         message)
    return numpy.sum((observed-expected)**2 / observed)

def log_likelihood(observed, expected):
    """ Calculates the (Baker-Cousins) log likelihood.

    .. note::

      For calculation of Poisson likelihood chi squared.

    .. note::

      Following the definition in `Baker and Cousins, 1984
      <http://www.sciencedirect.com/science/article/pii/0167508784900164>`_

    Args:
      observed (:class:`numpy.array`, *float*): Number of observed events
      expected (:class:`numpy.array`, *float*): Number of expected events

    Raises:
      ValueError: If either array contains a bin with content <= 0.0

    Returns:
      *float*. Calculated Neyman's chi squared
    """
    correct_bin_content, message = check_bin_content(observed)
    if not correct_bin_content:
        raise ValueError("chi_squared.pearson_chi_squared: observed " +
                         message)
    correct_bin_content, message = check_bin_content(expected)
    if not correct_bin_content:
        raise ValueError("chi_squared.pearson_chi_squared: expected " +
                         message)
    return numpy.sum(expected - observed +
                     observed*numpy.log(observed/expected))
<|endoftext|>


<commit_before>#!/bin/python3

from __future__ import print_function

"""
Benchmark creator, for Cassius.

Uses Selenium Webdriver to download new benchmarks for Cassius.
Opens a page in Firefox, causes it to execute get_bench.js, and saves the result.
"""

from selenium import webdriver
import os, sys
import warnings
try:
    import urllib.parse as parse
except:
    import urlparse as parse
import collections
import argparse

def jsfile(name):
    return open(os.path.join(os.path.dirname(__file__), name), "rt").read()

def measure_scrollbar(browser):
    browser.get("about:blank");
    browser.execute_script(jsfile("scrollbar.js") + "; estimate_scrollbar()");

def make_browser():
    profile = webdriver.FirefoxProfile()
    profile.set_preference("security.mixed_content.block_active_content", False)
    profile.set_preference("security.mixed_content.block_display_content", False)
    browser = webdriver.Firefox(firefox_profile=profile)
    measure_scrollbar(browser)
    return browser

def capture(browser, url, id, prerun=None):
    browser.get(url)
    if prerun: browser.execute_script(prerun)
    text = browser.execute_script(jsfile("all.js") + "; return page2text(arguments[0]);", id)
    return ";; From {}\n\n{}\n\n".format(url, text)

def main(urls, prerun=None, fd=None):
    urls = sorted([url if "://" in url else "file://" + os.path.abspath(url)
                   for url in urls])

    for url in urls:
        scheme, _, _, _, _, _ = parse.urlparse(url)
        if scheme not in ["http", "https", "file"]:
            warnings.warn("Only http and file scheme supported (not {})".format(scheme))

    try:
        browser = make_browser()

        print("Saving layout to {}:".format(fd.name), file=sys.stderr, end=" ")
        for i, url in enumerate(urls):
            id = str(i+1).rjust(len(str(len(urls))), "0")
            try:
                fd.write(capture(browser, url, "doc-" + id, prerun=prerun))
                print(id, file=sys.stderr, end=" ")
            except:
                import traceback
                traceback.print_exc()
                continue
        print(file=sys.stderr)
    finally:
        browser.quit()

if __name__ == "__main__":
    p = argparse.ArgumentParser(description="Download a website as Cassius test cases")
    p.add_argument("urls", metavar="URLs", type=str, nargs="+", help="URLs to dowload")
    p.add_argument("--output", type=argparse.FileType('w'), default=sys.stdout, help="File name under bench/.")
    p.add_argument("--prerun", type=argparse.FileType('r'), help="JS file to run before capturing.")
    args = p.parse_args()

    prerun = args.prerun.read() if args.prerun else None
    main(args.urls, prerun=prerun, fd=args.output)
<commit_msg>Handle this error more gracefully<commit_after>#!/bin/python3

from __future__ import print_function

"""
Benchmark creator, for Cassius.

Uses Selenium Webdriver to download new benchmarks for Cassius.
Opens a page in Firefox, causes it to execute get_bench.js, and saves the result.
"""

from selenium import webdriver
import os, sys
import warnings
try:
    import urllib.parse as parse
except:
    import urlparse as parse
import collections
import argparse

def jsfile(name):
    return open(os.path.join(os.path.dirname(__file__), name), "rt").read()

def measure_scrollbar(browser):
    browser.get("about:blank");
    browser.execute_script(jsfile("scrollbar.js") + "; estimate_scrollbar()");

def make_browser():
    profile = webdriver.FirefoxProfile()
    profile.set_preference("security.mixed_content.block_active_content", False)
    profile.set_preference("security.mixed_content.block_display_content", False)
    browser = webdriver.Firefox(firefox_profile=profile)
    measure_scrollbar(browser)
    return browser

def capture(browser, url, id, prerun=None):
    browser.get(url)
    if prerun: browser.execute_script(prerun)
    text = browser.execute_script(jsfile("all.js") + "; return page2text(arguments[0]);", id)
    return ";; From {}\n\n{}\n\n".format(url, text)

def main(urls, prerun=None, fd=None):
    urls = sorted([url if "://" in url else "file://" + os.path.abspath(url)
                   for url in urls])

    for url in urls:
        scheme, _, _, _, _, _ = parse.urlparse(url)
        if scheme not in ["http", "https", "file"]:
            warnings.warn("Only http and file scheme supported (not {})".format(scheme))

    try:
        browser = make_browser()
    else:
        raise

    try:
        print("Saving layout to {}:".format(fd.name), file=sys.stderr, end=" ")
        for i, url in enumerate(urls):
            id = str(i+1).rjust(len(str(len(urls))), "0")
            try:
                fd.write(capture(browser, url, "doc-" + id, prerun=prerun))
                print(id, file=sys.stderr, end=" ")
            except:
                import traceback
                traceback.print_exc()
                continue
        print(file=sys.stderr)
    finally:
        browser.quit()

if __name__ == "__main__":
    p = argparse.ArgumentParser(description="Download a website as Cassius test cases")
    p.add_argument("urls", metavar="URLs", type=str, nargs="+", help="URLs to dowload")
    p.add_argument("--output", type=argparse.FileType('w'), default=sys.stdout, help="File name under bench/.")
    p.add_argument("--prerun", type=argparse.FileType('r'), help="JS file to run before capturing.")
    args = p.parse_args()

    prerun = args.prerun.read() if args.prerun else None
    main(args.urls, prerun=prerun, fd=args.output)
<|endoftext|>


<commit_before>import os
import subprocess

def install_nodejs():
    subprocess.call(["add-apt-repository", "ppa:chris-lea/node.js"])
    subprocess.call(["apt-get", "update"])
    subprocess.call(["apt-get", "-y", "install", "nodejs"])
    subprocess.call(["apt-get", "-y", "install", "npm"])

def install_ipython():
	subprocess.call(["apt-get", "-y", "install", "ipython-notebook"])

def install_upstart():
	subprocess.call(["apt-get", "-y", "install", "upstart", "--force-yes"])

def install_bluez():
	subprocess.call(["apt-get", "-y", "install", "libusb-dev"])
	subprocess.call(["apt-get", "-y", "install", "libdbus-1-dev"])
	subprocess.call(["apt-get", "-y", "install", "libglib2.0-dev"])
	subprocess.call(["apt-get", "-y", "install", "libudev-dev"])
	subprocess.call(["apt-get", "-y", "install", "libical-dev"])
	subprocess.call(["apt-get", "-y", "install", "libreadline-dev"])

    # Install BlueZ
	os.chdir("/home/pi")
	subprocess.call(["mkdir", "bluez"])
	os.chdir("bluez")
	subprocess.call(["wget", "https://www.kernel.org/pub/linux/bluetooth/bluez-5.22.tar.xz"])
	subprocess.call(["unxz", "bluez-5.22.tar.xz"])
	subprocess.call(["tar", "xvf", "bluez-5.22.tar"])
	os.chdir("bluez-5.22")
	subprocess.call(["./configure", "--disable-systemd"])
	subprocess.call(["make"])
	subprocess.call(["make", "install"])

    # Clean up BlueZ build directory
	os.chdir("/home/pi")
	subprocess.call(["rm", "-r", "bluez"])

    # Install BluePy
	subprocess.call(["git", "clone", "https://github.com/IanHarvey/bluepy.git"])
	os.chdir("bluepy/bluepy")
	subprocess.call(["make"])

	subprocess.call(["cp", "bluepy-helper", "/usr/local/lib/python2.7/dist-packages/"])
	subprocess.call(["cp", "btle.py", "/usr/local/lib/python2.7/dist-packages/"])
	subprocess.call(["cp", "sensortag.py", "/usr/local/lib/python2.7/dist-packages/"])

    # Clean up BluePy build directory
	os.chdir("/home/pi")
	subprocess.call(["rm", "-r", "bluepy"])
<commit_msg>Fix node install, improve upstart<commit_after>import os
import subprocess

def install_nodejs():
	subprocess.call('curl -sL https://deb.nodesource.com/setup | bash -', shell=True)
	subprocess.call('apt-get -y install nodejs', shell=True)

def install_ipython():
	subprocess.call(["apt-get", "-y", "install", "ipython-notebook"])

def install_upstart():
	subprocess.call("echo Yes, do as I say! | sudo apt-get -y --force-yes install upstart", shell=True)

def install_bluez():
	subprocess.call(["apt-get", "-y", "install", "libusb-dev"])
	subprocess.call(["apt-get", "-y", "install", "libdbus-1-dev"])
	subprocess.call(["apt-get", "-y", "install", "libglib2.0-dev"])
	subprocess.call(["apt-get", "-y", "install", "libudev-dev"])
	subprocess.call(["apt-get", "-y", "install", "libical-dev"])
	subprocess.call(["apt-get", "-y", "install", "libreadline-dev"])

    # Install BlueZ
	os.chdir("/home/pi")
	subprocess.call(["mkdir", "bluez"])
	os.chdir("bluez")
	subprocess.call(["wget", "https://www.kernel.org/pub/linux/bluetooth/bluez-5.22.tar.xz"])
	subprocess.call(["unxz", "bluez-5.22.tar.xz"])
	subprocess.call(["tar", "xvf", "bluez-5.22.tar"])
	os.chdir("bluez-5.22")
	subprocess.call(["./configure", "--disable-systemd"])
	subprocess.call(["make"])
	subprocess.call(["make", "install"])

    # Clean up BlueZ build directory
	os.chdir("/home/pi")
	subprocess.call(["rm", "-r", "bluez"])

    # Install BluePy
	subprocess.call(["git", "clone", "https://github.com/IanHarvey/bluepy.git"])
	os.chdir("bluepy/bluepy")
	subprocess.call(["make"])

	subprocess.call(["cp", "bluepy-helper", "/usr/local/lib/python2.7/dist-packages/"])
	subprocess.call(["cp", "btle.py", "/usr/local/lib/python2.7/dist-packages/"])
	subprocess.call(["cp", "sensortag.py", "/usr/local/lib/python2.7/dist-packages/"])

    # Clean up BluePy build directory
	os.chdir("/home/pi")
	subprocess.call(["rm", "-r", "bluepy"])
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Standard packages
import sys
import argparse

# Third-party packages
from toil.job import Job

# Package methods
from ngsflow import gatk
from ngsflow import annotation
from ngsflow.align import bwa
from ngsflow.utils import configuration
from ngsflow.utils import utilities
from ngsflow.variation import variation
from ngsflow.variation import freebayes
from ngsflow.variation import mutect
from ngsflow.variation import platypus
from ngsflow.variation import vardict
from ngsflow.variation import scalpel
from ngsflow.variation import indelminer
from ngsflow.variation import pisces

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('-s', '--samples_file', help="Input configuration file for samples")
    parser.add_argument('-c', '--configuration', help="Configuration file for various settings")
    Job.Runner.addToilOptions(parser)
    args = parser.parse_args()
    args.logLevel = "INFO"

    sys.stdout.write("Parsing configuration data\n")
    config = configuration.configure_runtime(args.configuration)

    sys.stdout.write("Parsing sample data\n")
    samples = configuration.configure_samples(args.samples_file, config)

    # Workflow Graph definition. The following workflow definition should create a valid Directed Acyclic Graph (DAG)
    root_job = Job.wrapJobFn(utilities.spawn_batch_jobs, cores=1)

    # Per sample variant calling jobs
    for sample in samples:
        # freebayes_job = Job.wrapJobFn(freebayes.freebayes_single, config, sample, sample['bam'],
        #                               cores=1,
        #                               memory="{}G".format(config['freebayes']['max_mem']))
        #
        # mutect_job = Job.wrapJobFn(mutect.mutect_single, config, sample, sample['bam'],
        #                            cores=1,
        #                            memory="{}G".format(config['mutect']['max_mem']))

        vardict_job = Job.wrapJobFn(vardict.vardict_single, config, sample, samples, sample['bam'],
                                    cores=int(config['vardict']['num_cores']),
                                    memory="{}G".format(config['vardict']['max_mem']))

        scalpel_job = Job.wrapJobFn(scalpel.scalpel_single, config, sample, samples, sample['bam'],
                                    cores=int(config['scalpel']['num_cores']),
                                    memory="{}G".format(config['scalpel']['max_mem']))

        # indelminer_job = Job.wrapJobFn(indelminer.indelminer_single, config, sample, sample['bam'],
        #                                cores=1,
        #                                memory="{}G".format(config['indelminer']['max_mem']))

        # Platypus defined regions not created for all panels. Need to handle stranded pool libraries
        # platypus_job = Job.wrapJobFn(platypus.platypus_single, config, sample, sample['bam'],
        #                              cores=int(config['platypus']['num_cores']),
        #                              memory="{}G".format(config['platypus']['max_mem']))

        # Create workflow from created jobs
        root_job.addChild(freebayes_job)
        root_job.addChild(mutect_job)
        root_job.addChild(vardict_job)
        root_job.addChild(scalpel_job)
        # root_job.addChild(indelminer_job)
        # root_job.addChild(platypus_job)

    # Start workflow execution
    Job.Runner.startToil(root_job, args)
<commit_msg>Tweaked variant calling only algorithm<commit_after>#!/usr/bin/env python

# Standard packages
import sys
import argparse

# Third-party packages
from toil.job import Job

# Package methods
from ngsflow.utils import configuration
from ngsflow.utils import utilities
from ngsflow.variation import freebayes
from ngsflow.variation import mutect
from ngsflow.variation import platypus
from ngsflow.variation import vardict
from ngsflow.variation import scalpel
from ngsflow.variation import indelminer
from ngsflow.variation import pisces

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('-s', '--samples_file', help="Input configuration file for samples")
    parser.add_argument('-c', '--configuration', help="Configuration file for various settings")
    Job.Runner.addToilOptions(parser)
    args = parser.parse_args()
    args.logLevel = "INFO"

    sys.stdout.write("Parsing configuration data\n")
    config = configuration.configure_runtime(args.configuration)

    sys.stdout.write("Parsing sample data\n")
    samples = configuration.configure_samples(args.samples_file, config)

    # Workflow Graph definition. The following workflow definition should create a valid Directed Acyclic Graph (DAG)
    root_job = Job.wrapJobFn(utilities.spawn_batch_jobs, cores=1)

    # Per sample variant calling jobs
    for sample in samples:
        # freebayes_job = Job.wrapJobFn(freebayes.freebayes_single, config, sample, samples[sample]['bam'],
        #                               cores=1,
        #                               memory="{}G".format(config['freebayes']['max_mem']))
        #
        # mutect_job = Job.wrapJobFn(mutect.mutect_single, config, sample, samples[sample]['bam'],
        #                            cores=1,
        #                            memory="{}G".format(config['mutect']['max_mem']))

        vardict_job = Job.wrapJobFn(vardict.vardict_single, config, sample, samples, samples[sample]['bam'],
                                    cores=int(config['vardict']['num_cores']),
                                    memory="{}G".format(config['vardict']['max_mem']))

        scalpel_job = Job.wrapJobFn(scalpel.scalpel_single, config, sample, samples, samples[sample]['bam'],
                                    cores=int(config['scalpel']['num_cores']),
                                    memory="{}G".format(config['scalpel']['max_mem']))

        # indelminer_job = Job.wrapJobFn(indelminer.indelminer_single, config, sample, samples[sample]['bam'],
        #                                cores=1,
        #                                memory="{}G".format(config['indelminer']['max_mem']))

        # Platypus defined regions not created for all panels. Need to handle stranded pool libraries
        # platypus_job = Job.wrapJobFn(platypus.platypus_single, config, sample, samples[sample]['bam'],
        #                              cores=int(config['platypus']['num_cores']),
        #                              memory="{}G".format(config['platypus']['max_mem']))

        # Create workflow from created jobs
        # root_job.addChild(freebayes_job)
        # root_job.addChild(mutect_job)
        root_job.addChild(vardict_job)
        root_job.addChild(scalpel_job)
        # root_job.addChild(indelminer_job)
        # root_job.addChild(platypus_job)

    # Start workflow execution
    Job.Runner.startToil(root_job, args)
<|endoftext|>


<commit_before>import re
from io import StringIO
from pathlib import Path
from typing import Any, NamedTuple, Optional, TextIO, Tuple

from . import output, parse
from .exceptions import (
    ExistingSyntaxErrors,
    FileSkipComment,
    FileSkipSetting,
    IntroducedSyntaxErrors,
    UnableToDetermineEncoding,
)
from .format import remove_whitespace, show_unified_diff, format_natural
from .io import File
from .settings import DEFAULT_CONFIG, FILE_SKIP_COMMENT, Config

IMPORT_START_IDENTIFIERS = ("from ", "from.import", "import ", "import*")

def _config(
    path: Optional[Path] = None, config: Config = DEFAULT_CONFIG, **config_kwargs
) -> Config:
    if path:
        if (
            config is DEFAULT_CONFIG
            and "settings_path" not in config_kwargs
            and "settings_file" not in config_kwargs
        ):
            config_kwargs["settings_path"] = path

    if config_kwargs and config is not DEFAULT_CONFIG:
        raise ValueError(
            "You can either specify custom configuration options using kwargs or "
            "passing in a Config object. Not Both!"
        )
    elif config_kwargs:
        config = Config(**config_kwargs)

    return config

def sorted_imports(
    file_contents: str,
    extension: str = "py",
    config: Config = DEFAULT_CONFIG,
    file_path: Optional[Path] = None,
    disregard_skip: bool = False,
    **config_kwargs,
) -> str:
    config = _config(config=config, **config_kwargs)
    content_source = str(file_path or "Passed in content")
    if not disregard_skip:
        if FILE_SKIP_COMMENT in file_contents:
            raise FileSkipComment(content_source)

        elif file_path and config.is_skipped(file_path):
            raise FileSkipSetting(content_source)

    if config.atomic:
        try:
            compile(file_contents, content_source, "exec", 0, 1)
        except SyntaxError:
            raise ExistingSyntaxErrors(content_source)

    parsed_output = StringIO()
    sort_imports(StringIO(file_contents), parsed_output, extension=extension, config=config)
    parsed_output.seek(0)
    parsed_output = parsed_output.read()
    if config.atomic:
        try:
            compile(file_contents, content_source, "exec", 0, 1)
        except SyntaxError:
            raise IntroducedSyntaxErrors(content_source)
    return parsed_output

def check_imports(
    file_contents: str,
    show_diff: bool = False,
    extension: str = "py",
    config: Config = DEFAULT_CONFIG,
    file_path: Optional[Path] = None,
    disregard_skip: bool = False,
    **config_kwargs,
) -> bool:
    config = _config(config=config, **config_kwargs)

    sorted_output = sorted_imports(
        file_contents=file_contents,
        extension=extension,
        config=config,
        file_path=file_path,
        disregard_skip=disregard_skip,
        **config_kwargs,
    )
    if config.ignore_whitespace:
        line_separator = config.line_ending or parse._infer_line_separator(file_contents)
        compare_in = remove_whitespace(file_contents, line_separator=line_separator).strip()
        compare_out = remove_whitespace(sorted_output, line_separator=line_separator).strip()
    else:
        compare_in = file_contents.strip()
        compare_out = sorted_output.strip()

    if compare_out == compare_in:
        if config.verbose:
            print(f"SUCCESS: {file_path or ''} Everything Looks Good!")
        return True
    else:
        print(f"ERROR: {file_path or ''} Imports are incorrectly sorted.")
        if show_diff:
            show_unified_diff(
                file_input=file_contents, file_output=sorted_output, file_path=file_path
            )
        return False

def sorted_file(filename: str, config: Config = DEFAULT_CONFIG, **config_kwargs) -> str:
    file_data = File.read(filename)
    config = _config(path=file_data.path.parent, config=config)
    return sorted_imports(
        file_contents=file_data.contents,
        extension=file_data.extension,
        config=config,
        file_path=file_data.path,
        **config_kwargs,
    )

def sort_imports(
    input_stream: TextIO,
    output_stream: TextIO,
    extension: str = "py",
    config: Config = DEFAULT_CONFIG,
) -> None:
    """Parses stream identifying sections of contiguous imports and sorting them

    Code with unsorted imports is read from the provided `input_stream`, sorted and then
    outputted to the specified output_stream.

    - `input_stream`: Text stream with unsorted import sections.
    - `output_stream`: Text stream to output sorted inputs into.
    - `config`: Config settings to use when sorting imports. Defaults settings.DEFAULT_CONFIG.
    """
    add_imports = (format_natural(addition) for addition in config.add_imports)
    import_section: str = ""
    in_quote: str = ""
    first_comment_index_start: int = -1
    first_comment_index_end: int = -1
    contains_imports: bool = False
    in_top_comment: bool = False
    section_comments = [f"# {heading}" for heading in config.import_headings.values()]
    for index, line in enumerate(input_stream):
        if index == 1 and line.startswith("#"):
            in_top_comment = True
        elif in_top_comment:
            if not line.startswith("#") or line in section_comments:
                in_top_comment = False
                first_comment_index_end = index - 1
        elif '"' in line or "'" in line:
            char_index = 0
            if first_comment_index_start == -1 and (line.startswith('"') or line.startswith("'")):
                first_comment_index_start = index
            while char_index < len(line):
                if line[char_index] == "\\":
                    char_index += 1
                elif in_quote:
                    if line[char_index : char_index + len(in_quote)] == in_quote:
                        in_quote = ""
                        if first_comment_index_end < first_comment_index_start:
                            first_comment_index_end = index
                elif line[char_index] in ("'", '"'):
                    long_quote = line[char_index : char_index + 3]
                    if long_quote in ('"""', "'''"):
                        in_quote = long_quote
                        char_index += 2
                    else:
                        in_quote = line[char_index]
                elif line[char_index] == "#":
                    break
                char_index += 1

        not_imports = bool(in_quote) or in_top_comment
        if not (in_quote or in_top_comment):
            stripped_line = line.strip()
            if not stripped_line or stripped_line.startswith("#"):
                import_section += line
            elif stripped_line.startswith(IMPORT_START_IDENTIFIERS):
                import_section += line
                if "(" in stripped_line and ")" not in stripped_line:
                    nested_line = line
                    nested_stripped_line = nested_line.strip().split("#")[0]
                    while ")" not in nested_stripped_line:
                        nested_line = input_stream.readline()
                        nested_stripped_line = nested_line.strip()
                        import_section += nested_line

                if stripped_line.endswith("\\"):
                    nested_line = line
                    nested_stripped_line = nested_line.strip()
                    while nested_line and nested_stripped_line.endswith("\\"):
                        nested_line = input_stream.readline()
                        nested_stripped_line = nested_line.strip()
                        import_section += nested_line

                contains_imports = True
            else:
                not_imports = True

        if not_imports:
            if import_section:
                import_section += config.line_ending.join(add_imports)
                import_section += line
                if not contains_imports:
                    output_stream.write(import_section)
                else:
                    output_stream.write(
                        output.sorted_imports(
                            parse.file_contents(import_section, config=config), config, extension
                        )
                    )
                contains_imports = False
                import_section = ""
            else:
                output_stream.write(line)
                not_imports = False

    if import_section:
        if not contains_imports:
            output_stream.write(import_section)
        else:
            import_section += config.line_ending.join(add_imports)
            output_stream.write(
                output.sorted_imports(
                    parse.file_contents(import_section, config=config), config, extension
                )
            )
    output_stream.write(config.line_ending.join(add_imports))
<commit_msg>Fully move add_imports to main sort_imports entry point<commit_after>import re
from io import StringIO
from pathlib import Path
from typing import Any, NamedTuple, Optional, TextIO, Tuple

from . import output, parse
from .exceptions import (
    ExistingSyntaxErrors,
    FileSkipComment,
    FileSkipSetting,
    IntroducedSyntaxErrors,
    UnableToDetermineEncoding,
)
from .format import format_natural, remove_whitespace, show_unified_diff
from .io import File
from .settings import DEFAULT_CONFIG, FILE_SKIP_COMMENT, Config

IMPORT_START_IDENTIFIERS = ("from ", "from.import", "import ", "import*")
COMMENT_INDICATORS = ('"""', "'''", "'", '"', "#")

def _config(
    path: Optional[Path] = None, config: Config = DEFAULT_CONFIG, **config_kwargs
) -> Config:
    if path:
        if (
            config is DEFAULT_CONFIG
            and "settings_path" not in config_kwargs
            and "settings_file" not in config_kwargs
        ):
            config_kwargs["settings_path"] = path

    if config_kwargs and config is not DEFAULT_CONFIG:
        raise ValueError(
            "You can either specify custom configuration options using kwargs or "
            "passing in a Config object. Not Both!"
        )
    elif config_kwargs:
        config = Config(**config_kwargs)

    return config

def sorted_imports(
    file_contents: str,
    extension: str = "py",
    config: Config = DEFAULT_CONFIG,
    file_path: Optional[Path] = None,
    disregard_skip: bool = False,
    **config_kwargs,
) -> str:
    config = _config(config=config, **config_kwargs)
    content_source = str(file_path or "Passed in content")
    if not disregard_skip:
        if FILE_SKIP_COMMENT in file_contents:
            raise FileSkipComment(content_source)

        elif file_path and config.is_skipped(file_path):
            raise FileSkipSetting(content_source)

    if config.atomic:
        try:
            compile(file_contents, content_source, "exec", 0, 1)
        except SyntaxError:
            raise ExistingSyntaxErrors(content_source)

    parsed_output = StringIO()
    sort_imports(StringIO(file_contents), parsed_output, extension=extension, config=config)
    parsed_output.seek(0)
    parsed_output = parsed_output.read()
    if config.atomic:
        try:
            compile(file_contents, content_source, "exec", 0, 1)
        except SyntaxError:
            raise IntroducedSyntaxErrors(content_source)
    return parsed_output

def check_imports(
    file_contents: str,
    show_diff: bool = False,
    extension: str = "py",
    config: Config = DEFAULT_CONFIG,
    file_path: Optional[Path] = None,
    disregard_skip: bool = False,
    **config_kwargs,
) -> bool:
    config = _config(config=config, **config_kwargs)

    sorted_output = sorted_imports(
        file_contents=file_contents,
        extension=extension,
        config=config,
        file_path=file_path,
        disregard_skip=disregard_skip,
        **config_kwargs,
    )
    if config.ignore_whitespace:
        line_separator = config.line_ending or parse._infer_line_separator(file_contents)
        compare_in = remove_whitespace(file_contents, line_separator=line_separator).strip()
        compare_out = remove_whitespace(sorted_output, line_separator=line_separator).strip()
    else:
        compare_in = file_contents.strip()
        compare_out = sorted_output.strip()

    if compare_out == compare_in:
        if config.verbose:
            print(f"SUCCESS: {file_path or ''} Everything Looks Good!")
        return True
    else:
        print(f"ERROR: {file_path or ''} Imports are incorrectly sorted.")
        if show_diff:
            show_unified_diff(
                file_input=file_contents, file_output=sorted_output, file_path=file_path
            )
        return False

def sorted_file(filename: str, config: Config = DEFAULT_CONFIG, **config_kwargs) -> str:
    file_data = File.read(filename)
    config = _config(path=file_data.path.parent, config=config)
    return sorted_imports(
        file_contents=file_data.contents,
        extension=file_data.extension,
        config=config,
        file_path=file_data.path,
        **config_kwargs,
    )

def sort_imports(
    input_stream: TextIO,
    output_stream: TextIO,
    extension: str = "py",
    config: Config = DEFAULT_CONFIG,
) -> None:
    """Parses stream identifying sections of contiguous imports and sorting them

    Code with unsorted imports is read from the provided `input_stream`, sorted and then
    outputted to the specified output_stream.

    - `input_stream`: Text stream with unsorted import sections.
    - `output_stream`: Text stream to output sorted inputs into.
    - `config`: Config settings to use when sorting imports. Defaults settings.DEFAULT_CONFIG.
    """
    line_separator: str = config.line_ending
    add_imports = [format_natural(addition) for addition in config.add_imports]
    import_section: str = ""
    in_quote: str = ""
    first_comment_index_start: int = -1
    first_comment_index_end: int = -1
    contains_imports: bool = False
    in_top_comment: bool = False
    first_import_section: bool = True
    section_comments = [f"# {heading}" for heading in config.import_headings.values()]

    def additional_imports() -> str:
        nonlocal add_imports
        nonlocal line_separator
        nonlocal contains_imports

        if not add_imports:
            return ""

        if not line_separator:
            line_separator = "\n"

        fomatted_imports: str = line_separator.join(add_imports) + line_separator
        contains_imports = True
        add_imports = []
        return fomatted_imports

    index: int = 0
    for index, line in enumerate(input_stream):
        if not line_separator:
            line_separator = line[-1]

        if index == 1 and line.startswith("#"):
            in_top_comment = True
        elif in_top_comment:
            if not line.startswith("#") or line in section_comments:
                in_top_comment = False
                first_comment_index_end = index - 1
        elif '"' in line or "'" in line:
            char_index = 0
            if first_comment_index_start == -1 and (line.startswith('"') or line.startswith("'")):
                first_comment_index_start = index
            while char_index < len(line):
                if line[char_index] == "\\":
                    char_index += 1
                elif in_quote:
                    if line[char_index : char_index + len(in_quote)] == in_quote:
                        in_quote = ""
                        if first_comment_index_end < first_comment_index_start:
                            first_comment_index_end = index
                elif line[char_index] in ("'", '"'):
                    long_quote = line[char_index : char_index + 3]
                    if long_quote in ('"""', "'''"):
                        in_quote = long_quote
                        char_index += 2
                    else:
                        in_quote = line[char_index]
                elif line[char_index] == "#":
                    break
                char_index += 1

        not_imports = bool(in_quote) or in_top_comment
        if not (in_quote or in_top_comment):
            stripped_line = line.strip()
            if not stripped_line or stripped_line.startswith("#"):
                import_section += line
            elif stripped_line.startswith(IMPORT_START_IDENTIFIERS):
                import_section += line
                if "(" in stripped_line and ")" not in stripped_line:
                    nested_line = line
                    nested_stripped_line = nested_line.strip().split("#")[0]
                    while ")" not in nested_stripped_line:
                        nested_line = input_stream.readline()
                        nested_stripped_line = nested_line.strip()
                        import_section += nested_line

                if stripped_line.endswith("\\"):
                    nested_line = line
                    nested_stripped_line = nested_line.strip()
                    while nested_line and nested_stripped_line.endswith("\\"):
                        nested_line = input_stream.readline()
                        nested_stripped_line = nested_line.strip()
                        import_section += nested_line

                contains_imports = True
            else:
                not_imports = True

        if not_imports:
            if not in_top_comment and not in_quote and not import_section and not line.lstrip().startswith(COMMENT_INDICATORS):
                import_section = additional_imports()

            if import_section:
                import_section += additional_imports()
                import_section += line
                if not contains_imports:
                    output_stream.write(import_section)
                else:
                    if first_import_section:
                        import_section = import_section.lstrip(line_separator)
                        first_import_section = False
                    output_stream.write(
                        output.sorted_imports(
                            parse.file_contents(import_section, config=config), config, extension
                        )
                    )
                contains_imports = False
                import_section = ""
            else:
                output_stream.write(line)
                not_imports = False

    if import_section:
        import_section += additional_imports()
        if not contains_imports:
            output_stream.write(import_section)
        else:
            if first_import_section:
                import_section = import_section.lstrip(line_separator)
            output_stream.write(
                output.sorted_imports(
                    parse.file_contents(import_section, config=config), config, extension
                )
            )
    elif index > 1 or config.force_adds:
        output_stream.write(additional_imports())
<|endoftext|>


<commit_before>"""
A set of tools which allow specifying a model consisting of sheets
organized in levels, and projections connecting these sheets. The
sheets have an attribute matchconditions allowing to specify which
other (incoming) sheets a sheet should connect to.

Instances of the RegisteredMethod decorator are offered for setting
parameters/matchconditions for a sheet within a level, as well as
parameters for projections.
"""

from collections import OrderedDict
import itertools
from functools import wraps

import param

import topo
from topo import sheet,transferfn,pattern,projection
from topo.base.sheet import Sheet

from dataviews.collector import AttrTree

from topo.misc.commandline import global_params

class Specification(object):
    @property
    def spec_type(self):
        return self._spec_type

    def update_parameters(self, params):
        self.parameters.update(params)

    def resolve(self):
        """
        Returns the actual projection after it has been instantiated.
        """
        from topo import sim
        return eval('sim.'+str(self))

    def __init__(self, spec_type):
        self._spec_type = spec_type
        self.parameters = {}
        for param_name, default_value in spec_type.params().items():
            self.parameters[param_name]=default_value.default

class SheetSpec(Specification):
    """
    SheetSpec acts as a template for sheet objects. It is also
    possible to resolve the actual sheet object after it has been
    instantiated.

    The following attributes can be accessed:

    :'properties': Dictionary specifying the properties of the
    sheet. There must be a value given for the key 'level'.
    :'spec_type': Subclass of topo.base.sheet.Sheet.(read-only)
    :'parameters': Dictionary specifying which parameters should be
    passed to the sheet object specified with spec_type. Keys are the
    parameter names, values the parameter values.
    :'matchconditions': Dictionary specifying the matchconditions of
    the sheet. This may be used to determine which other sheets this
    sheet should connect to.
    """

    name_ordering = ['eye','level', 'cone', 'polarity','SF','opponent','surround']

    @property
    def level(self):
        return self.properties['level']

    def __init__(self, spec_type, properties):
        """
        Initialize a SheetSpec object. All arguments but parameters
        are just passed to the internal attributes. For parameters,
        additional key-value pairs with all possible parameters for
        the sheet type specified with sheet_type are added. This
        allows a lookup which parameters can be set.
        """
        super(SheetSpec,self).__init__(spec_type)

        if 'level' not in properties:
            raise Exception("SheetSpec always requires 'level' in properties")

        properties = [(k, properties[k]) for k in self.name_ordering
                      if k in properties]

        self.properties = OrderedDict(properties)
        self.matchconditions={}

    def update_matchconditions(self, matchconditions):
        self.matchconditions.update(matchconditions)

    def __call__(self):
        """
        Instantiate the sheet and register it in topo.sim.
        """
        topo.sim[str(self)]=self.spec_type(**self.parameters)

    def resolve(self):
        """
        Returns the actual sheet object after it has been instantiated.
        """
        from topo import sim
        return eval('sim.'+str(self))

    def __str__(self):
        """
        Returns a string representation combined from the properties values.
        This might be used as name for the actual sheet object.
        """
        name=''
        for prop in self.properties.itervalues():
            name+=prop

        return name

    def __repr__(self):
        type_name = self._spec_type.__name__
        properties_repr = ', '.join("%r:%r" % (k,v) for (k,v)
                                    in self.properties.items())
        return "SheetSpec(%s, {%s})" % (type_name, properties_repr)

class ProjectionSpec(Specification):
    """
    ProjectionSpec acts as a template for projections. It is also
    possible to resolve the actual projection after it has been
    instantiated.

    The following attributes can be accessed:

    :'src': SheetSpec of the source sheet
    :'dest': SheetSpec of the destination sheet
    :'spec_type': Subclass of topo.base.projection.Projection
    :'match_name': Name of the matchcondition which has been used to
    set this projection up.  This might be used to set the parameters
    of the ProjectionSpec
    :'parameters': Dictionary specifying which parameters should be
    passed to the projection specified with connection_type. Keys are
    the parameter names, values the parameter values.
    """

    def __init__(self, spec_type, src, dest, match_name=None, properties=None):
        """
        Initialize a ProjectionSpec object. All arguments but
        parameters are just passed to the internal attributes. For
        parameters, additional key-value pairs with all possible
        parameters for the projection type specified with
        connection_type are added. This allows a lookup which
        parameters can be set. Furthermore, parameters['src'] and
        parameters['dest'] are removed, as these must be set with the
        'src' and 'dest' arguments instead.
        """
        super(ProjectionSpec, self).__init__(spec_type)

        self.src = src
        self.dest = dest
        self.match_name = match_name

        self.properties = {} if properties is None else properties
        # These parameters are directly passed into topo.sim.connect()!
        ignored_keys = ['src', 'dest']
        self.parameters = dict((k,v) for (k,v) in self.parameters.items()
                               if k not in ignored_keys)

    def __call__(self):
        """
        Instantiate the projection and register it in topo.sim.
        """
        topo.sim.connect(str(self.src),str(self.dest),self.spec_type,
                         **self.parameters)

    def __str__(self):
        return str(self.dest)+'.'+self.parameters['name']

    def __repr__(self):
        type_name = self._spec_type.__name__
        properties_repr = "{"+ ', '.join("%r:%r" % (k,v) for (k,v) in self.properties.items()) +"}"
        return "ProjectionSpec(%s, %r, %r, %r%s)" % (type_name, self.src, self.dest,
                                                       self.match_name,
                                                       ", %s" % properties_repr if self.properties else '')

class LabelDecorator(object):
    """
    Decorator class which can be instantiated to create a decorator
    object. This object can then be used to decorate methods or
    functions with a label argument and optionally a type.

    After decorating several methods or functions, the dictionary of
    all the decorated callables can be accessed via the labelled
    attribute. Any types supplies are accessible through the types
    attribute.
    """
    def __init__(self):
        self.labelled = {}
        self.types = {}

    def __call__(self, label, obj_type=None):
        def decorator(f):
            @wraps(f)
            def inner(*args, **kwargs):
                return f(*args, **kwargs)

            self.labelled[label] = inner
            if obj_type is not None:
                self.types[label] = obj_type
            return inner
        return decorator

class Model(param.Parameterized):
    """
    The available setup options are:

        :'training_patterns': fills the training_patterns AttrTree
        with pattern generator instances. The path is the name of the
        input sheet. Usually calls PatternCoordinator to do this.
        :'setup_sheets': determines the number of sheets, their types
        and names sets sheet parameters according to the registered
        methods in level sets sheet matchconditions according to the
        registered methods in matchconditions
        :'projections': determines which connections should be present
        between the sheets according to the matchconditions of
        SheetSpec objects, using connect to specify the
        connection type and sets their parameters according to the
        registered methods in connect

    The available instantiate options are:

        :'sheets': instantiates all sheets and registers them in
        topo.sim
        :'projections': instantiates all projections and registers
        them in topo.sim
    """
    __abstract = True

    level =           LabelDecorator()
    matchconditions = LabelDecorator()
    connect =         LabelDecorator()

    def _register_global_params(self, params):
        """
        Register the parameters of this object as global parameters
        available for users to set from the command line.  Values
        supplied as global parameters will override those of the given
        dictionary of params.
        """

        for param_name, param_value in self.params().items():
            global_params.add(**{param_name:param_value})

        params.update(global_params.get_param_values())

    def __init__(self, setup_options=True, register=True, **params):
        if register:
            self._register_global_params(params)
        super(Model,self).__init__(**params)

        self.training_patterns = AttrTree()
        self.sheets = AttrTree()
        self.projections = AttrTree()

        self.setup(setup_options)

    #==============================================#
    # Public methods to be implemented by modelers #
    #==============================================#

    def setup_attributes(self):
        """
        Method to precompute any useful self attributes from the class
        parameters. For instance, if there is a ``num_lags``
        parameter, this method could compute the actual projection
        delays and store it in self.lags.
        """
        pass

    def setup_training_patterns(self):
        """
        Adds new PatternGenerators to self.training_patterns, with
        the name of the input sheet where the training patterns should
        be installed as path.
        """
        raise NotImplementedError

    def setup_sheets(self):
        """
        Adds new SheetSpec items to the self.sheets AttrTree.
        Must return a list of SheetSpec objects.
        """
        raise NotImplementedError

    def setup_analysis(self):
        """
        Set up appropriate defaults for analysis functions in topo.analysis.featureresponses
        """
        pass

    #====================================================#
    # Remaining methods should not need to be overridden #
    #====================================================#

    def setup(self,setup_options):
        """
        This method can be used to setup certain aspects of the
        submodel.  If setup_options=True, all setup methods are
        called.  setup_options can also be a list, whereas all list
        items of available_setup_options are accepted.

        Available setup options are:
        'training_patterns','sheets','projections' and 'analysis'.

        Please consult the docstring of the Model class for more
        information about each setup option.
        """
        available_setup_options = ['attributes',
                                   'training_patterns',
                                   'sheets',
                                   'projections',
                                   'analysis']

        if setup_options==True:
            setup_options = available_setup_options

        if 'attributes' in setup_options:
            self.setup_attributes()
        if 'training_patterns' in setup_options:
            self.setup_training_patterns()
        if 'sheets' in setup_options:
            sheet_specs = self.setup_sheets()
            for sheet_spec in sheet_specs:
                self.sheets.set_path(str(sheet_spec), sheet_spec)
            self._update_sheet_specs()
        if 'projections' in setup_options:
            self._compute_projection_specs()
        if 'analysis' in setup_options:
            self._setup_analysis()

    def _update_sheet_specs(self):
        for sheet_spec in self.sheets.path_items.values():
            param_method = self.level.labelled.get(sheet_spec.level, None)
            if not param_method:
                raise Exception("Parameters for sheet level %r not specified" % sheet_spec.level)

            updated_params = param_method(self,sheet_spec.properties)
            sheet_spec.update_parameters(updated_params)

            matchcondition = self.matchconditions.labelled.get(sheet_spec.level, False)
            if matchcondition:
                sheet_spec.update_matchconditions(matchcondition(self,sheet_spec.properties))

    def _matchcondition_applies(self, matchconditions, src_sheet):
        """
        Given a dictionary of properties to match and a target sheet
        spec, test if the matchcondition applies.
        """
        matches=True
        if matchconditions is None:
            return False
        for incoming_key, incoming_value in matchconditions.items():
            if incoming_key in src_sheet.properties and \
                    str(src_sheet.properties[incoming_key]) not in str(incoming_value):
                matches=False
                break

        return matches

    def _compute_projection_specs(self):
        """
        Loop through all possible combinations of SheetSpec objects in
        self.sheets If the src_sheet fulfills all criteria specified
        in dest_sheet.matchconditions, create a new ProjectionSpec
        object and add this item to self.projections.
        """

        sheetspec_product = itertools.product(self.sheets.path_items.values(),
                                              self.sheets.path_items.values())
        for src_sheet, dest_sheet in sheetspec_product:
            for matchname, matchconditions in dest_sheet.matchconditions.items():

                if self._matchcondition_applies(matchconditions, src_sheet):
                    proj = ProjectionSpec(self.connect.types[matchname],
                                          src_sheet, dest_sheet, matchname)
                    paramsets = self.connect.labelled[matchname](self, proj)
                    paramsets = [paramsets] if isinstance(paramsets, dict) else paramsets
                    for paramset in paramsets:
                        proj = ProjectionSpec(self.connect.types[matchname],
                                              src_sheet, dest_sheet, matchname)
                        proj.update_parameters(paramset)
                        path = (str(dest_sheet), str(src_sheet), paramset['name'])
                        self.projections.set_path(path, proj)

    def __call__(self,instantiate_options=True):
        """
        Instantiates all sheets or projections in self.sheets or
        self.projections and registers them in the topo.sim instance.

        If instantiate_options=True, all items are initialised
        instantiate_options can also be a list, whereas all list items
        of available_instantiate_options are accepted.

        Available instantiation options are: 'sheets' and
        'projections'.

        Please consult the docstring of the Model class for more
        information about each instantiation option.
        """

        available_instantiate_options = ['sheets','projections']
        if instantiate_options==True:
            instantiate_options=available_instantiate_options

        if 'sheets' in instantiate_options:
            self.message('Sheets:\n')
            for sheet_spec in self.sheets.path_items.itervalues():
                self.message(sheet_spec)
                sheet_spec()
            self.message('\n\n')

        if 'projections' in instantiate_options:
            self.message('Connections:\n')
            #Ugly hack to take the order to initialize sheets into account
            #As soon as weight initialization is done with time_dependent=True,
            #this can be simplified to:
            #for proj in self.projections.path_items.itervalues():
            connection_order=['Afferent','AfferentCenter','AfferentSurround',
                              'LateralGC','AfferentV1On','AfferentV1Off',
                              'LateralV1Excitatory','LateralV1Inhibitory']
            for proj in sorted(self.projections.path_items.itervalues(),
                               key=lambda projection: connection_order.index(projection.match_name)):
                self.message('Connect ' + str(proj.src) + ' with ' + str(proj.dest) + \
                             ' (Match name: ' + proj.match_name + \
                             ', connection name: ' + str(proj.parameters['name']) + ')')
                proj()
<commit_msg>Fixed docstring<commit_after>"""
A set of tools which allow specifying a model consisting of sheets
organized in levels, and projections connecting these sheets. The
sheets have an attribute matchconditions allowing to specify which
other (incoming) sheets a sheet should connect to.

Instances of the RegisteredMethod decorator are offered for setting
parameters/matchconditions for a sheet within a level, as well as
parameters for projections.
"""

from collections import OrderedDict
import itertools
from functools import wraps

import param

import topo
from topo import sheet,transferfn,pattern,projection
from topo.base.sheet import Sheet

from dataviews.collector import AttrTree

from topo.misc.commandline import global_params

class Specification(object):
    @property
    def spec_type(self):
        return self._spec_type

    def update_parameters(self, params):
        self.parameters.update(params)

    def resolve(self):
        """
        Returns the actual projection after it has been instantiated.
        """
        from topo import sim
        return eval('sim.'+str(self))

    def __init__(self, spec_type):
        self._spec_type = spec_type
        self.parameters = {}
        for param_name, default_value in spec_type.params().items():
            self.parameters[param_name]=default_value.default

class SheetSpec(Specification):
    """
    SheetSpec acts as a template for sheet objects. It is also
    possible to resolve the actual sheet object after it has been
    instantiated.

    The following attributes can be accessed:

    :'properties': Dictionary specifying the properties of the
    sheet. There must be a value given for the key 'level'.
    :'spec_type': Subclass of topo.base.sheet.Sheet.(read-only)
    :'parameters': Dictionary specifying which parameters should be
    passed to the sheet object specified with spec_type. Keys are the
    parameter names, values the parameter values.
    :'matchconditions': Dictionary specifying the matchconditions of
    the sheet. This may be used to determine which other sheets this
    sheet should connect to.
    """

    name_ordering = ['eye','level', 'cone', 'polarity','SF','opponent','surround']

    @property
    def level(self):
        return self.properties['level']

    def __init__(self, spec_type, properties):
        """
        Initialize a SheetSpec object. All arguments but parameters
        are just passed to the internal attributes. For parameters,
        additional key-value pairs with all possible parameters for
        the sheet type specified with sheet_type are added. This
        allows a lookup which parameters can be set.
        """
        super(SheetSpec,self).__init__(spec_type)

        if 'level' not in properties:
            raise Exception("SheetSpec always requires 'level' in properties")

        properties = [(k, properties[k]) for k in self.name_ordering
                      if k in properties]

        self.properties = OrderedDict(properties)
        self.matchconditions={}

    def update_matchconditions(self, matchconditions):
        self.matchconditions.update(matchconditions)

    def __call__(self):
        """
        Instantiate the sheet and register it in topo.sim.
        """
        topo.sim[str(self)]=self.spec_type(**self.parameters)

    def resolve(self):
        """
        Returns the actual sheet object after it has been instantiated.
        """
        from topo import sim
        return eval('sim.'+str(self))

    def __str__(self):
        """
        Returns a string representation combined from the properties values.
        This might be used as name for the actual sheet object.
        """
        name=''
        for prop in self.properties.itervalues():
            name+=prop

        return name

    def __repr__(self):
        type_name = self._spec_type.__name__
        properties_repr = ', '.join("%r:%r" % (k,v) for (k,v)
                                    in self.properties.items())
        return "SheetSpec(%s, {%s})" % (type_name, properties_repr)

class ProjectionSpec(Specification):
    """
    ProjectionSpec acts as a template for projections. It is also
    possible to resolve the actual projection after it has been
    instantiated.

    The following attributes can be accessed:

    :'src': SheetSpec of the source sheet
    :'dest': SheetSpec of the destination sheet
    :'spec_type': Subclass of topo.base.projection.Projection
    :'match_name': Name of the matchcondition which has been used to
    set this projection up.  This might be used to set the parameters
    of the ProjectionSpec
    :'parameters': Dictionary specifying which parameters should be
    passed to the projection specified with connection_type. Keys are
    the parameter names, values the parameter values.
    """

    def __init__(self, spec_type, src, dest, match_name=None, properties=None):
        """
        Initialize a ProjectionSpec object. All arguments but
        parameters are just passed to the internal attributes. For
        parameters, additional key-value pairs with all possible
        parameters for the projection type specified with
        connection_type are added. This allows a lookup which
        parameters can be set. Furthermore, parameters['src'] and
        parameters['dest'] are removed, as these must be set with the
        'src' and 'dest' arguments instead.
        """
        super(ProjectionSpec, self).__init__(spec_type)

        self.src = src
        self.dest = dest
        self.match_name = match_name

        self.properties = {} if properties is None else properties
        # These parameters are directly passed into topo.sim.connect()!
        ignored_keys = ['src', 'dest']
        self.parameters = dict((k,v) for (k,v) in self.parameters.items()
                               if k not in ignored_keys)

    def __call__(self):
        """
        Instantiate the projection and register it in topo.sim.
        """
        topo.sim.connect(str(self.src),str(self.dest),self.spec_type,
                         **self.parameters)

    def __str__(self):
        return str(self.dest)+'.'+self.parameters['name']

    def __repr__(self):
        type_name = self._spec_type.__name__
        properties_repr = "{"+ ', '.join("%r:%r" % (k,v) for (k,v) in self.properties.items()) +"}"
        return "ProjectionSpec(%s, %r, %r, %r%s)" % (type_name, self.src, self.dest,
                                                       self.match_name,
                                                       ", %s" % properties_repr if self.properties else '')

class LabelDecorator(object):
    """
    Decorator class which can be instantiated to create a decorator
    object. This object can then be used to decorate methods or
    functions with a label argument and optionally a type.

    After decorating several methods or functions, the dictionary of
    all the decorated callables can be accessed via the labelled
    attribute. Any types supplies are accessible through the types
    attribute.
    """
    def __init__(self):
        self.labelled = {}
        self.types = {}

    def __call__(self, label, obj_type=None):
        def decorator(f):
            @wraps(f)
            def inner(*args, **kwargs):
                return f(*args, **kwargs)

            self.labelled[label] = inner
            if obj_type is not None:
                self.types[label] = obj_type
            return inner
        return decorator

class Model(param.Parameterized):
    """
    The available setup options are:

        :'training_patterns': fills the training_patterns AttrTree
        with pattern generator instances. The path is the name of the
        input sheet. Usually calls PatternCoordinator to do this.
        :'setup_sheets': determines the number of sheets, their types
        and names sets sheet parameters according to the registered
        methods in level sets sheet matchconditions according to the
        registered methods in matchconditions
        :'projections': determines which connections should be present
        between the sheets according to the matchconditions of
        SheetSpec objects, using connect to specify the
        connection type and sets their parameters according to the
        registered methods in connect

    The available instantiate options are:

        :'sheets': instantiates all sheets and registers them in
        topo.sim
        :'projections': instantiates all projections and registers
        them in topo.sim
    """
    __abstract = True

    level =           LabelDecorator()
    matchconditions = LabelDecorator()
    connect =         LabelDecorator()

    def _register_global_params(self, params):
        """
        Register the parameters of this object as global parameters
        available for users to set from the command line.  Values
        supplied as global parameters will override those of the given
        dictionary of params.
        """

        for param_name, param_value in self.params().items():
            global_params.add(**{param_name:param_value})

        params.update(global_params.get_param_values())

    def __init__(self, setup_options=True, register=True, **params):
        if register:
            self._register_global_params(params)
        super(Model,self).__init__(**params)

        self.training_patterns = AttrTree()
        self.sheets = AttrTree()
        self.projections = AttrTree()

        self.setup(setup_options)

    #==============================================#
    # Public methods to be implemented by modelers #
    #==============================================#

    def setup_attributes(self):
        """
        Method to precompute any useful self attributes from the class
        parameters. For instance, if there is a ``num_lags``
        parameter, this method could compute the actual projection
        delays and store it in self.lags.

        In addition, this method can be used to configure class
        attributes of the model components.
        """
        pass

    def setup_training_patterns(self):
        """
        Adds new PatternGenerators to self.training_patterns, with
        the name of the input sheet where the training patterns should
        be installed as path.
        """
        raise NotImplementedError

    def setup_sheets(self):
        """
        Adds new SheetSpec items to the self.sheets AttrTree.
        Must return a list of SheetSpec objects.
        """
        raise NotImplementedError

    def setup_analysis(self):
        """
        Set up appropriate defaults for analysis functions in topo.analysis.featureresponses
        """
        pass

    #====================================================#
    # Remaining methods should not need to be overridden #
    #====================================================#

    def setup(self,setup_options):
        """
        This method can be used to setup certain aspects of the
        submodel.  If setup_options=True, all setup methods are
        called.  setup_options can also be a list, whereas all list
        items of available_setup_options are accepted.

        Available setup options are:
        'training_patterns','sheets','projections' and 'analysis'.

        Please consult the docstring of the Model class for more
        information about each setup option.
        """
        available_setup_options = ['attributes',
                                   'training_patterns',
                                   'sheets',
                                   'projections',
                                   'analysis']

        if setup_options==True:
            setup_options = available_setup_options

        if 'attributes' in setup_options:
            self.setup_attributes()
        if 'training_patterns' in setup_options:
            self.setup_training_patterns()
        if 'sheets' in setup_options:
            sheet_specs = self.setup_sheets()
            for sheet_spec in sheet_specs:
                self.sheets.set_path(str(sheet_spec), sheet_spec)
            self._update_sheet_specs()
        if 'projections' in setup_options:
            self._compute_projection_specs()
        if 'analysis' in setup_options:
            self._setup_analysis()

    def _update_sheet_specs(self):
        for sheet_spec in self.sheets.path_items.values():
            param_method = self.level.labelled.get(sheet_spec.level, None)
            if not param_method:
                raise Exception("Parameters for sheet level %r not specified" % sheet_spec.level)

            updated_params = param_method(self,sheet_spec.properties)
            sheet_spec.update_parameters(updated_params)

            matchcondition = self.matchconditions.labelled.get(sheet_spec.level, False)
            if matchcondition:
                sheet_spec.update_matchconditions(matchcondition(self,sheet_spec.properties))

    def _matchcondition_applies(self, matchconditions, src_sheet):
        """
        Given a dictionary of properties to match and a target sheet
        spec, test if the matchcondition applies.
        """
        matches=True
        if matchconditions is None:
            return False
        for incoming_key, incoming_value in matchconditions.items():
            if incoming_key in src_sheet.properties and \
                    str(src_sheet.properties[incoming_key]) not in str(incoming_value):
                matches=False
                break

        return matches

    def _compute_projection_specs(self):
        """
        Loop through all possible combinations of SheetSpec objects in
        self.sheets If the src_sheet fulfills all criteria specified
        in dest_sheet.matchconditions, create a new ProjectionSpec
        object and add this item to self.projections.
        """

        sheetspec_product = itertools.product(self.sheets.path_items.values(),
                                              self.sheets.path_items.values())
        for src_sheet, dest_sheet in sheetspec_product:
            for matchname, matchconditions in dest_sheet.matchconditions.items():

                if self._matchcondition_applies(matchconditions, src_sheet):
                    proj = ProjectionSpec(self.connect.types[matchname],
                                          src_sheet, dest_sheet, matchname)
                    paramsets = self.connect.labelled[matchname](self, proj)
                    paramsets = [paramsets] if isinstance(paramsets, dict) else paramsets
                    for paramset in paramsets:
                        proj = ProjectionSpec(self.connect.types[matchname],
                                              src_sheet, dest_sheet, matchname)
                        proj.update_parameters(paramset)
                        path = (str(dest_sheet), str(src_sheet), paramset['name'])
                        self.projections.set_path(path, proj)

    def __call__(self,instantiate_options=True):
        """
        Instantiates all sheets or projections in self.sheets or
        self.projections and registers them in the topo.sim instance.

        If instantiate_options=True, all items are initialised
        instantiate_options can also be a list, whereas all list items
        of available_instantiate_options are accepted.

        Available instantiation options are: 'sheets' and
        'projections'.

        Please consult the docstring of the Model class for more
        information about each instantiation option.
        """

        available_instantiate_options = ['sheets','projections']
        if instantiate_options==True:
            instantiate_options=available_instantiate_options

        if 'sheets' in instantiate_options:
            self.message('Sheets:\n')
            for sheet_spec in self.sheets.path_items.itervalues():
                self.message(sheet_spec)
                sheet_spec()
            self.message('\n\n')

        if 'projections' in instantiate_options:
            self.message('Connections:\n')
            #Ugly hack to take the order to initialize sheets into account
            #As soon as weight initialization is done with time_dependent=True,
            #this can be simplified to:
            #for proj in self.projections.path_items.itervalues():
            connection_order=['Afferent','AfferentCenter','AfferentSurround',
                              'LateralGC','AfferentV1On','AfferentV1Off',
                              'LateralV1Excitatory','LateralV1Inhibitory']
            for proj in sorted(self.projections.path_items.itervalues(),
                               key=lambda projection: connection_order.index(projection.match_name)):
                self.message('Connect ' + str(proj.src) + ' with ' + str(proj.dest) + \
                             ' (Match name: ' + proj.match_name + \
                             ', connection name: ' + str(proj.parameters['name']) + ')')
                proj()
<|endoftext|>