<commit_before># encoding=utf8
import datetime
from distutils.version import StrictVersion
import hashlib
import os.path
import random
from seesaw.config import realize, NumberConfigValue
from seesaw.item import ItemInterpolation, ItemValue
from seesaw.task import SimpleTask, LimitConcurrent
from seesaw.tracker import GetItemFromTracker, PrepareStatsForTracker, \
    UploadWithTracker, SendDoneToTracker
import shutil
import socket
import subprocess
import sys
import time

import seesaw
from seesaw.externalprocess import WgetDownload
from seesaw.pipeline import Pipeline
from seesaw.project import Project
from seesaw.util import find_executable

# check the seesaw version
if StrictVersion(seesaw.__version__) < StrictVersion("0.1.5"):
    raise Exception("This pipeline needs seesaw version 0.1.5 or higher.")

###########################################################################
# Find a useful Wget+Lua executable.
#
# WGET_LUA will be set to the first path that
# 1. does not crash with --version, and
# 2. prints the required version string
WGET_LUA = find_executable(
    "Wget+Lua",
    ["GNU Wget 1.14.lua.20130523-9a5c"],
    [
        "./wget-lua",
        "./wget-lua-warrior",
        "./wget-lua-local",
        "../wget-lua",
        "../../wget-lua",
        "/home/warrior/wget-lua",
        "/usr/bin/wget-lua"
    ]
)

if not WGET_LUA:
    raise Exception("No usable Wget+Lua found.")

###########################################################################
# The version number of this pipeline definition.
#
# Update this each time you make a non-cosmetic change.
# It will be added to the WARC files and reported to the tracker.
VERSION = "20140827.01"
USER_AGENT = 'ArchiveTeam'
TRACKER_ID = 'tele2'
TRACKER_HOST = 'tracker.archiveteam.org'

###########################################################################
# This section defines project-specific tasks.
#
# Simple tasks (tasks that do not need any concurrency) are based on the
# SimpleTask class and have a process(item) method that is called for
# each item.
class CheckIP(SimpleTask):
    def __init__(self):
        SimpleTask.__init__(self, "CheckIP")
        self._counter = 0

    def process(self, item):
        # NEW for 2014! Check if we are behind firewall/proxy

        if self._counter <= 0:
            item.log_output('Checking IP address.')
            ip_set = set()

            ip_set.add(socket.gethostbyname('twitter.com'))
            ip_set.add(socket.gethostbyname('facebook.com'))
            ip_set.add(socket.gethostbyname('youtube.com'))
            ip_set.add(socket.gethostbyname('microsoft.com'))
            ip_set.add(socket.gethostbyname('icanhas.cheezburger.com'))
            ip_set.add(socket.gethostbyname('archiveteam.org'))

            if len(ip_set) != 6:
                item.log_output('Got IP addresses: {0}'.format(ip_set))
                item.log_output(
                    'Are you behind a firewall/proxy? That is a big no-no!')
                raise Exception(
                    'Are you behind a firewall/proxy? That is a big no-no!')

        # Check only occasionally
        if self._counter <= 0:
            self._counter = 10
        else:
            self._counter -= 1

class PrepareDirectories(SimpleTask):
    def __init__(self, warc_prefix):
        SimpleTask.__init__(self, "PrepareDirectories")
        self.warc_prefix = warc_prefix

    def process(self, item):
        item_name = item["item_name"]
        escaped_item_name = item_name.replace(':', '_').replace('/', '_').replace('~', '_')
        dirname = "/".join((item["data_dir"], escaped_item_name))

        if os.path.isdir(dirname):
            shutil.rmtree(dirname)

        os.makedirs(dirname)

        item["item_dir"] = dirname
        item["warc_file_base"] = "%s-%s-%s" % (self.warc_prefix, escaped_item_name,
            time.strftime("%Y%m%d-%H%M%S"))

        open("%(item_dir)s/%(warc_file_base)s.warc.gz" % item, "w").close()

class MoveFiles(SimpleTask):
    def __init__(self):
        SimpleTask.__init__(self, "MoveFiles")

    def process(self, item):
        # NEW for 2014! Check if wget was compiled with zlib support
        if os.path.exists("%(item_dir)s/%(warc_file_base)s.warc"):
            raise Exception('Please compile wget with zlib support!')

        os.rename("%(item_dir)s/%(warc_file_base)s.warc.gz" % item,
              "%(data_dir)s/%(warc_file_base)s.warc.gz" % item)

        shutil.rmtree("%(item_dir)s" % item)

def get_hash(filename):
    with open(filename, 'rb') as in_file:
        return hashlib.sha1(in_file.read()).hexdigest()

CWD = os.getcwd()
PIPELINE_SHA1 = get_hash(os.path.join(CWD, 'pipeline.py'))
LUA_SHA1 = get_hash(os.path.join(CWD, 'tele2.lua'))

def stats_id_function(item):
    # NEW for 2014! Some accountability hashes and stats.
    d = {
        'pipeline_hash': PIPELINE_SHA1,
        'lua_hash': LUA_SHA1,
        'python_version': sys.version,
    }

    return d

class WgetArgs(object):
    def realize(self, item):
        wget_args = [
            WGET_LUA,
            "-U", USER_AGENT,
            "-nv",
            "--lua-script", "tele2.lua",
            "-o", ItemInterpolation("%(item_dir)s/wget.log"),
            "--no-check-certificate",
            "--output-document", ItemInterpolation("%(item_dir)s/wget.tmp"),
            "--truncate-output",
            "-e", "robots=off",
            "--no-cookies",
            "--rotate-dns",
            "--recursive", "--level=inf",
            "--no-parent",
            "--page-requisites",
            "--timeout", "30",
            "--tries", "inf",
            "--span-hosts",
            "--waitretry", "30",
            "--domains", "tele2.se",
            "--warc-file", ItemInterpolation("%(item_dir)s/%(warc_file_base)s"),
            "--warc-header", "operator: Archive Team",
            "--warc-header", "tele2-dld-script-version: " + VERSION,
            "--warc-header", ItemInterpolation("tele2-user: %(item_name)s"),
        ]

        item_name = item['item_name']
        assert ':' in item_name
        item_type, item_value = item_name.split(':', 1)

        assert item_type
        assert item_value

        item['item_type'] = item_type
        item['item_value'] = item_value

        wget_args.append('http://{0}.tele2.se/{1}/'.format(item_type, item_value))

        # wget_args.append('http://home.tele2.se/{0}/'.format(item_name))

        if 'bind_address' in globals():
            wget_args.extend(['--bind-address', globals()['bind_address']])
            print('')
            print('*** Wget will bind address at {0} ***'.format(
                globals()['bind_address']))
            print('')

        return realize(wget_args, item)

###########################################################################
# Initialize the project.
#
# This will be shown in the warrior management panel. The logo should not
# be too big. The deadline is optional.
project = Project(
    title="Tele2",
    project_html="""

home.tele2.se Website · Leaderboard

Archiving Swedish websites from home.tele2.se.

    """,
    utc_deadline=datetime.datetime(2014, 8, 31, 23, 59, 0)
)

pipeline = Pipeline(
    CheckIP(),
    GetItemFromTracker("http://%s/%s" % (TRACKER_HOST, TRACKER_ID), downloader,
        VERSION),
    PrepareDirectories(warc_prefix="tele2"),
    WgetDownload(
        WgetArgs(),
        max_tries=2,
        accept_on_exit_code=[0, 8],
        env={
            "item_dir": ItemValue("item_dir"),
            "item_value": ItemValue("item_value"),
            "item_type": ItemValue("item_type"),
        }
    ),
    PrepareStatsForTracker(
        defaults={"downloader": downloader, "version": VERSION},
        file_groups={
            "data": [
                ItemInterpolation("%(item_dir)s/%(warc_file_base)s.warc.gz")
            ]
        },
        id_function=stats_id_function,
    ),
    MoveFiles(),
    LimitConcurrent(NumberConfigValue(min=1, max=4, default="1",
        name="shared:rsync_threads", title="Rsync threads",
        description="The maximum number of concurrent uploads."),
        UploadWithTracker(
            "http://%s/%s" % (TRACKER_HOST, TRACKER_ID),
            downloader=downloader,
            version=VERSION,
            files=[
                ItemInterpolation("%(data_dir)s/%(warc_file_base)s.warc.gz")
            ],
            rsync_target_source_path=ItemInterpolation("%(data_dir)s/"),
            rsync_extra_args=[
                "--recursive",
                "--partial",
                "--partial-dir", ".rsync-tmp",
            ]
            ),
    ),
    SendDoneToTracker(
        tracker_url="http://%s/%s" % (TRACKER_HOST, TRACKER_ID),
        stats=ItemValue("stats")
    )
)
<commit_msg>pipeline.py: bump version<commit_after># encoding=utf8
import datetime
from distutils.version import StrictVersion
import hashlib
import os.path
import random
from seesaw.config import realize, NumberConfigValue
from seesaw.item import ItemInterpolation, ItemValue
from seesaw.task import SimpleTask, LimitConcurrent
from seesaw.tracker import GetItemFromTracker, PrepareStatsForTracker, \
    UploadWithTracker, SendDoneToTracker
import shutil
import socket
import subprocess
import sys
import time

import seesaw
from seesaw.externalprocess import WgetDownload
from seesaw.pipeline import Pipeline
from seesaw.project import Project
from seesaw.util import find_executable

# check the seesaw version
if StrictVersion(seesaw.__version__) < StrictVersion("0.1.5"):
    raise Exception("This pipeline needs seesaw version 0.1.5 or higher.")

###########################################################################
# Find a useful Wget+Lua executable.
#
# WGET_LUA will be set to the first path that
# 1. does not crash with --version, and
# 2. prints the required version string
WGET_LUA = find_executable(
    "Wget+Lua",
    ["GNU Wget 1.14.lua.20130523-9a5c"],
    [
        "./wget-lua",
        "./wget-lua-warrior",
        "./wget-lua-local",
        "../wget-lua",
        "../../wget-lua",
        "/home/warrior/wget-lua",
        "/usr/bin/wget-lua"
    ]
)

if not WGET_LUA:
    raise Exception("No usable Wget+Lua found.")

###########################################################################
# The version number of this pipeline definition.
#
# Update this each time you make a non-cosmetic change.
# It will be added to the WARC files and reported to the tracker.
VERSION = "20140831.01"
USER_AGENT = 'ArchiveTeam'
TRACKER_ID = 'tele2'
TRACKER_HOST = 'tracker.archiveteam.org'

###########################################################################
# This section defines project-specific tasks.
#
# Simple tasks (tasks that do not need any concurrency) are based on the
# SimpleTask class and have a process(item) method that is called for
# each item.
class CheckIP(SimpleTask):
    def __init__(self):
        SimpleTask.__init__(self, "CheckIP")
        self._counter = 0

    def process(self, item):
        # NEW for 2014! Check if we are behind firewall/proxy

        if self._counter <= 0:
            item.log_output('Checking IP address.')
            ip_set = set()

            ip_set.add(socket.gethostbyname('twitter.com'))
            ip_set.add(socket.gethostbyname('facebook.com'))
            ip_set.add(socket.gethostbyname('youtube.com'))
            ip_set.add(socket.gethostbyname('microsoft.com'))
            ip_set.add(socket.gethostbyname('icanhas.cheezburger.com'))
            ip_set.add(socket.gethostbyname('archiveteam.org'))

            if len(ip_set) != 6:
                item.log_output('Got IP addresses: {0}'.format(ip_set))
                item.log_output(
                    'Are you behind a firewall/proxy? That is a big no-no!')
                raise Exception(
                    'Are you behind a firewall/proxy? That is a big no-no!')

        # Check only occasionally
        if self._counter <= 0:
            self._counter = 10
        else:
            self._counter -= 1

class PrepareDirectories(SimpleTask):
    def __init__(self, warc_prefix):
        SimpleTask.__init__(self, "PrepareDirectories")
        self.warc_prefix = warc_prefix

    def process(self, item):
        item_name = item["item_name"]
        escaped_item_name = item_name.replace(':', '_').replace('/', '_').replace('~', '_')
        dirname = "/".join((item["data_dir"], escaped_item_name))

        if os.path.isdir(dirname):
            shutil.rmtree(dirname)

        os.makedirs(dirname)

        item["item_dir"] = dirname
        item["warc_file_base"] = "%s-%s-%s" % (self.warc_prefix, escaped_item_name,
            time.strftime("%Y%m%d-%H%M%S"))

        open("%(item_dir)s/%(warc_file_base)s.warc.gz" % item, "w").close()

class MoveFiles(SimpleTask):
    def __init__(self):
        SimpleTask.__init__(self, "MoveFiles")

    def process(self, item):
        # NEW for 2014! Check if wget was compiled with zlib support
        if os.path.exists("%(item_dir)s/%(warc_file_base)s.warc"):
            raise Exception('Please compile wget with zlib support!')

        os.rename("%(item_dir)s/%(warc_file_base)s.warc.gz" % item,
              "%(data_dir)s/%(warc_file_base)s.warc.gz" % item)

        shutil.rmtree("%(item_dir)s" % item)

def get_hash(filename):
    with open(filename, 'rb') as in_file:
        return hashlib.sha1(in_file.read()).hexdigest()

CWD = os.getcwd()
PIPELINE_SHA1 = get_hash(os.path.join(CWD, 'pipeline.py'))
LUA_SHA1 = get_hash(os.path.join(CWD, 'tele2.lua'))

def stats_id_function(item):
    # NEW for 2014! Some accountability hashes and stats.
    d = {
        'pipeline_hash': PIPELINE_SHA1,
        'lua_hash': LUA_SHA1,
        'python_version': sys.version,
    }

    return d

class WgetArgs(object):
    def realize(self, item):
        wget_args = [
            WGET_LUA,
            "-U", USER_AGENT,
            "-nv",
            "--lua-script", "tele2.lua",
            "-o", ItemInterpolation("%(item_dir)s/wget.log"),
            "--no-check-certificate",
            "--output-document", ItemInterpolation("%(item_dir)s/wget.tmp"),
            "--truncate-output",
            "-e", "robots=off",
            "--no-cookies",
            "--rotate-dns",
            "--recursive", "--level=inf",
            "--no-parent",
            "--page-requisites",
            "--timeout", "30",
            "--tries", "inf",
            "--span-hosts",
            "--waitretry", "30",
            "--domains", "tele2.se",
            "--warc-file", ItemInterpolation("%(item_dir)s/%(warc_file_base)s"),
            "--warc-header", "operator: Archive Team",
            "--warc-header", "tele2-dld-script-version: " + VERSION,
            "--warc-header", ItemInterpolation("tele2-user: %(item_name)s"),
        ]

        item_name = item['item_name']
        assert ':' in item_name
        item_type, item_value = item_name.split(':', 1)

        assert item_type
        assert item_value

        item['item_type'] = item_type
        item['item_value'] = item_value

        wget_args.append('http://{0}.tele2.se/{1}/'.format(item_type, item_value))

        # wget_args.append('http://home.tele2.se/{0}/'.format(item_name))

        if 'bind_address' in globals():
            wget_args.extend(['--bind-address', globals()['bind_address']])
            print('')
            print('*** Wget will bind address at {0} ***'.format(
                globals()['bind_address']))
            print('')

        return realize(wget_args, item)

###########################################################################
# Initialize the project.
#
# This will be shown in the warrior management panel. The logo should not
# be too big. The deadline is optional.
project = Project(
    title="Tele2",
    project_html="""

home.tele2.se Website · Leaderboard

Archiving Swedish websites from home.tele2.se.

    """,
    utc_deadline=datetime.datetime(2014, 8, 31, 23, 59, 0)
)

pipeline = Pipeline(
    CheckIP(),
    GetItemFromTracker("http://%s/%s" % (TRACKER_HOST, TRACKER_ID), downloader,
        VERSION),
    PrepareDirectories(warc_prefix="tele2"),
    WgetDownload(
        WgetArgs(),
        max_tries=2,
        accept_on_exit_code=[0, 8],
        env={
            "item_dir": ItemValue("item_dir"),
            "item_value": ItemValue("item_value"),
            "item_type": ItemValue("item_type"),
        }
    ),
    PrepareStatsForTracker(
        defaults={"downloader": downloader, "version": VERSION},
        file_groups={
            "data": [
                ItemInterpolation("%(item_dir)s/%(warc_file_base)s.warc.gz")
            ]
        },
        id_function=stats_id_function,
    ),
    MoveFiles(),
    LimitConcurrent(NumberConfigValue(min=1, max=4, default="1",
        name="shared:rsync_threads", title="Rsync threads",
        description="The maximum number of concurrent uploads."),
        UploadWithTracker(
            "http://%s/%s" % (TRACKER_HOST, TRACKER_ID),
            downloader=downloader,
            version=VERSION,
            files=[
                ItemInterpolation("%(data_dir)s/%(warc_file_base)s.warc.gz")
            ],
            rsync_target_source_path=ItemInterpolation("%(data_dir)s/"),
            rsync_extra_args=[
                "--recursive",
                "--partial",
                "--partial-dir", ".rsync-tmp",
            ]
            ),
    ),
    SendDoneToTracker(
        tracker_url="http://%s/%s" % (TRACKER_HOST, TRACKER_ID),
        stats=ItemValue("stats")
    )
)
<|endoftext|>


<commit_before>from skimage import draw, measure, segmentation, util, color, exposure
import sys, random
import numpy as np
import ndio.convert.png
import ndio.remote.OCP as OCP
import sklearn.ensemble as skl
from skimage.future import graph
from scipy.misc import toimage
import rag2 as graph_custom # edited skimage.future.graph module
import cutImageNAnnotations as cutImg

# Input: vol - the 3D image volume
#        nsegs - approx. number of superpixels to create
#        compactness - weight of spatial distance in SLIC (as opposed to intensity distance)
#        threshold - graph cutting threshold (0 to 1)
# Output: volume of labels assigned to different objects in images
def smartDownSample(vol, nsegs, compactness, threshold):
    label_vol = np.empty(vol.shape)
    for i in range(len(vol)):
        # Segment using SLIC
        labels = segmentation.slic(vol[i], nsegs, compactness, multichannel = False, enforce_connectivity = True)
        # Make a Region Adjacency Graph of the segmentation
        rag = graph_custom.rag_mean_int(vol[i], labels)
        # Cut this RAG based on threshold
        new_labels = graph.cut_threshold(labels, rag, threshold)
        # Make a new graph based on the new segmentation (post cut)
        # not using right now... rag = graph_custom.rag_mean_int(img, new_labels)

        new_labels = np.add(new_labels, np.ones(new_labels.shape, dtype=np.int8))
        label_vol[i] = new_labels
    return label_vol.astype('uint8')

def getConfig():
	"""Returns x-min,x-max,y-min,y-max in tuple form in respective order"""
	try:
		import config
		return (config.X_MIN, config.X_MAX, config.Y_MIN, config.Y_MAX)
	except ImportError:
		maxes = cutImg.getCutValues()
		cutImg.writeConfig(maxes) #creates config.py
		return maxes

# Input: db - True if pulling data directly from ndio, False if local
# Output: the raw image volume
def getRaw(db = False):
    # Careful of hardcoded filepath
    if db == False:
        vol = np.array([])
        data = ndio.convert.png.import_png_collection('newdata/mito_data_*')
        for img in data:
            vol = np.dstack([vol, img]) if vol.size else img
        return vol
    else:
		boundaries = getConfig()
		oo = OCP()
		#added +12 because first 10 images in stack aren't even annotated
		return oo.get_cutout('kasthuri11cc', 'image', 694 + boundaries[0], 694 + boundaries[1], 1750 + boundaries[2], 1750 + boundaries[3], 1004+13, 1154, resolution = 3)

# Input: db - True if pulling data directly from ndio, False if local
# Output: the annotation volume
def getTruth(db = False):
    # Careful of hardcoded filepath
    if db == False:
        vol = np.array([])
        data = ndio.convert.png.import_png_collection('newdata/mito_anno_*')
        for img in data:
            vol = np.dstack([vol, img]) if vol.size else img
        return vol
    else:
		boundaries = getConfig()
		oo = OCP()
		#added +12 because first 10 images in stack aren't even annotated
		return oo.get_cutout('kasthuri2015_ramon_v1', 'mitochondria', 694 + boundaries[0], 694 + boundaries[1], 1750 + boundaries[2], 1750 + boundaries[3], 1004+13, 1154, resolution = 3)

# Input: im_vol - the image volume of EM data
#        label_vol - the labels given to objects in EM data (from smartDownSample)
# Output: the feature array X
def extract_features(im_vol, label_vol):
    n_features = 4
    X = np.array([])
    if len(im_vol) != len(label_vol):
        print 'volumes not the same z dimension'
    for i in range(len(im_vol)):
        n_samples = label_vol[i].max() - 1
        props = measure.regionprops(label_vol[i], intensity_image = im_vol[i])
        tX = np.empty([n_samples, n_features])
        for j in range(1, label_vol[i].max()):
            tmp = props[j-1]
            tX[j-1][0] = tmp['mean_intensity']
            tX[j-1][1] = tmp['eccentricity']
            tX[j-1][2] = tmp['convex_area']
            tX[j-1][3] = tmp['area']
        X = np.vstack([X, tX]) if X.size else tX
    return X

# Input: im_vol - the image volume of EM data
#        label_vol - the labels given to objects in EM data (from smartDownSample)
# Output: a vector of the classes (mito or not mito) given to each object (according to label_vol)
def extract_classes(im_vol, label_vol):
    y = np.zeros([0,])
    if len(im_vol) != len(label_vol):
        print 'volumes not the same z dimension'
    for i in range(len(im_vol)):
        n_samples = label_vol[i].max()
        im_vol[i][im_vol[i] > 0] = 255
        props = measure.regionprops(label_vol[i], intensity_image = im_vol[i])
        for j in range(1, label_vol[i].max()):
            if props[j-1]['mean_intensity'] > 150:
                y = np.append(y, 1)
            else:
                y = np.append(y, 0)
    return y

# Input: volume - an image volume
# Output: the image volume after histogram equalization
def equalize_volume(volume):
    for ra in volume:
        ra = (np.floor(exposure.equalize_hist(ra) * 256)).astype('uint8')
    return volume

def main():
    print "Getting raw data..."
    mito_img = getRaw(db = False)

    print "Getting annotations..."
    mito_anno = getTruth(db = False)

    n_rows = len(mito_img[0])
    n_cols = len(mito_img[0][0])

    print "Initializing oversegmentation / threshold cutting algorithm..."
    n_segments = int(sys.argv[1])
    compactness = float(sys.argv[2])
    threshold = float(sys.argv[3])

    train = equalize_volume(mito_img[70:150])
    test = equalize_volume(mito_img[30:70])

    train_truth = mito_anno[70:150]
    test_truth = mito_anno[30:70]

    nri = int(sys.argv[4])
    y_pred = [0] * nri

    for k in range(nri):
        r1, r2, r3 = random.randrange(-100, 100), random.uniform(-.1, .1,), random.uniform(-.05, .05)
        train_labels = smartDownSample(train, n_segments + r1, compactness + r2, threshold + r3)
        test_labels = smartDownSample(test, n_segments + r1, compactness + r2, threshold + r3)

        print "Processing features..."
        X_train = extract_features(train, train_labels)
        y_train = extract_classes(train_truth, train_labels)
        X_test = extract_features(test, test_labels)

        print "Initializing random forest..."
        forest = skl.RandomForestClassifier(n_estimators = 200, max_depth = 100)

        print "Learning random forest..."
        forest.fit(X_train, y_train)

        print "Making predictions..."
        y_guess = forest.predict(X_test)

        # Move from prediction vector to prediction pictures
        j = 0
        y_pred[k] = np.zeros(test_labels.shape)
        for i in range(len(test_labels)):
            for g in range(1, test_labels[i].max()):
                y_pred[k][i][test_labels[i] == g] = y_guess[j]
                j = j + 1

#    print "Showing images..."
    y_pred_or = np.logical_or.reduce(y_pred)
#    for i in range(len(test_labels)):
#        toimage(test[i]).show()
#        label_img = np.empty(test_labels[i].shape)
#        for g in range(test_labels[i].max()):
#            props = measure.regionprops(test_labels[i], intensity_image = test[i])
#            label_img[test_labels[i] == g] = props[g - 1]['mean_intensity']
#        toimage(y_pred_or[i]).show()
#        toimage(label_img).show()
#        toimage(test_truth[i]).show()
    np.save('actual.npy', test)
    print X_test.shape
    np.save('predicted.npy', y_pred_or)
    print y_pred_or.shape

if __name__ == '__main__':
    main()
<commit_msg>reverted changes<commit_after>from skimage import draw, measure, segmentation, util, color, exposure
import sys, random
import numpy as np
import ndio.convert.png
import ndio.remote.OCP as OCP
import sklearn.ensemble as skl
from skimage.future import graph
from scipy.misc import toimage
import rag2 as graph_custom # edited skimage.future.graph module
import cutImageNAnnotations as cutImg

# Input: vol - the 3D image volume
#        nsegs - approx. number of superpixels to create
#        compactness - weight of spatial distance in SLIC (as opposed to intensity distance)
#        threshold - graph cutting threshold (0 to 1)
# Output: volume of labels assigned to different objects in images
def smartDownSample(vol, nsegs, compactness, threshold):
    label_vol = np.empty(vol.shape)
    for i in range(len(vol)):
        # Segment using SLIC
        labels = segmentation.slic(vol[i], nsegs, compactness, multichannel = False, enforce_connectivity = True)
        # Make a Region Adjacency Graph of the segmentation
        rag = graph_custom.rag_mean_int(vol[i], labels)
        # Cut this RAG based on threshold
        new_labels = graph.cut_threshold(labels, rag, threshold)
        # Make a new graph based on the new segmentation (post cut)
        # not using right now... rag = graph_custom.rag_mean_int(img, new_labels)

        new_labels = np.add(new_labels, np.ones(new_labels.shape, dtype=np.int8))
        label_vol[i] = new_labels
    return label_vol.astype('uint8')

def getConfig():
	"""Returns x-min,x-max,y-min,y-max in tuple form in respective order"""
	try:
		import config
		return (config.X_MIN, config.X_MAX, config.Y_MIN, config.Y_MAX)
	except ImportError:
		maxes = cutImg.getCutValues()
		cutImg.writeConfig(maxes) #creates config.py
		return maxes

# Input: db - True if pulling data directly from ndio, False if local
# Output: the raw image volume
def getRaw(db = False):
    # Careful of hardcoded filepath
    if db == False:
        vol = np.array([])
        data = ndio.convert.png.import_png_collection('newdata/mito_data_*')
        for img in data:
            vol = np.dstack([vol, img]) if vol.size else img
        return vol
    else:
		boundaries = getConfig()
		oo = OCP()
		#added +12 because first 10 images in stack aren't even annotated
		return oo.get_cutout('kasthuri11cc', 'image', 694 + boundaries[0], 694 + boundaries[1], 1750 + boundaries[2], 1750 + boundaries[3], 1004, 1154, resolution = 3)

# Input: db - True if pulling data directly from ndio, False if local
# Output: the annotation volume
def getTruth(db = False):
    # Careful of hardcoded filepath
    if db == False:
        vol = np.array([])
        data = ndio.convert.png.import_png_collection('newdata/mito_anno_*')
        for img in data:
            vol = np.dstack([vol, img]) if vol.size else img
        return vol
    else:
		boundaries = getConfig()
		oo = OCP()
		#added +12 because first 10 images in stack aren't even annotated
		return oo.get_cutout('kasthuri2015_ramon_v1', 'mitochondria', 694 + boundaries[0], 694 + boundaries[1], 1750 + boundaries[2], 1750 + boundaries[3], 1004, 1154, resolution = 3)

# Input: im_vol - the image volume of EM data
#        label_vol - the labels given to objects in EM data (from smartDownSample)
# Output: the feature array X
def extract_features(im_vol, label_vol):
    n_features = 4
    X = np.array([])
    if len(im_vol) != len(label_vol):
        print 'volumes not the same z dimension'
    for i in range(len(im_vol)):
        n_samples = label_vol[i].max() - 1
        props = measure.regionprops(label_vol[i], intensity_image = im_vol[i])
        tX = np.empty([n_samples, n_features])
        for j in range(1, label_vol[i].max()):
            tmp = props[j-1]
            tX[j-1][0] = tmp['mean_intensity']
            tX[j-1][1] = tmp['eccentricity']
            tX[j-1][2] = tmp['convex_area']
            tX[j-1][3] = tmp['area']
        X = np.vstack([X, tX]) if X.size else tX
    return X

# Input: im_vol - the image volume of EM data
#        label_vol - the labels given to objects in EM data (from smartDownSample)
# Output: a vector of the classes (mito or not mito) given to each object (according to label_vol)
def extract_classes(im_vol, label_vol):
    y = np.zeros([0,])
    if len(im_vol) != len(label_vol):
        print 'volumes not the same z dimension'
    for i in range(len(im_vol)):
        n_samples = label_vol[i].max()
        im_vol[i][im_vol[i] > 0] = 255
        props = measure.regionprops(label_vol[i], intensity_image = im_vol[i])
        for j in range(1, label_vol[i].max()):
            if props[j-1]['mean_intensity'] > 150:
                y = np.append(y, 1)
            else:
                y = np.append(y, 0)
    return y

# Input: volume - an image volume
# Output: the image volume after histogram equalization
def equalize_volume(volume):
    for ra in volume:
        ra = (np.floor(exposure.equalize_hist(ra) * 256)).astype('uint8')
    return volume

def main():
    print "Getting raw data..."
    mito_img = getRaw(db = False)

    print "Getting annotations..."
    mito_anno = getTruth(db = False)

    n_rows = len(mito_img[0])
    n_cols = len(mito_img[0][0])

    print "Initializing oversegmentation / threshold cutting algorithm..."
    n_segments = int(sys.argv[1])
    compactness = float(sys.argv[2])
    threshold = float(sys.argv[3])

    train = equalize_volume(mito_img[70:150])
    test = equalize_volume(mito_img[30:70])

    train_truth = mito_anno[70:150]
    test_truth = mito_anno[30:70]

    nri = int(sys.argv[4])
    y_pred = [0] * nri

    for k in range(nri):
        r1, r2, r3 = random.randrange(-100, 100), random.uniform(-.1, .1,), random.uniform(-.05, .05)
        train_labels = smartDownSample(train, n_segments + r1, compactness + r2, threshold + r3)
        test_labels = smartDownSample(test, n_segments + r1, compactness + r2, threshold + r3)

        print "Processing features..."
        X_train = extract_features(train, train_labels)
        y_train = extract_classes(train_truth, train_labels)
        X_test = extract_features(test, test_labels)

        print "Initializing random forest..."
        forest = skl.RandomForestClassifier(n_estimators = 200, max_depth = 100)

        print "Learning random forest..."
        forest.fit(X_train, y_train)

        print "Making predictions..."
        y_guess = forest.predict(X_test)

        # Move from prediction vector to prediction pictures
        j = 0
        y_pred[k] = np.zeros(test_labels.shape)
        for i in range(len(test_labels)):
            for g in range(1, test_labels[i].max()):
                y_pred[k][i][test_labels[i] == g] = y_guess[j]
                j = j + 1

#    print "Showing images..."
    y_pred_or = np.logical_or.reduce(y_pred)
#    for i in range(len(test_labels)):
#        toimage(test[i]).show()
#        label_img = np.empty(test_labels[i].shape)
#        for g in range(test_labels[i].max()):
#            props = measure.regionprops(test_labels[i], intensity_image = test[i])
#            label_img[test_labels[i] == g] = props[g - 1]['mean_intensity']
#        toimage(y_pred_or[i]).show()
#        toimage(label_img).show()
#        toimage(test_truth[i]).show()
    np.save('actual.npy', test)
    print X_test.shape
    np.save('predicted.npy', y_pred_or)
    print y_pred_or.shape

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import math
import numpy as np
try:
  import torch
  import torch.nn as nn
  import torch.nn.functional as F
except:
  raise ImportError('These classes require Torch to be installed.')

class ScaleNorm(nn.Module):
  """Apply Scale Normalization to input.

  The ScaleNorm layer first computes the square root of the scale, then computes the matrix/vector norm of the input tensor.
  The norm value is calculated as `sqrt(scale) / matrix norm`.
  Finally, the result is returned as `input_tensor * norm value`.

  This layer can be used instead of LayerNorm when a scaled version of the norm is required.
  Instead of performing the scaling operation (`scale / norm`) in a lambda-like layer, we are defining it within this layer to make prototyping more efficient.

  References
  ----------
  .. [1] Lukasz Maziarka et al. "Molecule Attention Transformer" Graph Representation Learning workshop and Machine Learning and the Physical Sciences workshop at NeurIPS 2019. 2020. https://arxiv.org/abs/2002.08264

  Examples
  --------
  >>> import deepchem as dc
  >>> scale = 0.35
  >>> layer = dc.models.torch_models.layers.ScaleNorm(scale)
  >>> input_tensor = torch.tensor([[1.269, 39.36], [0.00918, -9.12]])
  >>> output_tensor = layer(input_tensor)
  """

  def __init__(self, scale, eps=1e-5):
    """Initialize a ScaleNorm layer.

    Parameters
    ----------
    scale: Real number or single element tensor
      Scale magnitude.
    eps: float
      Epsilon value.
    """

    super(ScaleNorm, self).__init__()
    self.scale = nn.Parameter(torch.tensor(math.sqrt(scale)))
    self.eps = eps

  def forward(self, x):
    norm = self.scale / torch.norm(x, dim=-1, keepdim=True).clamp(min=self.eps)
    return x * norm

class MultiHeadedMATAttention(nn.Module):
  """Converts an existing attention layer to a multi-headed attention module.

  Multi-Headed attention the attention mechanism multiple times parallely through the multiple attention heads.
  Thus, different subsequences of a given sequences can be processed differently.
  The query, key and value parameters are split multiple ways and each split is passed separately through a different attention head.

  References
  ----------
  .. [1] Lukasz Maziarka et al. "Molecule Attention Transformer" Graph Representation Learning workshop and Machine Learning and the Physical Sciences workshop at NeurIPS 2019. 2020. https://arxiv.org/abs/2002.08264

  Examples
  --------
  >>> import deepchem as dc
  >>> attention = dc.models.torch_models.layers.MATAttention('softmax', 0.33, 0.33')
  >>> self_attn_layer = dc.models.torch_models.layers.MultiHeadedAttention(dist_kernel = 'softmax', lambda_attention = 0.33, lambda_adistance = 0.33, h = 8, hsize = 1024, dropout_p = 0.1)
  """

  def __init__(self,
               dist_kernel,
               lambda_attention,
               lambda_distance,
               h,
               hsize,
               dropout_p,
               output_bias=True):
    """Initialize a multi-headed attention layer.

    Parameters
    ----------
    dist_kernel: str
      Kernel activation to be used. Can be either 'softmax' for softmax or 'exp' for exponential.
    lambda_attention: float
      Constant to be multiplied with the attention matrix.
    lambda_distance: float
      Constant to be multiplied with the distance matrix.
    h: int
      Number of attention heads.
    hsize: int
      Size of dense layer.
    dropout_p: float
      Dropout probability.
    output_bias: bool
      If True, dense layers will use bias vectors.
    """

    super().__init__()
    if dist_kernel == "softmax":
      self.dist_kernel = lambda x: torch.softmax(-x, dim=-1)
    elif dist_kernel == "exp":
      self.dist_kernel = lambda x: torch.exp(-x)
    self.lambda_attention = lambda_attention
    self.lambda_distance = lambda_distance
    self.lambda_adjacency = 1.0 - self.lambda_attention - self.lambda_distance
    self.d_k = hsize // h
    self.h = h
    linear_layer = nn.Linear(hsize, hsize)
    self.linear_layers = nn.ModuleList([linear_layer for _ in range(3)])
    self.dropout_p = nn.Dropout(dropout_p)
    self.output_linear = nn.Linear(hsize, hsize, output_bias)

  def _singleAttention(self,
                       query,
                       key,
                       value,
                       mask,
                       dropout_p,
                       adj_matrix,
                       distance_matrix,
                       eps=1e-6,
                       inf=1e12):
    """Defining and computing output for a single MAT attention layer.

    Parameters
    ----------
    query: torch.Tensor
      Standard query parameter for attention.
    key: torch.Tensor
      Standard key parameter for attention.
    value: torch.Tensor
      Standard value parameter for attention.
    mask: torch.Tensor
      Masks out padding values so that they are not taken into account when computing the attention score.
    dropout_p: float
      Dropout probability.
    adj_matrix: np.ndarray
      Adjacency matrix of the input molecule, returned from dc.feat.MATFeaturizer()
    dist_matrix: np.ndarray
      Distance matrix of the input molecule, returned from dc.feat.MATFeaturizer()
    eps: float
      Epsilon value
    inf: float
      Value of infinity to be used.
    """

    d_k = query.size(-1)
    scores = torch.matmul(query, key.transpose(-2, -1)) / math.sqrt(d_k)

    if mask is not None:
      scores = scores.masked_fill(
          mask.unsqueeze(1).repeat(1, query.shape[1], query.shape[2], 1) == 0,
          -inf)
    p_attn = F.softmax(scores, dim=-1)

    adj_matrix = adj_matrix / (adj_matrix.sum(dim=-1).unsqueeze(2) + eps)
    p_adj = adj_matrix.unsqueeze(1).repeat(1, query.shape[1], 1, 1)

    distance_matrix = distance_matrix.masked_fill(
        mask.repeat(1, mask.shape[-1], 1) == 0, np.inf)
    distance_matrix = self.dist_kernel(distance_matrix)
    p_dist = distance_matrix.unsqueeze(1).repeat(1, query.shape[1], 1, 1)

    p_weighted = self.lambda_attention * p_attn + self.lambda_distance * p_dist + self.lambda_adjacency * p_adj
    p_weighted = dropout_p(p_weighted)

    return torch.matmul(p_weighted, value), p_attn

  def forward(self,
              query,
              key,
              value,
              mask,
              dropout_p,
              adj_matrix,
              distance_matrix,
              eps=1e-6,
              inf=1e12,
              **kwargs):
    """Output computation for the MultiHeadedAttention layer.

    Parameters
    ----------
    query: torch.Tensor
      Standard query parameter for attention.
    key: torch.Tensor
      Standard key parameter for attention.
    value: torch.Tensor
      Standard value parameter for attention.
    mask: torch.Tensor
      Masks out padding values so that they are not taken into account when computing the attention score.
    """

    if mask is not None:
      mask = mask.unsqueeze(1)

    batch_size = query.size(0)

    query, key, value = [
        layer(x).view(batch_size, -1, self.h, self.d_k).transpose(1, 2)
        for layer, x in zip(self.linear_layers, (query, key, value))
    ]

    x, _ = self._singleAttention(query, key, value, mask, dropout_p, adj_matrix,
                                 distance_matrix, eps, inf, **kwargs)
    x = x.transpose(1, 2).contiguous().view(batch_size, -1, self.h * self.d_k)

    return self.output_linear(x)
<commit_msg>Type annotations<commit_after>import math
import numpy as np
try:
  import torch
  import torch.nn as nn
  import torch.nn.functional as F
except:
  raise ImportError('These classes require Torch to be installed.')

class ScaleNorm(nn.Module):
  """Apply Scale Normalization to input.

  The ScaleNorm layer first computes the square root of the scale, then computes the matrix/vector norm of the input tensor.
  The norm value is calculated as `sqrt(scale) / matrix norm`.
  Finally, the result is returned as `input_tensor * norm value`.

  This layer can be used instead of LayerNorm when a scaled version of the norm is required.
  Instead of performing the scaling operation (`scale / norm`) in a lambda-like layer, we are defining it within this layer to make prototyping more efficient.

  References
  ----------
  .. [1] Lukasz Maziarka et al. "Molecule Attention Transformer" Graph Representation Learning workshop and Machine Learning and the Physical Sciences workshop at NeurIPS 2019. 2020. https://arxiv.org/abs/2002.08264

  Examples
  --------
  >>> import deepchem as dc
  >>> scale = 0.35
  >>> layer = dc.models.torch_models.layers.ScaleNorm(scale)
  >>> input_tensor = torch.tensor([[1.269, 39.36], [0.00918, -9.12]])
  >>> output_tensor = layer(input_tensor)
  """

  def __init__(self, scale: int, eps: float = 1e-5):
    """Initialize a ScaleNorm layer.

    Parameters
    ----------
    scale: Real number or single element tensor
      Scale magnitude.
    eps: float
      Epsilon value.
    """

    super(ScaleNorm, self).__init__()
    self.scale = nn.Parameter(torch.tensor(math.sqrt(scale)))
    self.eps = eps

  def forward(self, x):
    norm = self.scale / torch.norm(x, dim=-1, keepdim=True).clamp(min=self.eps)
    return x * norm

class MultiHeadedMATAttention(nn.Module):
  """Converts an existing attention layer to a multi-headed attention module.

  Multi-Headed attention the attention mechanism multiple times parallely through the multiple attention heads.
  Thus, different subsequences of a given sequences can be processed differently.
  The query, key and value parameters are split multiple ways and each split is passed separately through a different attention head.

  References
  ----------
  .. [1] Lukasz Maziarka et al. "Molecule Attention Transformer" Graph Representation Learning workshop and Machine Learning and the Physical Sciences workshop at NeurIPS 2019. 2020. https://arxiv.org/abs/2002.08264

  Examples
  --------
  >>> import deepchem as dc
  >>> attention = dc.models.torch_models.layers.MATAttention('softmax', 0.33, 0.33')
  >>> self_attn_layer = dc.models.torch_models.layers.MultiHeadedAttention(dist_kernel = 'softmax', lambda_attention = 0.33, lambda_adistance = 0.33, h = 8, hsize = 1024, dropout_p = 0.1)
  """

  def __init__(self,
               dist_kernel: str,
               lambda_attention: float,
               lambda_distance: float,
               h: int,
               hsize: int,
               dropout_p: float,
               output_bias: bool = True):
    """Initialize a multi-headed attention layer.

    Parameters
    ----------
    dist_kernel: str
      Kernel activation to be used. Can be either 'softmax' for softmax or 'exp' for exponential.
    lambda_attention: float
      Constant to be multiplied with the attention matrix.
    lambda_distance: float
      Constant to be multiplied with the distance matrix.
    h: int
      Number of attention heads.
    hsize: int
      Size of dense layer.
    dropout_p: float
      Dropout probability.
    output_bias: bool
      If True, dense layers will use bias vectors.
    """
    super().__init__()
    if dist_kernel == "softmax":
      self.dist_kernel = lambda x: torch.softmax(-x, dim=-1)
    elif dist_kernel == "exp":
      self.dist_kernel = lambda x: torch.exp(-x)
    self.lambda_attention = lambda_attention
    self.lambda_distance = lambda_distance
    self.lambda_adjacency = 1.0 - self.lambda_attention - self.lambda_distance
    self.d_k = hsize // h
    self.h = h
    linear_layer = nn.Linear(hsize, hsize)
    self.linear_layers = nn.ModuleList([linear_layer for _ in range(3)])
    self.dropout_p = nn.Dropout(dropout_p)
    self.output_linear = nn.Linear(hsize, hsize, output_bias)

  def _single_attention(self,
                        query: torch.Tensor,
                        key: torch.Tensor,
                        value: torch.Tensor,
                        mask: torch.Tensor,
                        dropout_p: float,
                        adj_matrix: np.ndarray,
                        distance_matrix: np.ndarray,
                        eps: float = 1e-6,
                        inf: float = 1e12):
    """Defining and computing output for a single MAT attention layer.

    Parameters
    ----------
    query: torch.Tensor
      Standard query parameter for attention.
    key: torch.Tensor
      Standard key parameter for attention.
    value: torch.Tensor
      Standard value parameter for attention.
    mask: torch.Tensor
      Masks out padding values so that they are not taken into account when computing the attention score.
    dropout_p: float
      Dropout probability.
    adj_matrix: np.ndarray
      Adjacency matrix of the input molecule, returned from dc.feat.MATFeaturizer()
    dist_matrix: np.ndarray
      Distance matrix of the input molecule, returned from dc.feat.MATFeaturizer()
    eps: float
      Epsilon value
    inf: float
      Value of infinity to be used.
    """
    d_k = query.size(-1)
    scores = torch.matmul(query, key.transpose(-2, -1)) / math.sqrt(d_k)

    if mask is not None:
      scores = scores.masked_fill(
          mask.unsqueeze(1).repeat(1, query.shape[1], query.shape[2], 1) == 0,
          -inf)
    p_attn = F.softmax(scores, dim=-1)

    adj_matrix = adj_matrix / (adj_matrix.sum(dim=-1).unsqueeze(2) + eps)
    p_adj = adj_matrix.unsqueeze(1).repeat(1, query.shape[1], 1, 1)

    distance_matrix = distance_matrix.masked_fill(
        mask.repeat(1, mask.shape[-1], 1) == 0, np.inf)
    distance_matrix = self.dist_kernel(distance_matrix)
    p_dist = distance_matrix.unsqueeze(1).repeat(1, query.shape[1], 1, 1)

    p_weighted = self.lambda_attention * p_attn + self.lambda_distance * p_dist + self.lambda_adjacency * p_adj
    p_weighted = dropout_p(p_weighted)

    return torch.matmul(p_weighted, value), p_attn

  def forward(self,
              query: torch.Tensor,
              key: torch.Tensor,
              value: torch.Tensor,
              mask: torch.Tensor,
              dropout_p: float,
              adj_matrix: np.ndarray,
              distance_matrix: np.ndarray,
              eps: float = 1e-6,
              inf: float = 1e12,
              **kwargs):
    """Output computation for the MultiHeadedAttention layer.

    Parameters
    ----------
    query: torch.Tensor
      Standard query parameter for attention.
    key: torch.Tensor
      Standard key parameter for attention.
    value: torch.Tensor
      Standard value parameter for attention.
    mask: torch.Tensor
      Masks out padding values so that they are not taken into account when computing the attention score.
    """
    if mask is not None:
      mask = mask.unsqueeze(1)

    batch_size = query.size(0)

    query, key, value = [
        layer(x).view(batch_size, -1, self.h, self.d_k).transpose(1, 2)
        for layer, x in zip(self.linear_layers, (query, key, value))
    ]

    x, _ = self._single_attention(query, key, value, mask, dropout_p,
                                  adj_matrix, distance_matrix, eps, inf,
                                  **kwargs)
    x = x.transpose(1, 2).contiguous().view(batch_size, -1, self.h * self.d_k)

    return self.output_linear(x)
<|endoftext|>


<commit_before>#-------------------------------------------------------------------------------
import multiprocessing
import os
from os import path
import subprocess
import re
import time
import config
import database

#-------------------------------------------------------------------------------
remotehost = config.REMOTEHOST
remotehome = config.REMOTEHOME
pipe_launch = config.PIPE_LAUNCH
data_dir = config.DATADIR

reSLURMLINE = re.compile(r"slurmids: (?P<slurmids>\d+(,\d+)*)")
reSLURMID = re.compile(r"\,")

#-------------------------------------------------------------------------------
def startJob( user, var1 ):
    jobid = database.createJob( user )

    p = multiprocessing.Process( target=runjob, args=(user, jobid, var1) )
    print var1
    p.start()

#-------------------------------------------------------------------------------
def cancelJob( user, jobid ):
    print "CANCELING JOB " + str(jobid)
    jobinfo = database.getJobInfo( jobid )
    print jobinfo
    if jobinfo['state'] == database.JOB_COMPLETED or jobinfo['state'] == database.JOB_CANCELED:
        print "job", jobid, "already canceled"
        return true

    # Canceling Jobid:
    for slurmid in jobinfo['slurmids']:
        print "canceling slurm job", slurmid

        command = ["ssh", remotehost, "mncancel", slurmid ]
        proc = subprocess.Popen( command, stdout=subprocess.PIPE )
        #output = proc.communicate()[0]

    # Removing job folders:
    jname = jobinfo[ 'name' ]
    print "Removing outputs from Job named: ", jname
    command =  [ 'ssh', remotehost,
                 "cd", data_dir + user,
                 "rm -r", jname ]
    proc = subprocess.Popen( command, stdout=subprocess.PIPE )
    #output = proc.communicate()[0]

    database.setJobCanceled( jobid )
    return True

#-------------------------------------------------------------------------------
def getSlurmIds( output ):
    mm = reSLURMLINE.search( output )
    sids = []
    if mm:
        slurmline = mm.group('slurmids')
        slurmids = reSLURMID.split( slurmline )
        sids = map(int,slurmids)

    return sids

#-------------------------------------------------------------------------------
def runjob( user, jobid, var1):
    print "RUNNING JOB " + str(jobid)

# stagein
    if "file" in var1:
        fileid1 = int(var1["file"])
        localfile1 = database.getFileFullName( fileid1 )
        remotefile1 = os.path.join( remotehome, localfile1 )
        database.addJobFile( jobid, fileid1, database.FILEIN )
        var1['file_read1'] = remotefile1

    if "file2" in var1:
        fileid2 = int(var1["file2"])
        localfile2 = database.getFileFullName( fileid2 )
        remotefile2 = os.path.join( remotehome, localfile2 )
        database.addJobFile( jobid, fileid2, database.FILEIN )
        var1['file_read2'] = remotefile2

    if "file3" in var1:
        fileid3 = int(var1["file3"])
        localfile3 = database.getFileFullName( fileid3 )
        remotefile3 = os.path.join( remotehome, localfile3 )
        database.addJobFile( jobid, fileid3, database.FILEIN )
        var1['file_ass'] = remotefile3

        # submit

    jinfo = database.getJobInfo( jobid )
    juid =  jinfo['juid']

    if var1["input_type"] == "single":
        command = [ pipe_launch, user, str(var1), remotefile1, str(juid) ]
    elif var1["input_type"] == "paired":
        command = [ pipe_launch, user, str(var1), remotefile1, remotefile2, str(juid) ]
    elif var1.input_type =="contigs":
        command = [ pipe_launch, user, str(var1), remotefile3, str(juid) ]
    elif var1.input_type =="contigs_with_single":
        command = [ pipe_launch, user, str(var1), remotefile1, remotefile3,
                    str(juid) ]
    elif var1.input_type =="contigs_with_paired":
        command = [ pipe_launch, user, str(var1), remotefile1, remotefile2,
                    remotefile3, str(juid) ]
    print command

    for k in var1:
        print k + " : " + var1[k]

    proc = subprocess.Popen( command, stdout=subprocess.PIPE )
    output = proc.communicate()[0]
    slurmids = getSlurmIds( output )

    if len(slurmids) > 0:
        database.setJobSubmitted( jobid )
    else:
        print "WARNING : task without slurm ids"

    for si in slurmids:
        database.addJobSlurmRef( jobid, si )

#-------------------------------------------------------------------------------
def run():
    p = multiprocessing.Process( target=pipelineLoop )
    p.start()
    return p

#-------------------------------------------------------------------------------
def checkSlurmJob( slurmids ):
    if len(slurmids) > 0:
        idsstr = ",".join(map(str,slurmids))
        command = ["ssh", remotehost, "mnq", "--job", idsstr ]
        proc = subprocess.Popen( command, stdout=subprocess.PIPE )
        output = proc.communicate()[0]

        if len(output.splitlines()) <= 1:
            return database.JOB_COMPLETED

        mm = re.search( "RUNNING", output )
        if mm is not None:
            return database.JOB_RUNNING

    return database.JOB_SUBMITTED

#-------------------------------------------------------------------------------
def pipelineLoop():
    try:
        print "start pipeline loop"
        while (1 == 1):
            time.sleep( 60 )
            # check
            zjobs = database.getJustCreatedJobs()
            if len(zjobs) > 0:
                zjobids = map( lambda js: js['jid'], zjobs )
                print "Warning: " + str(len(zjobs)) + " zombie jobs : " + str(zjobids)

            jobs = database.getActiveJobs()
            if len(jobs) > 0:
                print "Checking " + str(len(jobs)) + " job/s"

            for job in jobs:
                jobid = job['jid']
                newstate = checkSlurmJob( job['slurmids'] )
                if newstate == database.JOB_RUNNING and job['state'] != database.JOB_RUNNING:
                    print "Job " + str(jobid) + " start RUNNING"
                    database.setJobRunning( jobid )

                if newstate == database.JOB_COMPLETED:
            #         userid = job['uid']
            #         slurmid = job['slurmid']
            #         # stageout
            #         outs = ["jor_"+str(slurmid)+".out", "jor_"+str(slurmid)+".err"]
            #         for fileoutname in outs:
            #             fileout = database.createFile( userid, fileoutname )
            #             database.addJobFile( jobid, fileout, database.FILEOUT )
            #             localfile = database.getFileFullName( fileout )
            #             (localdir, localbase) = os.path.split( localfile )
            #             remotedir = os.path.join( remotehome, localdir )
            #             remotefile = os.path.join( remotehome, localfile )
            #             os.system('scp "%s:%s" "%s"' % (remotehost, remotefile, localfile) )

                    print "Job " + str(jobid) + " COMPLETED"
                    database.setJobCompleted( jobid )

    except KeyboardInterrupt:
        print "ending pipeline loop"

#-------------------------------------------------------------------------------
<commit_msg>completing canceling jobs function<commit_after>#-------------------------------------------------------------------------------
import multiprocessing
import os
from os import path
import subprocess
import re
import time
import config
import database

#-------------------------------------------------------------------------------
remotehost = config.REMOTEHOST
remotehome = config.REMOTEHOME
pipe_launch = config.PIPE_LAUNCH
data_dir = config.DATADIR

reSLURMLINE = re.compile(r"slurmids: (?P<slurmids>\d+(,\d+)*)")
reSLURMID = re.compile(r"\,")

#-------------------------------------------------------------------------------
def startJob( user, var1 ):
    jobid = database.createJob( user )

    p = multiprocessing.Process( target=runjob, args=(user, jobid, var1) )
    print var1
    p.start()

#-------------------------------------------------------------------------------
def cancelJob( user, jobid ):
    print "CANCELING JOB " + str(jobid)
    jobinfo = database.getJobInfo( jobid )
    print jobinfo

    if jobinfo['state'] == database.JOB_COMPLETED or jobinfo['state'] == database.JOB_CANCELED:
        print "job", jobid, "already canceled"
        return true

    # Canceling Jobid:
    print jobinfo['slurmids']
    for slurmid in jobinfo['slurmids']:
        print "canceling slurm job", slurmid["slurmid"]

        command = ["ssh", remotehost, "mncancel", str(slurmid["slurmid"]) ]
        proc = subprocess.Popen( command, stdout=subprocess.PIPE )
        output = proc.communicate()[0]
        print output

    # Removing job folders:
        # Web and server job names should be corresponding before activating this
    # jname = jobinfo[ 'name' ]
    # print "Removing outputs from Job named: ", jname
    # command =  [ 'ssh', remotehost,
    #              'cd', data_dir + user,
    #              'rm -r', jname,
    #              'touch', "DONE"]
    # proc = subprocess.Popen( command, stdout=subprocess.PIPE )
    #output = proc.communicate()[0]

    database.setJobCanceled( jobid )
    return True

#-------------------------------------------------------------------------------
def getSlurmIds( output ):
    mm = reSLURMLINE.search( output )
    sids = []
    if mm:
        slurmline = mm.group('slurmids')
        slurmids = reSLURMID.split( slurmline )
        sids = map(int,slurmids)

    return sids

#-------------------------------------------------------------------------------
def runjob( user, jobid, var1):
    print "RUNNING JOB " + str(jobid)

# stagein
    if "file" in var1:
        fileid1 = int(var1["file"])
        localfile1 = database.getFileFullName( fileid1 )
        remotefile1 = os.path.join( remotehome, localfile1 )
        database.addJobFile( jobid, fileid1, database.FILEIN )
        var1['file_read1'] = remotefile1

    if "file2" in var1:
        fileid2 = int(var1["file2"])
        localfile2 = database.getFileFullName( fileid2 )
        remotefile2 = os.path.join( remotehome, localfile2 )
        database.addJobFile( jobid, fileid2, database.FILEIN )
        var1['file_read2'] = remotefile2

    if "file3" in var1:
        fileid3 = int(var1["file3"])
        localfile3 = database.getFileFullName( fileid3 )
        remotefile3 = os.path.join( remotehome, localfile3 )
        database.addJobFile( jobid, fileid3, database.FILEIN )
        var1['file_ass'] = remotefile3

        # submit

    jinfo = database.getJobInfo( jobid )
    juid =  jinfo['juid']

    if var1["input_type"] == "single":
        command = [ pipe_launch, user, str(var1), remotefile1, str(juid) ]
    elif var1["input_type"] == "paired":
        command = [ pipe_launch, user, str(var1), remotefile1, remotefile2, str(juid) ]
    elif var1.input_type =="contigs":
        command = [ pipe_launch, user, str(var1), remotefile3, str(juid) ]
    elif var1.input_type =="contigs_with_single":
        command = [ pipe_launch, user, str(var1), remotefile1, remotefile3,
                    str(juid) ]
    elif var1.input_type =="contigs_with_paired":
        command = [ pipe_launch, user, str(var1), remotefile1, remotefile2,
                    remotefile3, str(juid) ]
    print command

    for k in var1:
        print k + " : " + var1[k]

    proc = subprocess.Popen( command, stdout=subprocess.PIPE )
    output = proc.communicate()[0]
    slurmids = getSlurmIds( output )

    if len(slurmids) > 0:
        database.setJobSubmitted( jobid )
    else:
        print "WARNING : task without slurm ids"

    for si in slurmids:
        database.addJobSlurmRef( jobid, si )

#-------------------------------------------------------------------------------
def run():
    p = multiprocessing.Process( target=pipelineLoop )
    p.start()
    return p

#-------------------------------------------------------------------------------
def checkSlurmJob( slurmids ):
    if len(slurmids) > 0:
        idsstr = ",".join(map(str,slurmids))
        command = ["ssh", remotehost, "mnq", "--job", idsstr ]
        proc = subprocess.Popen( command, stdout=subprocess.PIPE )
        output = proc.communicate()[0]

        if len(output.splitlines()) <= 1:
            return database.JOB_COMPLETED

        mm = re.search( "RUNNING", output )
        if mm is not None:
            return database.JOB_RUNNING

    return database.JOB_SUBMITTED

#-------------------------------------------------------------------------------
def pipelineLoop():
    try:
        print "start pipeline loop"
        while (1 == 1):
            time.sleep( 60 )
            # check
            zjobs = database.getJustCreatedJobs()
            if len(zjobs) > 0:
                zjobids = map( lambda js: js['jid'], zjobs )
                print "Warning: " + str(len(zjobs)) + " zombie jobs : " + str(zjobids)

            jobs = database.getActiveJobs()
            if len(jobs) > 0:
                print "Checking " + str(len(jobs)) + " job/s"

            for job in jobs:
                jobid = job['jid']
                newstate = checkSlurmJob( job['slurmids'] )
                if newstate == database.JOB_RUNNING and job['state'] != database.JOB_RUNNING:
                    print "Job " + str(jobid) + " start RUNNING"
                    database.setJobRunning( jobid )

                if newstate == database.JOB_COMPLETED:
            #         userid = job['uid']
            #         slurmid = job['slurmid']
            #         # stageout
            #         outs = ["jor_"+str(slurmid)+".out", "jor_"+str(slurmid)+".err"]
            #         for fileoutname in outs:
            #             fileout = database.createFile( userid, fileoutname )
            #             database.addJobFile( jobid, fileout, database.FILEOUT )
            #             localfile = database.getFileFullName( fileout )
            #             (localdir, localbase) = os.path.split( localfile )
            #             remotedir = os.path.join( remotehome, localdir )
            #             remotefile = os.path.join( remotehome, localfile )
            #             os.system('scp "%s:%s" "%s"' % (remotehost, remotefile, localfile) )

                    print "Job " + str(jobid) + " COMPLETED"
                    database.setJobCompleted( jobid )

    except KeyboardInterrupt:
        print "ending pipeline loop"

#-------------------------------------------------------------------------------
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright 2016 Curtis Sand
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
#     Unless required by applicable law or agreed to in writing, software
#     distributed under the License is distributed on an "AS IS" BASIS,
#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#     See the License for the specific language governing permissions and
#     limitations under the License.

"""A Waveform or Signal Generator Library for creating audio waveforms."""

import math

import numpy

VERSION = "0.1"

class Generator(object):
    def __init__(self, length=1.0, framerate=44100, verbose=False):
        self.length = length
        self.framerate = framerate
        self.verbose = verbose

    def _init(self, length=None, framerate=None, verbose=None, **kwargs):
        if length:
            self.length = length
        if framerate:
            self.framerate = framerate
        if verbose:
            self.verbose = verbose

        # framecount = frames/sec * sec
        self.framecount = int(self.framerate * self.length)
        # rectify length to actual framecount
        self.length = float(self.framecount)/self.framerate
        self.dprint('framecount = %s' % self.framecount)
        self.dprint('rectified length = %s' % self.length)
        self.wavedata = numpy.zeros((self.framecount, 1))

    def dprint(self, msg):
        """Conditionally print a debugging message."""
        if self.verbose:
            print(msg)

    def whitenoise(self, *args, **kwargs):
        """Random Gaussian White Noise."""
        self._init(*args, **kwargs)
        self.wavedata = numpy.random.randn(self.framecount, 1)
        return self.wavedata

    def _sinusoid_amplitude(self, frame, frequency):
        """Calculate the amplitude of a sinusoid wave at a given frequency."""
        # avoid divide by zero
        frame = 0.001 if frame is 0 else frame
        return math.sin(frame /
                        ((self.framerate / frequency) / math.pi))

    def sin_constant(self, frequency, *args, **kwargs):
        """Sinusoid wave of constant frequency."""
        self._init(*args, **kwargs)
        frequency = float(frequency)
        for frame in range(self.framecount):
            amplitude = self._sinusoid_amplitude(frame, frequency)
            self.wavedata[frame, 0] = amplitude
        return self.wavedata

    def sin_linear(self, start_freq, end_freq, *args, **kwargs):
        """Sinusoid wave of linearly changing frequency."""
        self._init(*args, **kwargs)
        for frame in range(self.framecount):
            # freq = start_freq + frame * freq_rate
            # freq_rate = total_freq_change / framecount
            frequency = start_freq + frame * (
                float(end_freq - start_freq) / self.framecount)
            amplitude = self._sinusoid_amplitude(frame, frequency)
            self.wavedata[frame, 0] = amplitude
        return self.wavedata

if __name__ == '__main__':
    import common

    DESC = "A Demonstration Program for the Signal Generator."

    def whitenoise(args, generator):
        generator.whitenoise()

    def sin_constant(args, generator):
        generator.sin_constant(args.frequency)

    def sin_linear(args, generator):
        generator.sin_linear(args.frequency/2, args.frequency*2)

    UI_MAP = {'noise': whitenoise, 'constant': sin_constant,
              'linear': sin_linear}

    def main():
        parser = common.get_cmd_line_parser(description=DESC,
                                           version=VERSION)
        parser.add_argument(
            '-t', '--type', help='Type of signal to generate',
            choices=UI_MAP.keys())
        common.ParserArguments.filename(parser)
        common.ParserArguments.length(parser)
        common.ParserArguments.plot(parser)
        common.ParserArguments.frequency(parser)
        common.ParserArguments.set_defaults(parser, filename='signal.wav',
                                            length=1.0, frequency=480,
                                            type='constant')
        args = parser.parse_args()

        sg = Generator(length=args.length, verbose=args.debug)

        UI_MAP[args.type](args, sg)

        if args.plot:
            import plot
            plot.plot_waveform(sg.wavedata, 1, 0, sg.framecount)
        else:
            import wav_file
            wfile = wav_file.WavFile(args.filename, 1, sg.framecount)
            try:
                wfile.write(sg.wavedata)
            finally:
                wfile.close()

        return 0

    common.call_main(main)
<commit_msg>waveform.py: remove VERSION from script<commit_after>#!/usr/bin/env python
# Copyright 2016 Curtis Sand
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
#     Unless required by applicable law or agreed to in writing, software
#     distributed under the License is distributed on an "AS IS" BASIS,
#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#     See the License for the specific language governing permissions and
#     limitations under the License.

"""A Waveform or Signal Generator Library for creating audio waveforms."""

import math

import numpy

class Generator(object):
    def __init__(self, length=1.0, framerate=44100, verbose=False):
        self.length = length
        self.framerate = framerate
        self.verbose = verbose

    def _init(self, length=None, framerate=None, verbose=None, **kwargs):
        if length:
            self.length = length
        if framerate:
            self.framerate = framerate
        if verbose:
            self.verbose = verbose

        # framecount = frames/sec * sec
        self.framecount = int(self.framerate * self.length)
        # rectify length to actual framecount
        self.length = float(self.framecount)/self.framerate
        self.dprint('framecount = %s' % self.framecount)
        self.dprint('rectified length = %s' % self.length)
        self.wavedata = numpy.zeros((self.framecount, 1))

    def dprint(self, msg):
        """Conditionally print a debugging message."""
        if self.verbose:
            print(msg)

    def whitenoise(self, *args, **kwargs):
        """Random Gaussian White Noise."""
        self._init(*args, **kwargs)
        self.wavedata = numpy.random.randn(self.framecount, 1)
        return self.wavedata

    def _sinusoid_amplitude(self, frame, frequency):
        """Calculate the amplitude of a sinusoid wave at a given frequency."""
        # avoid divide by zero
        frame = 0.001 if frame is 0 else frame
        return math.sin(frame /
                        ((self.framerate / frequency) / math.pi))

    def sin_constant(self, frequency, *args, **kwargs):
        """Sinusoid wave of constant frequency."""
        self._init(*args, **kwargs)
        frequency = float(frequency)
        for frame in range(self.framecount):
            amplitude = self._sinusoid_amplitude(frame, frequency)
            self.wavedata[frame, 0] = amplitude
        return self.wavedata

    def sin_linear(self, start_freq, end_freq, *args, **kwargs):
        """Sinusoid wave of linearly changing frequency."""
        self._init(*args, **kwargs)
        for frame in range(self.framecount):
            # freq = start_freq + frame * freq_rate
            # freq_rate = total_freq_change / framecount
            frequency = start_freq + frame * (
                float(end_freq - start_freq) / self.framecount)
            amplitude = self._sinusoid_amplitude(frame, frequency)
            self.wavedata[frame, 0] = amplitude
        return self.wavedata

if __name__ == '__main__':
    import common

    DESC = "A Demonstration Program for the Signal Generator."

    def whitenoise(args, generator):
        generator.whitenoise()

    def sin_constant(args, generator):
        generator.sin_constant(args.frequency)

    def sin_linear(args, generator):
        generator.sin_linear(args.frequency/2, args.frequency*2)

    UI_MAP = {'noise': whitenoise, 'constant': sin_constant,
              'linear': sin_linear}

    def main():
        parser = common.get_cmd_line_parser(description=DESC)
        parser.add_argument(
            '-t', '--type', help='Type of signal to generate',
            choices=UI_MAP.keys())
        common.ParserArguments.filename(parser)
        common.ParserArguments.length(parser)
        common.ParserArguments.plot(parser)
        common.ParserArguments.frequency(parser)
        common.ParserArguments.set_defaults(parser, filename='signal.wav',
                                            length=1.0, frequency=480,
                                            type='constant')
        args = parser.parse_args()

        sg = Generator(length=args.length, verbose=args.debug)

        UI_MAP[args.type](args, sg)

        if args.plot:
            import plot
            plot.plot_waveform(sg.wavedata, 1, 0, sg.framecount)
        else:
            import wav_file
            wfile = wav_file.WavFile(args.filename, 1, sg.framecount)
            try:
                wfile.write(sg.wavedata)
            finally:
                wfile.close()

        return 0

    common.call_main(main)
<|endoftext|>


<commit_before>"""
.. module:: sweep_analysis
  :platform: Unix, Windows, Mac
  :synopsis: Submodule containing modules for analysing

.. moduleauthor:: Dominic Etienne Charrier <dominic.e.charrier@durham.ac.uk>,

:synopsis: Generate benchmark suites for ExaHyPE.
"""
import sys
import configparser
import csv
import collections

def parseList(string):
    """
    Decomposes strings like '"val1,val2",val3,"val4,val5"'
    into a list of strings:
    [ 'val1,val2' ,'val3', 'val4,val5' ]
    """
    for line in csv.reader([string],delimiter=","):
      values = line
      #for i,value in enumerate(values):
      #    values[i] = value.replace(" ","")
      return values

def parseEnvironment(config):
    """
    Parse the environment section.
    """
    environmentSpace = {}
    if "environment" in config and len(config["environment"].keys()):
        for key, value in config["environment"].items():
            environmentSpace[key] = parseList(value)

    return environmentSpace

def parseParameters(config):
    """
    Parse the parameters section.
    """
    parameterSpace = {}
    if "parameters" in config and len(config["parameters"].keys()):
        for key, value in config["parameters"].items():
            parameterSpace[key] = parseList(value)

        if "time-steps" not in parameterSpace:
            print("ERROR: 'order' missing in section 'parameters'.",file=sys.stderr)
            sys.exit()
    else:
        print("ERROR: Section 'parameters' is missing or empty! (Must contain at least 'dimension' and 'order'.)",file=sys.stderr)
        sys.exit()

    return parameterSpace

def parseOptionsFile(optionsFile,ignoreMetadata=False):
    configParser = configparser.ConfigParser()
    configParser.optionxform=str
    configParser.read(optionsFile)

    general          = dict(configParser["general"])
    deltaRoot        = general["delta_root"]
    outputPath       = general["output_path"]
    projectPath      = general["project_path"]
    projectName      = general["project_name"]

    buildFolderPath   = deltaRoot+"/"+outputPath+"/"
    scriptsFolderPath = deltaRoot+"/"+outputPath+"/" + "scripts"
    resultsFolderPath = projectPath+"/"+projectName+"/"
    historyFolderPath = deltaRoot+"/"+outputPath+"/"+"history"

    jobs             = dict(configParser["jobs"])
    environmentSpace = parseEnvironment(configParser)
    parameterSpace   = parseParameters(configParser)

    nodeCounts = [x.strip() for x in jobs["nodes"].split(",")]
    taskCounts = [x.strip() for x in jobs["tasks"].split(",")]
    coreCounts = [x.strip() for x in jobs["cores"].split(",")]
    ompthreadCounts = [x.strip() for x in jobs["ompthreads"].split(",")]

    Options = collections.namedtuple("options", \
           ("general jobs environmentSpace parameterSpace "
            "deltaRoot outputPath projectPath projectName "
            "buildFolderPath scriptsFolderPath "
            "resultsFolderPath historyFolderPath "
            "nodeCounts taskCounts coreCounts ompthreadCounts"))

    options = Options(
      general          = general,
      jobs             = jobs,\
      environmentSpace = environmentSpace,\
      parameterSpace   = parameterSpace,\
      \
      deltaRoot      = deltaRoot,\
      outputPath       = outputPath,\
      projectPath      = projectPath,\
      projectName      = projectName,\
      \
      buildFolderPath   = buildFolderPath,\
      scriptsFolderPath = scriptsFolderPath,\
      resultsFolderPath = resultsFolderPath,\
      historyFolderPath = historyFolderPath,\
      \
      nodeCounts = nodeCounts,\
      taskCounts = taskCounts,\
      coreCounts = coreCounts, \
      ompthreadCounts = ompthreadCounts \
    )

    return options
<commit_msg>directory change<commit_after>"""
.. module:: sweep_analysis
  :platform: Unix, Windows, Mac
  :synopsis: Submodule containing modules for analysing

.. moduleauthor:: Dominic Etienne Charrier <dominic.e.charrier@durham.ac.uk>,

:synopsis: Generate benchmark suites for ExaHyPE.
"""
import sys
import configparser
import csv
import collections

def parseList(string):
    """
    Decomposes strings like '"val1,val2",val3,"val4,val5"'
    into a list of strings:
    [ 'val1,val2' ,'val3', 'val4,val5' ]
    """
    for line in csv.reader([string],delimiter=","):
      values = line
      #for i,value in enumerate(values):
      #    values[i] = value.replace(" ","")
      return values

def parseEnvironment(config):
    """
    Parse the environment section.
    """
    environmentSpace = {}
    if "environment" in config and len(config["environment"].keys()):
        for key, value in config["environment"].items():
            environmentSpace[key] = parseList(value)

    return environmentSpace

def parseParameters(config):
    """
    Parse the parameters section.
    """
    parameterSpace = {}
    if "parameters" in config and len(config["parameters"].keys()):
        for key, value in config["parameters"].items():
            parameterSpace[key] = parseList(value)

        if "time-steps" not in parameterSpace:
            print("ERROR: 'order' missing in section 'parameters'.",file=sys.stderr)
            sys.exit()
    else:
        print("ERROR: Section 'parameters' is missing or empty! (Must contain at least 'dimension' and 'order'.)",file=sys.stderr)
        sys.exit()

    return parameterSpace

def parseOptionsFile(optionsFile,ignoreMetadata=False):
    configParser = configparser.ConfigParser()
    configParser.optionxform=str
    configParser.read(optionsFile)

    general          = dict(configParser["general"])
    deltaRoot        = general["delta_root"]
    outputPath       = general["output_path"]
    projectPath      = general["project_path"]
    projectName      = general["project_name"]

    buildFolderPath   = deltaRoot+"/output/"
    scriptsFolderPath = deltaRoot+"/output/" + "scripts"
    resultsFolderPath = projectPath+"/"+projectName+"/"
    historyFolderPath = deltaRoot+"/output/"+"history"

    jobs             = dict(configParser["jobs"])
    environmentSpace = parseEnvironment(configParser)
    parameterSpace   = parseParameters(configParser)

    nodeCounts = [x.strip() for x in jobs["nodes"].split(",")]
    taskCounts = [x.strip() for x in jobs["tasks"].split(",")]
    coreCounts = [x.strip() for x in jobs["cores"].split(",")]
    ompthreadCounts = [x.strip() for x in jobs["ompthreads"].split(",")]

    Options = collections.namedtuple("options", \
           ("general jobs environmentSpace parameterSpace "
            "deltaRoot outputPath projectPath projectName "
            "buildFolderPath scriptsFolderPath "
            "resultsFolderPath historyFolderPath "
            "nodeCounts taskCounts coreCounts ompthreadCounts"))

    options = Options(
      general          = general,
      jobs             = jobs,\
      environmentSpace = environmentSpace,\
      parameterSpace   = parameterSpace,\
      \
      deltaRoot      = deltaRoot,\
      outputPath       = outputPath,\
      projectPath      = projectPath,\
      projectName      = projectName,\
      \
      buildFolderPath   = buildFolderPath,\
      scriptsFolderPath = scriptsFolderPath,\
      resultsFolderPath = resultsFolderPath,\
      historyFolderPath = historyFolderPath,\
      \
      nodeCounts = nodeCounts,\
      taskCounts = taskCounts,\
      coreCounts = coreCounts, \
      ompthreadCounts = ompthreadCounts \
    )

    return options
<|endoftext|>


<commit_before>import argparse

from parse import read_vcf, parse_snps_geo, read_phenotypes, generate_snp_acc_mapping, rename_snps
from SubSetMatrix import subset_wrap
from Grouping import method_map
from FSA import fsa_wrapper

def main(args):
    snp_map = generate_snp_acc_mapping(args.snpmap_file)

    if args.ftype=='VCF':
        snps = read_vcf(args.fname)
    elif args.ftype=='GEO':
        snps = parse_snps_geo(args.fname)

    snps = rename_snps(snps, snp_map)

    #snps contains DataFrame with SNP columns and sample rows
    phenotypes = read_phenotypes(args.pheno)

    group_method = method_map[args.group_method]

    groups = group_method(*args.group_method_args)

    subsets = subset_wrap(snps, groups, phenotypes)

    for group, subset in subsets.items():
        #call FSA here
        print(fsa_wrapper(subset, group))

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Associate SNP pair interactions with phenotype.')

    class M():
        fname = 'lapro/GSE63236_series_matrix_noheader.txt'
        ftype = 'GEO'
        pheno = 'phenos.txt'
        snpmap_file = 'GPL3718-44346.txt'
        group_method = 'fake'
        group_method_args = []

    args = M()
    main(args)<commit_msg>Pipeline testing argument addition<commit_after>import argparse

from parse import read_vcf, parse_snps_geo, read_phenotypes, generate_snp_acc_mapping, rename_snps
from SubSetMatrix import subset_wrap
from Grouping import method_map
from FSA import fsa_wrapper

def main(args):
    snp_map = generate_snp_acc_mapping(args.snpmap_file)

    if args.ftype=='VCF':
        snps = read_vcf(args.fname)
    elif args.ftype=='GEO':
        snps = parse_snps_geo(args.fname)

    snps = rename_snps(snps, snp_map)

    #snps contains DataFrame with SNP columns and sample rows
    phenotypes = read_phenotypes(args.pheno)

    group_method = method_map[args.group_method]

    groups = group_method(*args.group_method_args)

    subsets = subset_wrap(snps, groups, phenotypes)

    for group, subset in subsets.items():
        #call FSA here
        print(fsa_wrapper(subset, group))

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Associate SNP pair interactions with phenotype.')

    class M():
        fname = 'lapro/GSE63236_series_matrix_noheader.txt'
        ftype = 'GEO'
        pheno = 'phenos.txt'
        snpmap_file = 'GPL3718-44346.txt'
        group_method = 'fake'
        group_method_args = [250]

    args = M()
    main(args)<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import division
from math import floor
import numpy as np

class Temperature(object):
    def __init__(self, time_step, periods, n):
        self.time_step = time_step
        self.periods = periods
        self.n = n

    @property
    def initial_temp(self):
        return np.array([.7307, .0068])

    @property
    def forcing_ghg_2000(self):
        """Value from DICE2010
        """
        return .83

    @property
    def forcing_ghg_2100(self):
        return .30

    @property
    def transfer_matrix(self):
        """Values from DICE2010
        """
        return np.array([.208, 0., .310, .050])

    @property
    def forcing_co2_doubling(self):
        return 3.8

    @property
    def temp_co2_doubling(self):
        return 3.2

    @property
    def mass_pi(self):
        """Pre-industrial carbon in GtC

        :rtype: float
        """
        return 592.14

    @property
    def forcing_ghg(self):
        """F_EX, Exogenous forcing for other greenhouse gases

        Returns:
            :return: Array of forcing values
            :rtype: np.ndarray

        """
        _n = int(floor(100 / self.time_step))
        a = self.forcing_ghg_2000 + (1. / _n) * (
                self.forcing_ghg_2100 - self.forcing_ghg_2000
            ) * np.arange(_n + 1)
        b = self.forcing_ghg_2100 * np.ones(self.n - (_n + 1)) \
            if self.n > _n \
            else np.array([])
        return np.concatenate((a, b))[:self.n]

    def forcing(self, index, mass_atmosphere):
        """F, Forcing, W/m^2

        Args:
            :param index: Current time step
            :type index: int
            :param mass_atmosphere: Carbon mass in the atmosphere, GtC
            :type mass_atmosphere: float

        Returns:
            :return: Forcing
            :rtype: float

        """
        return (
            self.forcing_co2_doubling *
            (np.log(
                mass_atmosphere / self.mass_pi
            ) / np.log(2)) + self.forcing_ghg[index]
        )

    def temp_ocean(self, temp_atmosphere, temp_ocean):
        """T_LO, increase in atmospheric temperature since 1750, degrees C

         Args:
            :param temp_atmosphere: Atmospheric temperature at t-1
            :type temp_atmosphere: float
            :param temp_ocean: Ocean temperature at t-1
            :type temp_ocean: float

        Returns:
            :returns: T_Ocean(t-1) + ξ_4 * (T_AT(t-1) - T_Ocean(t-1))
            :rtype: float
        """
        return (
            temp_ocean + self.transfer_matrix[3] *
            (temp_atmosphere - temp_ocean)
        )

class DICETemperature(Temperature):

    def temp_atmosphere(self, index, temp_atmosphere, temp_ocean,
                        mass_atmosphere, **kwargs):
        """T_AT, increase in atmospheric temperature since 1750, degrees C

         Args:
            :param temp_atmosphere: Atmospheric temperature at t-1
            :type temp_atmosphere: float
            :param temp_ocean: Lower ocean temperature at t-1
            :type temp_ocean: float
            :param forcing: Forcings at t
            :type forcing: float

        Returns:
            :returns: T_AT(t-1) + ξ_1 * (F(t) - F2xCO2 / T2xCO2 * T_AT(t-1) - ξ_3 * (T_AT(t-1) - T_Ocean(t-1)))
            :rtype: float
        """
        return (
            temp_atmosphere +
            self.transfer_matrix[0] * (
                self.forcing(index, mass_atmosphere) - (
                    self.forcing_co2_doubling / self.temp_co2_doubling) *
                temp_atmosphere - self.transfer_matrix[2] *
                (temp_atmosphere - temp_ocean)
            )
        )

class LinearTemperature(Temperature):
    def temp_atmosphere(self, *args, **kwargs):
        # return (
        #     self.initial_temp[0] +
        #     kwargs['carbon'] * (
        #         self.temp_co2_doubling / ((
        #             2 * self.mass_pi + (
        #                 2 * self.mass_pi -
        #                 (kwargs['initial_carbon'][0] * kwargs['phi11']) -
        #                 (kwargs['initial_carbon'][1] * kwargs['phi21'])
        #             )) * 1e-3)
        #     ) * 1e-3
        # )
        return self.initial_temp[0] + 1.7e-3 * kwargs['carbon']<commit_msg>Cleanup<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import division
from math import floor
import numpy as np

class Temperature(object):
    def __init__(self, time_step, periods, n):
        self.time_step = time_step
        self.periods = periods
        self.n = n

    @property
    def initial_temp(self):
        return np.array([.7307, .0068])

    @property
    def forcing_ghg_2000(self):
        """Value from DICE2010
        """
        return .83

    @property
    def forcing_ghg_2100(self):
        return .30

    @property
    def transfer_matrix(self):
        """Values from DICE2010
        """
        return np.array([.208, 0., .310, .050])

    @property
    def forcing_co2_doubling(self):
        return 3.8

    @property
    def temp_co2_doubling(self):
        return 3.2

    @property
    def mass_pi(self):
        """Pre-industrial carbon in GtC

        :rtype: float
        """
        return 592.14

    @property
    def forcing_ghg(self):
        """F_EX, Exogenous forcing for other greenhouse gases

        Returns:
            :return: Array of forcing values
            :rtype: np.ndarray

        """
        _n = int(floor(100 / self.time_step))
        a = self.forcing_ghg_2000 + (1. / _n) * (
                self.forcing_ghg_2100 - self.forcing_ghg_2000
            ) * np.arange(_n + 1)
        b = self.forcing_ghg_2100 * np.ones(self.n - (_n + 1)) \
            if self.n > _n \
            else np.array([])
        return np.concatenate((a, b))[:self.n]

    def forcing(self, index, mass_atmosphere):
        """F, Forcing, W/m^2

        Args:
            :param index: Current time step
            :type index: int
            :param mass_atmosphere: Carbon mass in the atmosphere, GtC
            :type mass_atmosphere: float

        Returns:
            :return: Forcing
            :rtype: float

        """
        return (
            self.forcing_co2_doubling *
            (np.log(
                mass_atmosphere / self.mass_pi
            ) / np.log(2)) + self.forcing_ghg[index]
        )

    def temp_ocean(self, temp_atmosphere, temp_ocean):
        """T_LO, increase in atmospheric temperature since 1750, degrees C

         Args:
            :param temp_atmosphere: Atmospheric temperature at t-1
            :type temp_atmosphere: float
            :param temp_ocean: Ocean temperature at t-1
            :type temp_ocean: float

        Returns:
            :returns: T_Ocean(t-1) + ξ_4 * (T_AT(t-1) - T_Ocean(t-1))
            :rtype: float
        """
        return (
            temp_ocean + self.transfer_matrix[3] *
            (temp_atmosphere - temp_ocean)
        )

class DICETemperature(Temperature):

    def temp_atmosphere(self, index, temp_atmosphere, temp_ocean,
                        mass_atmosphere, **kwargs):
        """T_AT, increase in atmospheric temperature since 1750, degrees C

         Args:
            :param temp_atmosphere: Atmospheric temperature at t-1
            :type temp_atmosphere: float
            :param temp_ocean: Lower ocean temperature at t-1
            :type temp_ocean: float
            :param forcing: Forcings at t
            :type forcing: float

        Returns:
            :returns: T_AT(t-1) + ξ_1 * (F(t) - F2xCO2 / T2xCO2 * T_AT(t-1) - ξ_3 * (T_AT(t-1) - T_Ocean(t-1)))
            :rtype: float
        """
        return (
            temp_atmosphere +
            self.transfer_matrix[0] * (
                self.forcing(index, mass_atmosphere) - (
                    self.forcing_co2_doubling / self.temp_co2_doubling) *
                temp_atmosphere - self.transfer_matrix[2] *
                (temp_atmosphere - temp_ocean)))

class LinearTemperature(Temperature):
    def temp_atmosphere(self, *args, **kwargs):
        # return (
        #     self.initial_temp[0] +
        #     kwargs['carbon'] * (
        #         self.temp_co2_doubling / ((
        #             2 * self.mass_pi + (
        #                 2 * self.mass_pi -
        #                 (kwargs['initial_carbon'][0] * kwargs['phi11']) -
        #                 (kwargs['initial_carbon'][1] * kwargs['phi21'])
        #             )) * 1e-3)
        #     ) * 1e-3
        # )
        return self.initial_temp[0] + 1.7e-3 * kwargs['carbon']<|endoftext|>


<commit_before>from desertbot.datastore import Base
from sqlalchemy import Column, Integer, String

from twisted.plugin import IPlugin
from desertbot.datastore import sessionScope
from desertbot.moduleinterface import IModule
from desertbot.modules.commandinterface import BotCommand
from zope.interface import implementer

from desertbot.message import IRCMessage
from desertbot.response import IRCResponse, ResponseType

class Pronoun(Base):
    __tablename__ = "pronouns"

    id = Column(Integer, primary_key=True)
    nick = Column(String)
    pronouns = Column(String)

@implementer(IPlugin, IModule)
class Pronouns(BotCommand):
    def triggers(self):
        return ["pronouns", "setpron", "rmpron"]

    def help(self, query):
        return "Commands: pronouns <user>, setpron <pronouns>, rmpron | "\
               "Query the user's pronouns, specify your own pronouns, or remove your pronouns from the database."

    def onLoad(self):
        Base.metadata.create_all(self.bot.databaseEngine)

    def onUnload(self):
        Base.metadata.remove(Pronoun.__table__)

    def execute(self, message: IRCMessage):
        with sessionScope() as session:
            if message.command == "setpron":
                if len(message.parameterList) < 1:
                    return IRCResponse(ResponseType.Say, "Your pronouns are... blank?", message.replyTo)
                userPronouns = Pronoun(nick=message.user.nick.lower(), pronouns=message.parameters)
                session.add(userPronouns)
                return IRCResponse(ResponseType.Say, "Your pronouns have been set as <{}>.".format(message.parameters),
                                   message.replyTo)
            elif message.command == "rmpron":
                userPronouns = session.query(Pronoun).filter(Pronoun.nick == message.user.nick.lower()).first()
                if userPronouns is None:
                    return IRCResponse(ResponseType.Say, "I don't even know your pronouns!", message.replyTo)
                else:
                    session.delete(userPronouns)
                    return IRCResponse(ResponseType.Say, "Your pronouns have been deleted.", message.replyTo)
            elif message.command == "pronouns":
                if len(message.parameterList) < 1:
                    lookup = message.user.nick
                else:
                    lookup = message.parameterList[0]

                user = session.query(Pronoun).filter(Pronoun.nick == lookup.lower()).first()
                if user is None:
                    return IRCResponse(ResponseType.Say, "User's pronouns have not been specified.", message.replyTo)
                else:
                    return IRCResponse(ResponseType.Say, "{} uses <{}> pronouns.".format(lookup, str(user.pronouns)),
                                       message.replyTo)

pronouns = Pronouns()
<commit_msg>[Pronouns] Update pronouns if they already exist<commit_after>from desertbot.datastore import Base
from sqlalchemy import Column, Integer, String

from twisted.plugin import IPlugin
from desertbot.datastore import sessionScope
from desertbot.moduleinterface import IModule
from desertbot.modules.commandinterface import BotCommand
from zope.interface import implementer

from desertbot.message import IRCMessage
from desertbot.response import IRCResponse, ResponseType

class Pronoun(Base):
    __tablename__ = "pronouns"

    id = Column(Integer, primary_key=True)
    nick = Column(String)
    pronouns = Column(String)

@implementer(IPlugin, IModule)
class Pronouns(BotCommand):
    def triggers(self):
        return ["pronouns", "setpron", "rmpron"]

    def help(self, query):
        return "Commands: pronouns <user>, setpron <pronouns>, rmpron | "\
               "Query the user's pronouns, specify your own pronouns, or remove your pronouns from the database."

    def onLoad(self):
        Base.metadata.create_all(self.bot.databaseEngine)

    def onUnload(self):
        Base.metadata.remove(Pronoun.__table__)

    def execute(self, message: IRCMessage):
        with sessionScope() as session:
            if message.command == "setpron":
                if len(message.parameterList) < 1:
                    return IRCResponse(ResponseType.Say, "Your pronouns are... blank?", message.replyTo)
                userPronouns = session.query(Pronoun).filter(Pronoun.nick == message.user.nick.lower()).first()
                if userPronouns is not None:
                    userPronouns.pronouns = message.parameters
                else:
                    userPronouns = Pronoun(nick=message.user.nick.lower(), pronouns=message.parameters)
                session.add(userPronouns)
                return IRCResponse(ResponseType.Say, "Your pronouns have been set as <{}>.".format(message.parameters),
                                   message.replyTo)
            elif message.command == "rmpron":
                userPronouns = session.query(Pronoun).filter(Pronoun.nick == message.user.nick.lower()).first()
                if userPronouns is None:
                    return IRCResponse(ResponseType.Say, "I don't even know your pronouns!", message.replyTo)
                else:
                    session.delete(userPronouns)
                    return IRCResponse(ResponseType.Say, "Your pronouns have been deleted.", message.replyTo)
            elif message.command == "pronouns":
                if len(message.parameterList) < 1:
                    lookup = message.user.nick
                else:
                    lookup = message.parameterList[0]

                user = session.query(Pronoun).filter(Pronoun.nick == lookup.lower()).first()
                if user is None:
                    return IRCResponse(ResponseType.Say, "User's pronouns have not been specified.", message.replyTo)
                else:
                    return IRCResponse(ResponseType.Say, "{} uses <{}> pronouns.".format(lookup, str(user.pronouns)),
                                       message.replyTo)

pronouns = Pronouns()
<|endoftext|>