<commit_before>#!/usr/bin/env python
# Written by Greg Ver Steeg
# See readme.pdf for documentation
# Or go to http://www.isi.edu/~gregv/npeet.html

import scipy.spatial as ss
from scipy.special import digamma
from math import log
import numpy as np
import random
import warnings

# CONTINUOUS ESTIMATORS

def entropy(x, k=3, base=2):
    """ The classic K-L k-nearest neighbor continuous entropy estimator
        x should be a list of vectors, e.g. x = [[1.3], [3.7], [5.1], [2.4]]
        if x is a one-dimensional scalar and we have four samples
    """
    assert k <= len(x) - 1, "Set k smaller than num. samples - 1"
    x = np.asarray(x)
    n_elements, n_features = x.shape
    x = add_noise(x)
    tree = ss.cKDTree(x)
    nn = query_neighbors(tree, x, k)
    const = digamma(n_elements) - digamma(k) + n_features * log(2)
    return (const + n_features * np.log(nn).mean()) / log(base)

def centropy(x, y, k=3, base=2):
    """ The classic K-L k-nearest neighbor continuous entropy estimator for the
        entropy of X conditioned on Y.
    """
    xy = np.c_[x, y]
    entropy_union_xy = entropy(xy, k=k, base=base)
    entropy_y = entropy(y, k=k, base=base)
    return entropy_union_xy - entropy_y

def tc(xs, k=3, base=2):
    xs_columns = np.expand_dims(xs, axis=0).T
    entropy_features = [entropy(col, k=k, base=base) for col in xs_columns]
    return np.sum(entropy_features) - entropy(xs, k, base)

def ctc(xs, y, k=3, base=2):
    xs_columns = np.expand_dims(xs, axis=0).T
    centropy_features = [centropy(col, y, k=k, base=base) for col in xs_columns]
    return np.sum(centropy_features) - centropy(xs, y, k, base)

def corex(xs, ys, k=3, base=2):
    xs_columns = np.expand_dims(xs, axis=0).T
    cmi_features = [mi(col, ys, k=k, base=base) for col in xs_columns]
    return np.sum(cmi_features) - mi(xs, ys, k=k, base=base)

def mi(x, y, z=None, k=3, base=2):
    """ Mutual information of x and y (conditioned on z if z is not None)
        x, y should be a list of vectors, e.g. x = [[1.3], [3.7], [5.1], [2.4]]
        if x is a one-dimensional scalar and we have four samples
    """
    assert len(x) == len(y), "Arrays should have same length"
    assert k <= len(x) - 1, "Set k smaller than num. samples - 1"
    x, y = np.asarray(x), np.asarray(y)
    x = add_noise(x)
    y = add_noise(y)
    points = [x, y]
    if z is not None:
        points.append(z)
    points = np.hstack(points)
    # Find nearest neighbors in joint space, p=inf means max-norm
    tree = ss.cKDTree(points)
    dvec = query_neighbors(tree, points, k)
    if z is None:
        a, b, c, d = avgdigamma(x, dvec), avgdigamma(y, dvec), digamma(k), digamma(len(x))
    else:
        xz = np.c_[x, z]
        yz = np.c_[y, z]
        a, b, c, d = avgdigamma(xz, dvec), avgdigamma(yz, dvec), avgdigamma(z, dvec), digamma(k)
    return (-a - b + c + d) / log(base)

def cmi(x, y, z, k=3, base=2):
    """ Mutual information of x and y, conditioned on z
        Legacy function. Use mi(x, y, z) directly.
    """
    return mi(x, y, z=z, k=k, base=base)

def kldiv(x, xp, k=3, base=2):
    """ KL Divergence between p and q for x~p(x), xp~q(x)
        x, xp should be a list of vectors, e.g. x = [[1.3], [3.7], [5.1], [2.4]]
        if x is a one-dimensional scalar and we have four samples
    """
    assert k < min(len(x), len(xp)), "Set k smaller than num. samples - 1"
    assert len(x[0]) == len(xp[0]), "Two distributions must have same dim."
    d = len(x[0])
    n = len(x)
    m = len(xp)
    const = log(m) - log(n - 1)
    tree = ss.cKDTree(x)
    treep = ss.cKDTree(xp)
    nn = query_neighbors(tree, x, k)
    nnp = query_neighbors(treep, x, k - 1)
    return (const + d * (np.log(nnp).mean() - np.log(nn).mean())) / log(base)

# DISCRETE ESTIMATORS
def entropyd(sx, base=2):
    """ Discrete entropy estimator
        sx is a list of samples
    """
    unique, count = np.unique(sx, return_counts=True, axis=0)
    # Convert to float as otherwise integer division results in all 0 for proba.
    proba = count.astype(float) / len(sx)
    # Avoid 0 division; remove probabilities == 0.0 (removing them does not change the entropy estimate as 0 * log(1/0) = 0.
    proba = proba[proba > 0.0]
    return np.sum(proba * np.log(1. / proba)) / log(base)

def midd(x, y, base=2):
    """ Discrete mutual information estimator
        Given a list of samples which can be any hashable object
    """
    assert len(x) == len(y), "Arrays should have same length"
    return entropyd(x, base) - centropyd(x, y, base)

def cmidd(x, y, z, base=2):
    """ Discrete mutual information estimator
        Given a list of samples which can be any hashable object
    """
    assert len(x) == len(y) == len(z), "Arrays should have same length"
    xz = np.c_[x, z]
    yz = np.c_[y, z]
    xyz = np.c_[x, y, z]
    return entropyd(xz, base) + entropyd(yz, base) - entropyd(xyz, base) - entropyd(z, base)

def centropyd(x, y, base=2):
    """ The classic K-L k-nearest neighbor continuous entropy estimator for the
        entropy of X conditioned on Y.
    """
    xy = np.c_[x, y]
    return entropyd(xy, base) - entropyd(y, base)

def tcd(xs, base=2):
    xs_columns = np.expand_dims(xs, axis=0).T
    entropy_features = [entropyd(col, base=base) for col in xs_columns]
    return np.sum(entropy_features) - entropyd(xs, base)

def ctcd(xs, y, base=2):
    xs_columns = np.expand_dims(xs, axis=0).T
    centropy_features = [centropyd(col, y, base=base) for col in xs_columns]
    return np.sum(centropy_features) - centropyd(xs, y, base)

def corexd(xs, ys, base=2):
    xs_columns = np.expand_dims(xs, axis=0).T
    cmi_features = [midd(col, ys, base=base) for col in xs_columns]
    return np.sum(cmi_features) - midd(xs, ys, base)

# MIXED ESTIMATORS
def micd(x, y, k=3, base=2, warning=True):
    """ If x is continuous and y is discrete, compute mutual information
    """
    assert len(x) == len(y), "Arrays should have same length"
    entropy_x = entropy(x, k, base)

    y_unique, y_count = np.unique(y, return_counts=True, axis=0)
    y_proba = y_count / len(y)

    entropy_x_given_y = 0.
    for yval, py in zip(y_unique, y_proba):
        x_given_y = x[(y == yval).all(axis=1)]
        if k <= len(x_given_y) - 1:
            entropy_x_given_y += py * entropy(x_given_y, k, base)
        else:
            if warning:
                warnings.warn("Warning, after conditioning, on y={yval} insufficient data. "
                              "Assuming maximal entropy in this case.".format(yval=yval))
            entropy_x_given_y += py * entropy_x
    return abs(entropy_x - entropy_x_given_y)  # units already applied

def midc(x, y, k=3, base=2, warning=True):
    return micd(y, x, k, base, warning)

def centropycd(x, y, k=3, base=2, warning=True):
    return entropy(x, base) - micd(x, y, k, base, warning)

def centropydc(x, y, k=3, base=2, warning=True):
    return centropycd(y, x, k=k, base=base, warning=warning)

def ctcdc(xs, y, k=3, base=2, warning=True):
    xs_columns = np.expand_dims(xs, axis=0).T
    centropy_features = [centropydc(col, y, k=k, base=base, warning=warning) for col in xs_columns]
    return np.sum(centropy_features) - centropydc(xs, y, k, base, warning)

def ctccd(xs, y, k=3, base=2, warning=True):
    return ctcdc(y, xs, k=k, base=base, warning=warning)

def corexcd(xs, ys, k=3, base=2, warning=True):
    return corexdc(ys, xs, k=k, base=base, warning=warning)

def corexdc(xs, ys, k=3, base=2, warning=True):
    return tcd(xs, base) - ctcdc(xs, ys, k, base, warning)

# UTILITY FUNCTIONS

def add_noise(x, intens=1e-10):
    # small noise to break degeneracy, see doc.
    return x + intens * np.random.random_sample(x.shape)

def query_neighbors(tree, x, k):
    return tree.query(x, k=k + 1, p=float('inf'), n_jobs=-1)[0][:, k]

def avgdigamma(points, dvec):
    # This part finds number of neighbors in some radius in the marginal space
    # returns expectation value of <psi(nx)>
    n_elements = len(points)
    tree = ss.cKDTree(points)
    avg = 0.
    dvec = dvec - 1e-15
    for point, dist in zip(points, dvec):
        # subtlety, we don't include the boundary point,
        # but we are implicitly adding 1 to kraskov def bc center point is included
        num_points = len(tree.query_ball_point(point, dist, p=float('inf')))
        avg += digamma(num_points) / n_elements
    return avg

# TESTS

def shuffle_test(measure, x, y, z=False, ns=200, ci=0.95, **kwargs):
    """ Shuffle test
        Repeatedly shuffle the x-values and then estimate measure(x, y, [z]).
        Returns the mean and conf. interval ('ci=0.95' default) over 'ns' runs.
        'measure' could me mi, cmi, e.g. Keyword arguments can be passed.
        Mutual information and CMI should have a mean near zero.
    """
    x_clone = np.copy(x)  # A copy that we can shuffle
    outputs = []
    for i in range(ns):
        np.random.shuffle(x_clone)
        if z:
            outputs.append(measure(x_clone, y, z, **kwargs))
        else:
            outputs.append(measure(x_clone, y, **kwargs))
    outputs.sort()
    return np.mean(outputs), (outputs[int((1. - ci) / 2 * ns)], outputs[int((1. + ci) / 2 * ns)])

if __name__ == "__main__":
    print("MI between two independent continuous random variables X and Y:")
    print(mi(np.random.rand(1000, 10), np.random.rand(1000, 3), base=2))
<commit_msg>Add seed to make tests consistent<commit_after>#!/usr/bin/env python
# Written by Greg Ver Steeg
# See readme.pdf for documentation
# Or go to http://www.isi.edu/~gregv/npeet.html

import scipy.spatial as ss
from scipy.special import digamma
from math import log
import numpy as np
import random
import warnings

# CONTINUOUS ESTIMATORS

def entropy(x, k=3, base=2):
    """ The classic K-L k-nearest neighbor continuous entropy estimator
        x should be a list of vectors, e.g. x = [[1.3], [3.7], [5.1], [2.4]]
        if x is a one-dimensional scalar and we have four samples
    """
    assert k <= len(x) - 1, "Set k smaller than num. samples - 1"
    x = np.asarray(x)
    n_elements, n_features = x.shape
    x = add_noise(x)
    tree = ss.cKDTree(x)
    nn = query_neighbors(tree, x, k)
    const = digamma(n_elements) - digamma(k) + n_features * log(2)
    return (const + n_features * np.log(nn).mean()) / log(base)

def centropy(x, y, k=3, base=2):
    """ The classic K-L k-nearest neighbor continuous entropy estimator for the
        entropy of X conditioned on Y.
    """
    xy = np.c_[x, y]
    entropy_union_xy = entropy(xy, k=k, base=base)
    entropy_y = entropy(y, k=k, base=base)
    return entropy_union_xy - entropy_y

def tc(xs, k=3, base=2):
    xs_columns = np.expand_dims(xs, axis=0).T
    entropy_features = [entropy(col, k=k, base=base) for col in xs_columns]
    return np.sum(entropy_features) - entropy(xs, k, base)

def ctc(xs, y, k=3, base=2):
    xs_columns = np.expand_dims(xs, axis=0).T
    centropy_features = [centropy(col, y, k=k, base=base) for col in xs_columns]
    return np.sum(centropy_features) - centropy(xs, y, k, base)

def corex(xs, ys, k=3, base=2):
    xs_columns = np.expand_dims(xs, axis=0).T
    cmi_features = [mi(col, ys, k=k, base=base) for col in xs_columns]
    return np.sum(cmi_features) - mi(xs, ys, k=k, base=base)

def mi(x, y, z=None, k=3, base=2):
    """ Mutual information of x and y (conditioned on z if z is not None)
        x, y should be a list of vectors, e.g. x = [[1.3], [3.7], [5.1], [2.4]]
        if x is a one-dimensional scalar and we have four samples
    """
    assert len(x) == len(y), "Arrays should have same length"
    assert k <= len(x) - 1, "Set k smaller than num. samples - 1"
    x, y = np.asarray(x), np.asarray(y)
    x = add_noise(x)
    y = add_noise(y)
    points = [x, y]
    if z is not None:
        points.append(z)
    points = np.hstack(points)
    # Find nearest neighbors in joint space, p=inf means max-norm
    tree = ss.cKDTree(points)
    dvec = query_neighbors(tree, points, k)
    if z is None:
        a, b, c, d = avgdigamma(x, dvec), avgdigamma(y, dvec), digamma(k), digamma(len(x))
    else:
        xz = np.c_[x, z]
        yz = np.c_[y, z]
        a, b, c, d = avgdigamma(xz, dvec), avgdigamma(yz, dvec), avgdigamma(z, dvec), digamma(k)
    return (-a - b + c + d) / log(base)

def cmi(x, y, z, k=3, base=2):
    """ Mutual information of x and y, conditioned on z
        Legacy function. Use mi(x, y, z) directly.
    """
    return mi(x, y, z=z, k=k, base=base)

def kldiv(x, xp, k=3, base=2):
    """ KL Divergence between p and q for x~p(x), xp~q(x)
        x, xp should be a list of vectors, e.g. x = [[1.3], [3.7], [5.1], [2.4]]
        if x is a one-dimensional scalar and we have four samples
    """
    assert k < min(len(x), len(xp)), "Set k smaller than num. samples - 1"
    assert len(x[0]) == len(xp[0]), "Two distributions must have same dim."
    d = len(x[0])
    n = len(x)
    m = len(xp)
    const = log(m) - log(n - 1)
    tree = ss.cKDTree(x)
    treep = ss.cKDTree(xp)
    nn = query_neighbors(tree, x, k)
    nnp = query_neighbors(treep, x, k - 1)
    return (const + d * (np.log(nnp).mean() - np.log(nn).mean())) / log(base)

# DISCRETE ESTIMATORS
def entropyd(sx, base=2):
    """ Discrete entropy estimator
        sx is a list of samples
    """
    unique, count = np.unique(sx, return_counts=True, axis=0)
    # Convert to float as otherwise integer division results in all 0 for proba.
    proba = count.astype(float) / len(sx)
    # Avoid 0 division; remove probabilities == 0.0 (removing them does not change the entropy estimate as 0 * log(1/0) = 0.
    proba = proba[proba > 0.0]
    return np.sum(proba * np.log(1. / proba)) / log(base)

def midd(x, y, base=2):
    """ Discrete mutual information estimator
        Given a list of samples which can be any hashable object
    """
    assert len(x) == len(y), "Arrays should have same length"
    return entropyd(x, base) - centropyd(x, y, base)

def cmidd(x, y, z, base=2):
    """ Discrete mutual information estimator
        Given a list of samples which can be any hashable object
    """
    assert len(x) == len(y) == len(z), "Arrays should have same length"
    xz = np.c_[x, z]
    yz = np.c_[y, z]
    xyz = np.c_[x, y, z]
    return entropyd(xz, base) + entropyd(yz, base) - entropyd(xyz, base) - entropyd(z, base)

def centropyd(x, y, base=2):
    """ The classic K-L k-nearest neighbor continuous entropy estimator for the
        entropy of X conditioned on Y.
    """
    xy = np.c_[x, y]
    return entropyd(xy, base) - entropyd(y, base)

def tcd(xs, base=2):
    xs_columns = np.expand_dims(xs, axis=0).T
    entropy_features = [entropyd(col, base=base) for col in xs_columns]
    return np.sum(entropy_features) - entropyd(xs, base)

def ctcd(xs, y, base=2):
    xs_columns = np.expand_dims(xs, axis=0).T
    centropy_features = [centropyd(col, y, base=base) for col in xs_columns]
    return np.sum(centropy_features) - centropyd(xs, y, base)

def corexd(xs, ys, base=2):
    xs_columns = np.expand_dims(xs, axis=0).T
    cmi_features = [midd(col, ys, base=base) for col in xs_columns]
    return np.sum(cmi_features) - midd(xs, ys, base)

# MIXED ESTIMATORS
def micd(x, y, k=3, base=2, warning=True):
    """ If x is continuous and y is discrete, compute mutual information
    """
    assert len(x) == len(y), "Arrays should have same length"
    entropy_x = entropy(x, k, base)

    y_unique, y_count = np.unique(y, return_counts=True, axis=0)
    y_proba = y_count / len(y)

    entropy_x_given_y = 0.
    for yval, py in zip(y_unique, y_proba):
        x_given_y = x[(y == yval).all(axis=1)]
        if k <= len(x_given_y) - 1:
            entropy_x_given_y += py * entropy(x_given_y, k, base)
        else:
            if warning:
                warnings.warn("Warning, after conditioning, on y={yval} insufficient data. "
                              "Assuming maximal entropy in this case.".format(yval=yval))
            entropy_x_given_y += py * entropy_x
    return abs(entropy_x - entropy_x_given_y)  # units already applied

def midc(x, y, k=3, base=2, warning=True):
    return micd(y, x, k, base, warning)

def centropycd(x, y, k=3, base=2, warning=True):
    return entropy(x, base) - micd(x, y, k, base, warning)

def centropydc(x, y, k=3, base=2, warning=True):
    return centropycd(y, x, k=k, base=base, warning=warning)

def ctcdc(xs, y, k=3, base=2, warning=True):
    xs_columns = np.expand_dims(xs, axis=0).T
    centropy_features = [centropydc(col, y, k=k, base=base, warning=warning) for col in xs_columns]
    return np.sum(centropy_features) - centropydc(xs, y, k, base, warning)

def ctccd(xs, y, k=3, base=2, warning=True):
    return ctcdc(y, xs, k=k, base=base, warning=warning)

def corexcd(xs, ys, k=3, base=2, warning=True):
    return corexdc(ys, xs, k=k, base=base, warning=warning)

def corexdc(xs, ys, k=3, base=2, warning=True):
    return tcd(xs, base) - ctcdc(xs, ys, k, base, warning)

# UTILITY FUNCTIONS

def add_noise(x, intens=1e-10):
    # small noise to break degeneracy, see doc.
    return x + intens * np.random.random_sample(x.shape)

def query_neighbors(tree, x, k):
    return tree.query(x, k=k + 1, p=float('inf'), n_jobs=-1)[0][:, k]

def avgdigamma(points, dvec):
    # This part finds number of neighbors in some radius in the marginal space
    # returns expectation value of <psi(nx)>
    n_elements = len(points)
    tree = ss.cKDTree(points)
    avg = 0.
    dvec = dvec - 1e-15
    for point, dist in zip(points, dvec):
        # subtlety, we don't include the boundary point,
        # but we are implicitly adding 1 to kraskov def bc center point is included
        num_points = len(tree.query_ball_point(point, dist, p=float('inf')))
        avg += digamma(num_points) / n_elements
    return avg

# TESTS

def shuffle_test(measure, x, y, z=False, ns=200, ci=0.95, **kwargs):
    """ Shuffle test
        Repeatedly shuffle the x-values and then estimate measure(x, y, [z]).
        Returns the mean and conf. interval ('ci=0.95' default) over 'ns' runs.
        'measure' could me mi, cmi, e.g. Keyword arguments can be passed.
        Mutual information and CMI should have a mean near zero.
    """
    x_clone = np.copy(x)  # A copy that we can shuffle
    outputs = []
    for i in range(ns):
        np.random.shuffle(x_clone)
        if z:
            outputs.append(measure(x_clone, y, z, **kwargs))
        else:
            outputs.append(measure(x_clone, y, **kwargs))
    outputs.sort()
    return np.mean(outputs), (outputs[int((1. - ci) / 2 * ns)], outputs[int((1. + ci) / 2 * ns)])

if __name__ == "__main__":
    print("MI between two independent continuous random variables X and Y:")
    np.random.seed(0)
    print(mi(np.random.rand(1000, 10), np.random.rand(1000, 3), base=2))
<|endoftext|>


<commit_before>from flask import Flask, abort, redirect, render_template, request, url_for
from flask.ext import restful

from flask_admin import Admin
from flask_admin import helpers as admin_helpers
from flask_admin.contrib import sqla

from flask_security import SQLAlchemyUserDatastore, Security, current_user

from .models import Location, Role, User, db

app = Flask(__name__)
app.config.from_pyfile('config.py')
db.init_app(app)
app.db = db

# Setup api
app.api = restful.Api(app)
from .api import api_initialize
api_initialize()

@app.route('/')
def index():
    return render_template('index.html')

# Setup Flask-Security
user_datastore = SQLAlchemyUserDatastore(app.db, User, Role)
security = Security(app, user_datastore)

# Create customized model view classes
class SecureView(sqla.ModelView):
    def is_accessible(self):
        """Deny access if current_user isn't a logged-in admin"""
        return (current_user.is_active and
                current_user.is_authenticated and
                current_user.has_role('Administrator'))

    def _handle_view(self, name, **kwargs):
        """Redirect users when a view is not accessible"""
        if not self.is_accessible():
            if current_user.is_authenticated:
                # permission denied
                abort(403)
            else:
                # login
                return redirect(url_for('security.login', next=request.url))

class StandardFilteredView(sqla.ModelView):
    def is_accessible(self):
        """Deny access if current_user isn't a logged-in admin"""
        return (current_user.is_active and
                current_user.is_authenticated and
                (current_user.has_role('Administrator') or current_user.has_role('Standard')))

    def _handle_view(self, name, **kwargs):
        """Redirect users when a view is not accessible"""
        if not self.is_accessible():
            if current_user.is_authenticated:
                # permission denied
                abort(403)
            else:
                # login
                return redirect(url_for('security.login', next=request.url))

        self.can_create = self.can_delete = current_user.has_role('Administrator')

    # Given a location id, are we allowed to edit it?
    def is_owned(self, id):
        if current_user.has_role('Administrator'):
            return True
        allowed_locations = [location.id for location in current_user.locations]
        return int(id) in allowed_locations

    # Overrides to check model ownership
    def on_model_change(self, form, model, is_created):
        if not self.is_owned(model.id):
            abort(403)

    def on_form_prefill(self, form, id):
        if not self.is_owned(id):
            abort(403)

    def on_model_delete(self, model):
        if not self.is_owned(model.id):
            abort(403)

    # Query Overrides to limit Standard Users to Locations they Own

    def get_query(self):
        allowed_locations = [location.id for location in current_user.locations]
        if current_user.has_role('Standard') and not current_user.has_role('Administrator'):
            return self.session.query(self.model).filter(self.model.id.in_(allowed_locations))
        elif current_user.has_role('Administrator'):
            return self.session.query(self.model)

    def get_count_query(self):
        if current_user.has_role('Administrator'):
            return super(StandardFilteredView, self).get_count_query()
        elif current_user.has_role('Standard'):
            allowed_locations = [location.id for location in current_user.locations]
            return super(StandardFilteredView, self).get_count_query().filter(self.model.id.in_(allowed_locations))

class LocationModelView(StandardFilteredView):
    _list_columns = ["name", "services", "city", "state"]
    _cols = [
        ("name", "e.g. Food Bank of Alaska"),
        ("services", "Click for drop-down choices. May select multiple "
            "services. Type to filter."),
        ("description", ""),
        ("address_line1", "e.g. 123 Main St."),
        ("address_line2", "e.g. Ste. 200"),
        ("address_line3", "e.g. Fairbanks, AK 99775"),
        ("phone", "e.g. xxx-xxx-xxxx"),
        ("contact_email", "e.g. example@email.com"),
        ("website", "e.g. wehelppeople.org or www.wehelppeople.org"),
        ("opening_time", "Useful for locations with regular hours."),
        ("closing_time", "Useful for locations with regular hours."),
        ("days_of_week", "Useful for locations with regular hours."),
        ("city", "Useful for sorting locations.  Not sent to mobile devices."),
        ("state", "Useful for sorting locations.  Not sent to mobile devices."),
    ]
    can_view_details = True
    column_details_list = [name for name, _ in _cols]
    column_default_sort = "name"
    column_descriptions = dict(_cols)
    column_editable_list = ["city", "state"]
    column_list = _list_columns  # List view only
    form_columns = [name for name, _ in _cols if name not in ["city", "state"]]

class UserModelView(SecureView):
    _cols = [
        ("username", "Account Name"),
        ("roles", "User Permissions"),
        ("locations", "Locations Standard user is allowed to edit"),
        ("email", "User Email Address"),
        ("password", "User Password"),
        ("active", "Is User Enabled?"),
    ]
    column_default_sort = "username"
    column_descriptions = dict(_cols)
    column_exclude_list = ['password', ]

# Setup Flask-Admin
admin = Admin(app, name='Time of Need Admin', template_mode='bootstrap3',
              base_template='my_master.html')
admin.add_view(LocationModelView(Location, db.session, name="Locations"))
admin.add_view(UserModelView(User, db.session, name="Users"))

# Define a context processor for merging flask-admin's template context into
# the flask-security views.
@security.context_processor
def security_context_processor():
    return dict(
        admin_base_template=admin.base_template,
        admin_view=admin.index_view,
        h=admin_helpers,
    )
<commit_msg>Flake8 lint<commit_after>from flask import Flask, abort, redirect, render_template, request, url_for
from flask.ext import restful
from flask_admin import Admin
from flask_admin import helpers as admin_helpers
from flask_admin.contrib import sqla
from flask_security import SQLAlchemyUserDatastore, Security, current_user

from .models import Location, Role, User, db

app = Flask(__name__)
app.config.from_pyfile('config.py')
db.init_app(app)
app.db = db

# Setup api
app.api = restful.Api(app)
from .api import api_initialize  # noqa
api_initialize()

@app.route('/')
def index():
    return render_template('index.html')

# Setup Flask-Security
user_datastore = SQLAlchemyUserDatastore(app.db, User, Role)
security = Security(app, user_datastore)

# Create customized model view classes
class SecureView(sqla.ModelView):
    def is_accessible(self):
        """Deny access if current_user isn't a logged-in admin"""
        return (current_user.is_active and
                current_user.is_authenticated and
                current_user.has_role('Administrator'))

    def _handle_view(self, name, **kwargs):
        """Redirect users when a view is not accessible"""
        if not self.is_accessible():
            if current_user.is_authenticated:
                # permission denied
                abort(403)
            else:
                # login
                return redirect(url_for('security.login', next=request.url))

class StandardFilteredView(sqla.ModelView):
    def is_accessible(self):
        """Deny access if current_user isn't a logged-in admin"""
        return (current_user.is_active and
                current_user.is_authenticated and (
                    current_user.has_role('Administrator') or
                    current_user.has_role('Standard')))

    def _handle_view(self, name, **kwargs):
        """Redirect users when a view is not accessible"""
        if not self.is_accessible():
            if current_user.is_authenticated:
                # permission denied
                abort(403)
            else:
                # login
                return redirect(url_for('security.login', next=request.url))

        self.can_create = self.can_delete = current_user.has_role('Administrator')  # noqa

    # Given a location id, are we allowed to edit it?
    def is_owned(self, id):
        if current_user.has_role('Administrator'):
            return True
        allowed_locations = [location.id for location in current_user.locations]
        return int(id) in allowed_locations

    # Overrides to check model ownership
    def on_model_change(self, form, model, is_created):
        if not self.is_owned(model.id):
            abort(403)

    def on_form_prefill(self, form, id):
        if not self.is_owned(id):
            abort(403)

    def on_model_delete(self, model):
        if not self.is_owned(model.id):
            abort(403)

    # Query Overrides to limit Standard Users to Locations they Own

    def get_query(self):
        allowed_locations = [location.id for location in current_user.locations]
        if current_user.has_role('Administrator'):
            return self.session.query(self.model)
        elif current_user.has_role('Standard'):
            return self.session.query(self.model).filter(
                self.model.id.in_(allowed_locations))

    def get_count_query(self):
        allowed_locations = [location.id for location in current_user.locations]
        if current_user.has_role('Administrator'):
            return super(StandardFilteredView, self).get_count_query()
        elif current_user.has_role('Standard'):
            return super(StandardFilteredView, self).get_count_query().filter(
                self.model.id.in_(allowed_locations))

class LocationModelView(StandardFilteredView):
    _list_columns = ["name", "services", "city", "state"]
    _cols = [
        ("name", "e.g. Food Bank of Alaska"),
        ("services", "Click for drop-down choices. May select multiple "
            "services. Type to filter."),
        ("description", ""),
        ("address_line1", "e.g. 123 Main St."),
        ("address_line2", "e.g. Ste. 200"),
        ("address_line3", "e.g. Fairbanks, AK 99775"),
        ("phone", "e.g. xxx-xxx-xxxx"),
        ("contact_email", "e.g. example@email.com"),
        ("website", "e.g. wehelppeople.org or www.wehelppeople.org"),
        ("opening_time", "Useful for locations with regular hours."),
        ("closing_time", "Useful for locations with regular hours."),
        ("days_of_week", "Useful for locations with regular hours."),
        ("city", "Useful for sorting locations.  Not sent to mobile devices."),
        ("state", "Useful for sorting locations.  Not sent to mobile devices."),
    ]
    can_view_details = True
    column_details_list = [name for name, _ in _cols]
    column_default_sort = "name"
    column_descriptions = dict(_cols)
    column_editable_list = ["city", "state"]
    column_list = _list_columns  # List view only
    form_columns = [name for name, _ in _cols if name not in ["city", "state"]]

class UserModelView(SecureView):
    _cols = [
        ("username", "Account Name"),
        ("roles", "User Permissions"),
        ("locations", "Locations Standard user is allowed to edit"),
        ("email", "User Email Address"),
        ("password", "User Password"),
        ("active", "Is User Enabled?"),
    ]
    column_default_sort = "username"
    column_descriptions = dict(_cols)
    column_exclude_list = ['password', ]

# Setup Flask-Admin
admin = Admin(app, name='Time of Need Admin', template_mode='bootstrap3',
              base_template='my_master.html')
admin.add_view(LocationModelView(Location, db.session, name="Locations"))
admin.add_view(UserModelView(User, db.session, name="Users"))

# Define a context processor for merging flask-admin's template context into
# the flask-security views.
@security.context_processor
def security_context_processor():
    return dict(
        admin_base_template=admin.base_template,
        admin_view=admin.index_view,
        h=admin_helpers,
    )
<|endoftext|>


<commit_before>import sys
from yt.mods import *

filetype = "sedov_hdf5_chk_"

argvs = sys.argv
argc  = len(argvs)
if (argc != 4):
#    print 'python plot1d.py <tbgn> <tend> <dtsp>'
    print ('python plot1d.py <tbgn> <tend> <dtsp>')
    quit()

tbgn = int(argvs[1])
tend = int(argvs[2])
dtsp = int(argvs[3])

for time in range(tbgn,tend,dtsp):
    ctime = str(time).zfill(4)
    ifile = filetype + ctime
    ofile = "mesh_t" + ctime + ".dat"

    if not os.path.exists(ifile) or os.path.exists(ofile) :
        continue

    ds = load(ifile)
    ad = ds.h.all_data()
    obj = open(ofile, 'w')
    for i in range(ad["temp"].size):
        print ("%e %e %e %e %e %e" % (ad["r"][i], ad["dens"][i], ad["temp"][i], ad["pres"][i], ad["ener"][i], ad["velx"][i]), file=obj)
#        print ("%e %e %e %e %e %e" % (ad["x"][i], ad["dens"][i], ad["temp"][i], ad["pres"][i], ad["ener"][i], ad["velx"][i]), file=obj)

<commit_msg>revise a bit<commit_after>import sys
from yt.mods import *

filetype = "sedov_hdf5_chk_"

argvs = sys.argv
argc  = len(argvs)
if (argc != 4):
#    print 'python plot1d.py <tbgn> <tend> <dtsp>'
    print ('python plot1d.py <tbgn> <tend> <dtsp>')
    quit()

tbgn = int(argvs[1])
tend = int(argvs[2])
dtsp = int(argvs[3])

for time in range(tbgn,tend,dtsp):
    ctime = str(time).zfill(4)
    ifile = filetype + ctime
    ofile = "mesh_t" + ctime + ".dat"

    if not os.path.exists(ifile) or os.path.exists(ofile) :
        continue

    ds = load(ifile)
    ad = ds.h.all_data()
    obj = open(ofile, 'w')
    for i in range(ad["temp"].size):
        print ("%e %e %e %e %e %e" % (ad["r"][i], ad["dens"][i], ad["temp"][i], ad["pres"][i], ad["ener"][i], ad["velx"][i]), file=obj)

<|endoftext|>


<commit_before>import os
import sys
import radical.pilot
import saga

# DBURL defines the MongoDB server URL and has the format mongodb://host:port.
# For the installation of a MongoDB server, refer to the MongoDB website:
# http://docs.mongodb.org/manual/installation/
DBURL = os.getenv("RADICALPILOT_DBURL")
if DBURL is None:
    print "ERROR: RADICALPILOT_DBURL (MongoDB server URL) is not defined."
    sys.exit(1)

# RCONF points to the resource configuration files. Read more about resource
# configuration files at http://saga-pilot.readthedocs.org/en/latest/machconf.html
RCONF = ["https://raw.github.com/radical-cybertools/radical.pilot/master/configs/xsede.json",
          "https://raw.github.com/radical-cybertools/radical.pilot/master/configs/futuregrid.json"]

INDIA_STAGING = '///N/u/marksant/staging_area/'
INDIA_HOST = 'india.futuregrid.org'

SHARED_INPUT_FILE = 'shared_input_file.txt'

#------------------------------------------------------------------------------
#
if __name__ == "__main__":

    try:
        # Create a new session. A session is the 'root' object for all other
        # RADICAL-Pilot objects. It encapsualtes the MongoDB connection(s) as
        # well as security crendetials.
        session = radical.pilot.Session(database_url=DBURL)

        # Add a Pilot Manager. Pilot managers manage one or more ComputePilots.
        pmgr = radical.pilot.PilotManager(session=session, resource_configurations=RCONF)

        # Define a 32-core on stamped that runs for 15 mintutes and
        # uses $HOME/radical.pilot.sandbox as sandbox directoy.
        pdesc = radical.pilot.ComputePilotDescription()
        pdesc.resource = "india.futuregrid.org"
        pdesc.runtime = 15 # minutes
        pdesc.cores = 8

        # Launch the pilot.
        pilot = pmgr.submit_pilots(pdesc)

        # Define and open staging directory on the remote machine
        remote_dir_url = saga.Url()
        remote_dir_url.scheme = 'sftp'
        remote_dir_url.host = INDIA_HOST
        remote_dir_url.path = INDIA_STAGING
        remote_dir = saga.filesystem.Directory(remote_dir_url)

        # Define the url of the local file in the local directory
        shared_input_file_url = saga.Url('file://%s/%s' % (os.getcwd(), SHARED_INPUT_FILE))
        # Upload the local file to the remote staging area
        remote_dir.copy(shared_input_file_url, '.')

        # Configure the staging directive for shared input file.
        sd_shared = radical.pilot.StagingDirectives()
        sd_shared.source = os.path.join(INDIA_STAGING, SHARED_INPUT_FILE)
        sd_shared.target = SHARED_INPUT_FILE
        sd_shared.action = radical.pilot.StagingDirectives.LINK

        # Combine the ComputePilot, the ComputeUnits and a scheduler via
        # a UnitManager object.
        umgr = radical.pilot.UnitManager(
            session=session,
            scheduler=radical.pilot.SCHED_DIRECT_SUBMISSION)

        # Add the previsouly created ComputePilot to the UnitManager.
        umgr.add_pilots(pilot)

        compute_units = []

        for unit_count in range(4):

            # Configure the staging directive for input file.
            sd_input = radical.pilot.StagingDirectives()
            sd_input.source = 'input_file-%d' % unit_count
            sd_input.target = 'input_file-%d' % unit_count
            sd_input.action = radical.pilot.StagingDirectives.TRANSFER

            # Configure the staging directive for output file.
            sd_output = radical.pilot.StagingDirectives()
            sd_output.source = 'output_file-%d' % unit_count
            sd_output.target = 'output_file-%d' % unit_count
            sd_output.action = radical.pilot.StagingDirectives.TRANSFER

            # Actual task description.
            # Concatenate the shared input and the task specific input.
            cu = radical.pilot.ComputeUnitDescription()
            cu.executable = '/bin/cat'
            cu.arguments = ('shared_input_file.txt input_file-%d.txt > output-file-%d.txt' % (unit_count, unit_count)).split()
            cu.cores = 1
            cu.input_staging = [sd_shared, sd_input]
            cu.output_staging = sd_output

            compute_units.append(cu)

        # Submit the previously created ComputeUnit descriptions to the
        # PilotManager. This will trigger the selected scheduler to start
        # assigning ComputeUnits to the ComputePilots.
        units = umgr.submit_units(compute_units)

        # Wait for all compute units to finish.
        umgr.wait_units()

        for unit in umgr.get_units():
            # Print some information about the unit.
            #print "\n{0}".format(str(unit))

            # Get the stdout and stderr streams of the ComputeUnit.
            print " STDOUT: {0}".format(unit.stdout)
            print " STDERR: {0}".format(unit.stderr)

        session.close()

    except radical.pilot.PilotException, ex:
        print "Error: %s" % ex
<commit_msg>Minor changes to example.<commit_after>import os
import sys
import radical.pilot
import saga

# DBURL defines the MongoDB server URL and has the format mongodb://host:port.
# For the installation of a MongoDB server, refer to the MongoDB website:
# http://docs.mongodb.org/manual/installation/
DBURL = os.getenv("RADICALPILOT_DBURL")
if DBURL is None:
    print "ERROR: RADICALPILOT_DBURL (MongoDB server URL) is not defined."
    sys.exit(1)

# RCONF points to the resource configuration files. Read more about resource
# configuration files at http://saga-pilot.readthedocs.org/en/latest/machconf.html
RCONF = ["https://raw.github.com/radical-cybertools/radical.pilot/master/configs/xsede.json",
         "https://raw.github.com/radical-cybertools/radical.pilot/master/configs/futuregrid.json"]

INDIA_STAGING = '///N/u/marksant/staging_area/'
INDIA_HOST = 'india.futuregrid.org'

SHARED_INPUT_FILE = 'shared_input_file.txt'

#------------------------------------------------------------------------------
#
if __name__ == "__main__":

    try:
        # Create a new session. A session is the 'root' object for all other
        # RADICAL-Pilot objects. It encapsualtes the MongoDB connection(s) as
        # well as security crendetials.
        session = radical.pilot.Session(database_url=DBURL)

        # Add a Pilot Manager. Pilot managers manage one or more ComputePilots.
        pmgr = radical.pilot.PilotManager(session=session, resource_configurations=RCONF)

        # Define a 32-core on stamped that runs for 15 mintutes and
        # uses $HOME/radical.pilot.sandbox as sandbox directoy.
        pdesc = radical.pilot.ComputePilotDescription()
        pdesc.resource = "india.futuregrid.org"
        pdesc.runtime = 15 # minutes
        pdesc.cores = 8

        # Launch the pilot.
        pilot = pmgr.submit_pilots(pdesc)

        # Define and open staging directory on the remote machine
        remote_dir_url = saga.Url()
        remote_dir_url.scheme = 'sftp'
        remote_dir_url.host = INDIA_HOST
        remote_dir_url.path = INDIA_STAGING
        remote_dir = saga.filesystem.Directory(remote_dir_url)

        # Define the url of the local file in the local directory
        shared_input_file_url = saga.Url('file://%s/%s' % (os.getcwd(), SHARED_INPUT_FILE))
        # Upload the local file to the remote staging area
        remote_dir.copy(shared_input_file_url, '.')

        # Configure the staging directive for shared input file.
        sd_shared = radical.pilot.StagingDirectives()
        sd_shared.source = os.path.join(INDIA_STAGING, SHARED_INPUT_FILE)
        sd_shared.target = SHARED_INPUT_FILE
        sd_shared.action = radical.pilot.LINK

        # Combine the ComputePilot, the ComputeUnits and a scheduler via
        # a UnitManager object.
        umgr = radical.pilot.UnitManager(
            session=session,
            scheduler=radical.pilot.SCHED_DIRECT_SUBMISSION)

        # Add the previsouly created ComputePilot to the UnitManager.
        umgr.add_pilots(pilot)

        compute_units = []

        for unit_count in range(1):

            # Configure the staging directive for input file.
            sd_input = radical.pilot.StagingDirectives()
            sd_input.source = 'input_file-%d.txt' % unit_count
            sd_input.target = 'input_file-%d.txt' % unit_count
            sd_input.action = radical.pilot.TRANSFER

            # Configure the staging directive for output file.
            sd_output = radical.pilot.StagingDirectives()
            sd_output.source = 'output_file-%d.txt' % unit_count
            sd_output.target = 'output_file-%d.txt' % unit_count
            sd_output.action = radical.pilot.TRANSFER

            # Actual task description.
            # Concatenate the shared input and the task specific input.
            cu = radical.pilot.ComputeUnitDescription()
            cu.executable = '/bin/cat'
            cu.arguments = ('shared_input_file.txt input_file-%d.txt > output-file-%d.txt' % (unit_count, unit_count)).split()
            cu.cores = 1
            cu.input_staging = [sd_shared, sd_input]
            cu.output_staging = sd_output

            compute_units.append(cu)

        # Submit the previously created ComputeUnit descriptions to the
        # PilotManager. This will trigger the selected scheduler to start
        # assigning ComputeUnits to the ComputePilots.
        units = umgr.submit_units(compute_units)

        # Wait for all compute units to finish.
        umgr.wait_units()

        for unit in umgr.get_units():
            # Print some information about the unit.
            #print "\n{0}".format(str(unit))

            # Get the stdout and stderr streams of the ComputeUnit.
            print " STDOUT: {0}".format(unit.stdout)
            print " STDERR: {0}".format(unit.stderr)

        session.close()

    except radical.pilot.PilotException, ex:
        print "Error: %s" % ex
<|endoftext|>


<commit_before># ----------------------------------------------------------------------
# Copyright (c) 2011-2016 Raytheon BBN Technologies
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and/or hardware specification (the "Work") to
# deal in the Work without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Work, and to permit persons to whom the Work
# is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Work.
#
# THE WORK IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE WORK OR THE USE OR OTHER DEALINGS
# IN THE WORK.
# ----------------------------------------------------------------------

# This gets called when we invoke any of the CH methods

import xmlrpclib

import tools.pluginmanager as pm
import plugins.chapiv1rpc.plugin
import plugins.chrm.plugin
import plugins.csrm.plugin
import plugins.flaskrest.plugin
import plugins.logging.plugin
import plugins.opsmon.plugin
import plugins.marm.plugin
import plugins.sarm.plugin

from tools.chapi_log import *

pm.registerService('xmlrpc', pm.XMLRPCHandler())
pm.registerService('config', pm.ConfigDB())
pm.registerService('rpcserver', pm.RESTServer())
pm.registerService(pm.ENVIRONMENT_SERVICE, pm.WSGIEnvironment())

initialized = False

def initialize():
    global initialized
    if not initialized:
        plugins.chapiv1rpc.plugin.setup()
        plugins.chrm.plugin.setup()
        plugins.csrm.plugin.setup()
        plugins.flaskrest.plugin.setup()
        plugins.logging.plugin.setup()
        plugins.opsmon.plugin.setup()
        plugins.marm.plugin.setup()
        plugins.sarm.plugin.setup()
        chapi_info("CH_SERVER",  "INITIALIZED CH_SERVER")
        initialized = True

def application(environ, start_response):

    initialize()

#    print "ENV = " + str(environ)
    start_response('200 OK', [('Content-Type', 'text/html')])
    try:
        result = handleCall(environ)
        return [result]
    except Exception as e:
        msg = "%s: %s" % (type(e).__name__, str(e))
        fault = xmlrpclib.Fault(1, msg)
        response = xmlrpclib.dumps(fault, allow_none=True)
        return [response]

def handleCall(environ):
    # Try to handle REST invocations, registered with rpcserver
    rest_output = handle_REST_call(environ)
    if rest_output:
        # It is a REST invocation
        return rest_output
    else:
        # Otherwise it is an XMLRPC invocation
        xmlrpc_output = handle_XMLRPC_call(environ)
        return xmlrpc_output

# Handle XMLRPC invocation
def handle_XMLRPC_call(environ):

    xmlrpc_endpoint = environ['REQUEST_URI']
    xmlrpc = pm.getService('xmlrpc')
    handler_entry = xmlrpc.lookupByEndpoint(xmlrpc_endpoint)
    handler = handler_entry._instance

    length = int(environ['CONTENT_LENGTH'])
    wsgi_input = environ['wsgi.input']
    data = wsgi_input.read(length)
    decoded_data = xmlrpclib.loads(data)

    args = decoded_data[0]
    method = decoded_data[1]
    fcn = getattr(handler, method)

    envService = pm.getService(pm.ENVIRONMENT_SERVICE)
    envService.setEnvironment(environ)
    try:
        method_response = fcn(*args)
        # print "RESPONSE = %r" % (method_response)
    finally:
        # Always clear the environment after the call is complete,
        # even if there was an exception
        envService.clearEnvironment()
    response = xmlrpclib.dumps((method_response, ), allow_none=True)
    return response

# Determine if this is a REST call. If so, make the call and return output
def handle_REST_call(environ):
    if 'PATH_INFO' in environ:
        path_info = environ['PATH_INFO']
        rpcserver = pm.getService('rpcserver')
        handler = rpcserver.app.lookup_handler(path_info)
        if handler:
            pieces = path_info.split('/')
            if len(pieces) < 4:
                return None
            variety = pieces[2]
            id = pieces[3]
            output = handler(variety, id)
            return output
        return None
<commit_msg>Fix XML-RPC responses<commit_after># ----------------------------------------------------------------------
# Copyright (c) 2011-2016 Raytheon BBN Technologies
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and/or hardware specification (the "Work") to
# deal in the Work without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Work, and to permit persons to whom the Work
# is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Work.
#
# THE WORK IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE WORK OR THE USE OR OTHER DEALINGS
# IN THE WORK.
# ----------------------------------------------------------------------

# This gets called when we invoke any of the CH methods

import xmlrpclib

import tools.pluginmanager as pm
import plugins.chapiv1rpc.plugin
import plugins.chrm.plugin
import plugins.csrm.plugin
import plugins.flaskrest.plugin
import plugins.logging.plugin
import plugins.opsmon.plugin
import plugins.marm.plugin
import plugins.sarm.plugin

from tools.chapi_log import *

pm.registerService('xmlrpc', pm.XMLRPCHandler())
pm.registerService('config', pm.ConfigDB())
pm.registerService('rpcserver', pm.RESTServer())
pm.registerService(pm.ENVIRONMENT_SERVICE, pm.WSGIEnvironment())

initialized = False

def initialize():
    global initialized
    if not initialized:
        plugins.chapiv1rpc.plugin.setup()
        plugins.chrm.plugin.setup()
        plugins.csrm.plugin.setup()
        plugins.flaskrest.plugin.setup()
        plugins.logging.plugin.setup()
        plugins.opsmon.plugin.setup()
        plugins.marm.plugin.setup()
        plugins.sarm.plugin.setup()
        chapi_info("CH_SERVER",  "INITIALIZED CH_SERVER")
        initialized = True

def application(environ, start_response):

    initialize()

#    print "ENV = " + str(environ)
    start_response('200 OK', [('Content-Type', 'text/html')])
    try:
        result = handleCall(environ)
        return [result]
    except Exception as e:
        msg = "%s: %s" % (type(e).__name__, str(e))
        fault = xmlrpclib.Fault(1, msg)
        response = xmlrpclib.dumps(fault, methodresponse=True, allow_none=True)
        return [response]

def handleCall(environ):
    # Try to handle REST invocations, registered with rpcserver
    rest_output = handle_REST_call(environ)
    if rest_output:
        # It is a REST invocation
        return rest_output
    else:
        # Otherwise it is an XMLRPC invocation
        xmlrpc_output = handle_XMLRPC_call(environ)
        return xmlrpc_output

# Handle XMLRPC invocation
def handle_XMLRPC_call(environ):

    xmlrpc_endpoint = environ['REQUEST_URI']
    xmlrpc = pm.getService('xmlrpc')
    handler_entry = xmlrpc.lookupByEndpoint(xmlrpc_endpoint)
    handler = handler_entry._instance

    length = int(environ['CONTENT_LENGTH'])
    wsgi_input = environ['wsgi.input']
    data = wsgi_input.read(length)
    decoded_data = xmlrpclib.loads(data)

    args = decoded_data[0]
    method = decoded_data[1]
    fcn = getattr(handler, method)

    envService = pm.getService(pm.ENVIRONMENT_SERVICE)
    envService.setEnvironment(environ)
    try:
        method_response = fcn(*args)
        # print "RESPONSE = %r" % (method_response)
    finally:
        # Always clear the environment after the call is complete,
        # even if there was an exception
        envService.clearEnvironment()
    response = xmlrpclib.dumps((method_response,), methodresponse=True, allow_none=True)
    return response

# Determine if this is a REST call. If so, make the call and return output
def handle_REST_call(environ):
    if 'PATH_INFO' in environ:
        path_info = environ['PATH_INFO']
        rpcserver = pm.getService('rpcserver')
        handler = rpcserver.app.lookup_handler(path_info)
        if handler:
            pieces = path_info.split('/')
            if len(pieces) < 4:
                return None
            variety = pieces[2]
            id = pieces[3]
            output = handler(variety, id)
            return output
        return None
<|endoftext|>


<commit_before>#!/usr/bin/python

import os
import os.path
import random
import string
import sys
import time
import traceback

import github_update
import universe_builder

class CIUploader(object):

    def __init__(self, package_name, input_dir_path, artifact_paths, package_version = 'stub-universe'):
        self.__pkg_name = package_name
        self.__pkg_version = package_version
        self.__input_dir_path = input_dir_path

        self.__aws_region = os.environ.get('AWS_UPLOAD_REGION', 'us-west-2')
        # sample s3_directory: 'infinity-artifacts/autodelete7d/kafka/20160815-134747-S6vxd0gRQBw43NNy'
        s3_base_path = os.environ.get('S3_BASE_PATH', 'infinity-artifacts/autodelete7d')
        dir_name = '{}-{}'.format(
            time.strftime("%Y%m%d-%H%M%S"),
            ''.join([random.SystemRandom().choice(string.ascii_letters + string.digits) for i in range(16)]))
        self.__s3_directory = '/'.join([s3_base_path, self.__pkg_name, dir_name])
        self.__http_directory = 'https://s3-{}.amazonaws.com/{}'.format(self.__aws_region, self.__s3_directory)

        self.__github_updater = github_update.GithubStatusUpdater('upload:{}'.format(package_name))

        if not os.path.isdir(input_dir_path):
            err = 'Provided package path is not a directory: {}'.format(input_dir_path)
            self.__github_updater.update('error', err)
            raise Exception(err)

        self.__artifact_paths = []
        for artifact_path in artifact_paths:
            if not os.path.isfile(artifact_path):
                err = 'Provided package path is not a file: {} (full list: {})'.format(artifact_path, artifact_paths)
                raise Exception(err)
            if artifact_path in self.__artifact_paths:
                err = 'Duplicate filename between "{}" and "{}". Artifact filenames must be unique.'.format(prior_path, artifact_path)
                self.__github_updater.update('error', err)
                raise Exception(err)
            self.__artifact_paths.append(artifact_path)

    def __upload_artifact(self, filepath):
        filename = os.path.basename(filepath)
        cmd = 'aws s3 --region={} cp --acl public-read {} s3://{}/{}'.format(
            self.__aws_region, filepath, self.__s3_directory, filename)
        print(cmd)
        ret = os.system(cmd)
        if not ret == 0:
            err = 'Failed to upload {} to S3'.format(filename)
            self.__github_updater.update('error', err)
            raise Exception(err)
        return '/'.join([self.__http_directory, filename])

    def __spam_universe_url(self, universe_url):
        if os.environ.get('JENKINS_HOME', ''):
            # write jenkins properties file:
            properties_file = open(os.path.join(os.environ['WORKSPACE'], 'stub-universe.properties'), 'w')
            properties_file.write('STUB_UNIVERSE_URL={}\n'.format(universe_url))
            properties_file.write('STUB_UNIVERSE_S3_DIR={}\n'.format(universe_s3_dir))
            properties_file.flush()
            properties_file.close()
        custom_universes_path = os.environ.get('CUSTOM_UNIVERSES_PATH', '')
        if custom_universes_path:
            # write custom universes file for dcos-tests:
            custom_universes_file = open(custom_universes_path, 'w')
            properties_file.write('stub {}\n'.format(universe_url))
            properties_file.flush()
            properties_file.close()
        self.__github_updater.update(
            'success',
            'Uploaded {} artifacts and stub universe'.format(len(self.__artifact_paths)),
            universe_url)

    def upload(self):
        '''generates a unique directory, then uploads artifacts and a new stub universe to that directory'''
        try:
            universe_path = universe_builder.UniversePackageBuilder(
                self.__pkg_name, self.__pkg_version,
                self.__input_dir_path, self.__http_directory, self.__artifact_paths).build_zip()
        except Exception as e:
            traceback.format_exc()
            err = 'Failed to create stub universe: {}'.format(str(e))
            self.__github_updater.update('error', err)
            raise Exception(err)

        # print universe url early
        universe_url = self.__upload_artifact(universe_path)
        print('---')
        print('Built and uploaded stub universe:')
        print(universe_url)
        print('---')
        print('Uploading {} artifacts:'.format(len(self.__artifact_paths)))

        for path in self.__artifact_paths:
            self.__upload_artifact(path)

        self.__spam_universe_url(universe_url)

        return universe_url

def print_help(argv):
    print('Syntax: {} <package-name> <template-package-dir> [artifact files ...]'.format(argv[0]))
    print('  Example: $ {} kafka /path/to/template/jsons/ /path/to/artifact1.zip /path/to/artifact2.zip /path/to/artifact3.zip'.format(argv[0]))
    print('In addition, environment variables named \'TEMPLATE_SOME_PARAMETER\' will be inserted against the provided package template (with params of the form \'{{some-parameter}}\')')

def main(argv):
    if len(argv) < 3:
        print_help(argv)
        return 1
    # the package name:
    package_name = argv[1]
    # local path where the package template is located:
    package_dir_path = argv[2].rstrip('/')
    # artifact paths (to upload along with stub universe)
    artifact_paths = argv[3:]
    print('''###
Package:         {}
Template path:   {}
Artifacts:       {}
###'''.format(package_name, package_dir_path, ','.join(artifact_paths)))

    CIUploader(package_name, package_dir_path, artifact_paths).upload()
    return 0

if __name__ == '__main__':
    sys.exit(main(sys.argv))
<commit_msg>Fix properties file handling<commit_after>#!/usr/bin/python

import os
import os.path
import random
import string
import sys
import time
import traceback

import github_update
import universe_builder

class CIUploader(object):

    def __init__(self, package_name, input_dir_path, artifact_paths, package_version = 'stub-universe'):
        self.__pkg_name = package_name
        self.__pkg_version = package_version
        self.__input_dir_path = input_dir_path

        self.__aws_region = os.environ.get('AWS_UPLOAD_REGION', 'us-west-2')
        # sample s3_directory: 'infinity-artifacts/autodelete7d/kafka/20160815-134747-S6vxd0gRQBw43NNy'
        s3_base_path = os.environ.get('S3_BASE_PATH', 'infinity-artifacts/autodelete7d')
        dir_name = '{}-{}'.format(
            time.strftime("%Y%m%d-%H%M%S"),
            ''.join([random.SystemRandom().choice(string.ascii_letters + string.digits) for i in range(16)]))
        s3_path = '/'.join([s3_base_path, self.__pkg_name, dir_name])
        self.__s3_directory = 's3://{}'.format(s3_path)
        self.__http_directory = 'https://s3-{}.amazonaws.com/{}'.format(self.__aws_region, s3_path)

        self.__github_updater = github_update.GithubStatusUpdater('upload:{}'.format(package_name))

        if not os.path.isdir(input_dir_path):
            err = 'Provided package path is not a directory: {}'.format(input_dir_path)
            self.__github_updater.update('error', err)
            raise Exception(err)

        self.__artifact_paths = []
        for artifact_path in artifact_paths:
            if not os.path.isfile(artifact_path):
                err = 'Provided package path is not a file: {} (full list: {})'.format(artifact_path, artifact_paths)
                raise Exception(err)
            if artifact_path in self.__artifact_paths:
                err = 'Duplicate filename between "{}" and "{}". Artifact filenames must be unique.'.format(prior_path, artifact_path)
                self.__github_updater.update('error', err)
                raise Exception(err)
            self.__artifact_paths.append(artifact_path)

    def __upload_artifact(self, filepath):
        filename = os.path.basename(filepath)
        cmd = 'aws s3 --region={} cp --acl public-read {} {}/{}'.format(
            self.__aws_region, filepath, self.__s3_directory, filename)
        print(cmd)
        ret = os.system(cmd)
        if not ret == 0:
            err = 'Failed to upload {} to S3'.format(filename)
            self.__github_updater.update('error', err)
            raise Exception(err)
        return '{}/{}'.format(self.__http_directory, filename)

    def __spam_universe_url(self, universe_url):
        if os.environ.get('JENKINS_HOME', ''):
            # write jenkins properties file:
            properties_file = open(os.path.join(os.environ['WORKSPACE'], 'stub-universe.properties'), 'w')
            properties_file.write('STUB_UNIVERSE_URL={}\n'.format(universe_url))
            properties_file.write('STUB_UNIVERSE_S3_DIR={}\n'.format(self.__s3_directory))
            properties_file.flush()
            properties_file.close()
        custom_universes_path = os.environ.get('CUSTOM_UNIVERSES_PATH', '')
        if custom_universes_path:
            # write custom universes file for dcos-tests:
            custom_universes_file = open(custom_universes_path, 'w')
            properties_file.write('stub {}\n'.format(universe_url))
            properties_file.flush()
            properties_file.close()
        self.__github_updater.update(
            'success',
            'Uploaded {} artifacts and stub universe'.format(len(self.__artifact_paths)),
            universe_url)

    def upload(self):
        '''generates a unique directory, then uploads artifacts and a new stub universe to that directory'''
        try:
            universe_path = universe_builder.UniversePackageBuilder(
                self.__pkg_name, self.__pkg_version,
                self.__input_dir_path, self.__http_directory, self.__artifact_paths).build_zip()
        except Exception as e:
            traceback.format_exc()
            err = 'Failed to create stub universe: {}'.format(str(e))
            self.__github_updater.update('error', err)
            raise Exception(err)

        # print universe url early
        universe_url = self.__upload_artifact(universe_path)
        print('---')
        print('Built and uploaded stub universe:')
        print(universe_url)
        print('---')
        print('Uploading {} artifacts:'.format(len(self.__artifact_paths)))

        for path in self.__artifact_paths:
            self.__upload_artifact(path)

        self.__spam_universe_url(universe_url)

        return universe_url

def print_help(argv):
    print('Syntax: {} <package-name> <template-package-dir> [artifact files ...]'.format(argv[0]))
    print('  Example: $ {} kafka /path/to/template/jsons/ /path/to/artifact1.zip /path/to/artifact2.zip /path/to/artifact3.zip'.format(argv[0]))
    print('In addition, environment variables named \'TEMPLATE_SOME_PARAMETER\' will be inserted against the provided package template (with params of the form \'{{some-parameter}}\')')

def main(argv):
    if len(argv) < 3:
        print_help(argv)
        return 1
    # the package name:
    package_name = argv[1]
    # local path where the package template is located:
    package_dir_path = argv[2].rstrip('/')
    # artifact paths (to upload along with stub universe)
    artifact_paths = argv[3:]
    print('''###
Package:         {}
Template path:   {}
Artifacts:       {}
###'''.format(package_name, package_dir_path, ','.join(artifact_paths)))

    CIUploader(package_name, package_dir_path, artifact_paths).upload()
    return 0

if __name__ == '__main__':
    sys.exit(main(sys.argv))
<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- encoding: utf-8 -*-

"""
Dumps texts from typeracerdata.com.
"""

from HTMLParser import HTMLParser
from multiprocessing import Pool as ThreadPool
import argparse
import hashlib
import json
import os
import urllib2

def normalize(s):
    # Unescape html
    s = s.decode("utf-8")
    s = HTMLParser().unescape(s)
    s = s.replace("\n", " ")
    s = s.replace("\r", " ")

    while "  " in s:
        s = s.replace("  ", " ")

    return s.strip()

def loadurl(url):
    cache = os.path.join("cache", hashlib.sha1(url).hexdigest())
    if os.path.isfile(cache):
        with open(cache, "rt") as f:
            return f.read()

    html = urllib2.urlopen(url).read()

    with open(cache, "wt") as f:
        f.write(html)

    return html

def get_texts():
    return loadurl("http://www.typeracerdata.com/texts")

def get_text(text_id):
    url = "http://typeracerdata.com/text?id=%s" % text_id
    html = loadurl(url)
    return html

def process(text_id, html):
    """

Text #3550073

 Plato is always concerned to advocate views that will make people what he
 thinks virtuous; he is hardly ever intellectually honest, because he allows
 himself to judge doctrines by their social consequences.

- from The History of Western Philosophy, a book Bertrand
Russel

    """
    def skip(s, tag):
        pos = s.index(tag)
        assert(pos >= 0)
        return s[pos + len(tag):]

    def extract(s, tag):
        pos = s.index(tag)
        assert(pos >= 0)
        return s[:pos]

    html = skip(skip(html, "
"), "
")
    quote = extract(html, "
").strip()

    html = skip(html, "")
    title = extract(html, "").strip()
    html = skip(html, "")

    author = extract(html, "
").strip()
    while author[0] in ", ":
        author = author[1:].strip()

    prefixes = (
        "a",
        "book",
        "by",
        "directed",
        "movie",
        "other",
        "song",
        "speech",
        "television series",
    )
    for p in prefixes:
        if author.lower().startswith(p.lower()):
            author = author[len(p):].strip()

    author = normalize(author)
    title = normalize(title)
    quote = normalize(quote)

    return author, title, quote

def readquote(text_id):
        html = get_text(text_id)
        author, title, quote = process(text_id, html)

        print("id %s: '%s' by %s" % (text_id, title, author))
        print("   \"%s\"" % quote)

        return author, title, quote

def read_ids(url="http://www.typeracerdata.com/texts"):
    html = loadurl(url)

    ids = set()
    for line in html.split("\n"):
        find = "/text?id="
        if not find in line:
            continue
        start = line.index(find)
        stop = line.index('"', start + len(find))
        text_id = line[start + len(find):stop]
        ids.add(text_id)

    return ids

def main(verbose=False, threads=1):
    print("Reading list of quotes")
    ids = read_ids()

    print("Found %d quotes" % len(ids))
    print("Downloading each quote w/%d threads" % threads)

    if threads > 1:
        pool = ThreadPool(threads)
        results = pool.map(readquote, ids)
        pool.close()
        pool.join()
    else:
        try:
            results = []
            for count, id in enumerate(ids):
                print("%d/%d" % (count, len(ids)))
                results.append(readquote(id))
        except KeyboardInterrupt:
            pass

    print("Adding %d quotes to database" % len(results))

    quotes = []
    for author, title, quote in results:
        quotes.append({
            "author": author,
            "title": title,
            "text": quote})

    filename = "../wpm/data/examples.json"

    with open(filename, "rt") as f:
        examples = json.load(f)
        oldcount = len(examples)

    # Make quotes unique
    unique = set()
    for quote in examples + quotes:
        author = quote["author"]
        title = quote["title"]
        text = quote["text"]
        unique.add((author, title, text))

    examples = []
    for author, title, text in unique:
        examples.append({
            "author": author,
            "title": title,
            "text": text,
        })

    with open(filename, "wt") as f:
        json.dump(examples, f)

    print("Total quote count in database: %d" % len(examples))

if __name__ == "__main__":
    p = argparse.ArgumentParser()
    p.add_argument("-t", "--threads", default=1, type=int,
            help="Number of concurrent downloads")
    p.add_argument("-v", "--verbose", default=False,
            action="store_true")

    opts = p.parse_args()
    main(opts.verbose, opts.threads)
<commit_msg>Fixes prefix cleaner<commit_after>#! /usr/bin/env python
# -*- encoding: utf-8 -*-

"""
Dumps texts from typeracerdata.com.
"""

from HTMLParser import HTMLParser
from multiprocessing import Pool as ThreadPool
import argparse
import hashlib
import json
import os
import urllib2

def normalize(s):
    # Unescape html
    s = s.decode("utf-8")
    s = HTMLParser().unescape(s)
    s = s.replace("\n", " ")
    s = s.replace("\r", " ")

    while "  " in s:
        s = s.replace("  ", " ")

    return s.strip()

def loadurl(url):
    cache = os.path.join("cache", hashlib.sha1(url).hexdigest())
    if os.path.isfile(cache):
        with open(cache, "rt") as f:
            return f.read()

    html = urllib2.urlopen(url).read()

    with open(cache, "wt") as f:
        f.write(html)

    return html

def get_texts():
    return loadurl("http://www.typeracerdata.com/texts")

def get_text(text_id):
    url = "http://typeracerdata.com/text?id=%s" % text_id
    html = loadurl(url)
    return html

def process(text_id, html):
    """

Text #3550073

 Plato is always concerned to advocate views that will make people what he
 thinks virtuous; he is hardly ever intellectually honest, because he allows
 himself to judge doctrines by their social consequences.

- from The History of Western Philosophy, a book Bertrand
Russel

    """
    def skip(s, tag):
        pos = s.index(tag)
        assert(pos >= 0)
        return s[pos + len(tag):]

    def extract(s, tag):
        pos = s.index(tag)
        assert(pos >= 0)
        return s[:pos]

    html = skip(skip(html, "
"), "
")
    quote = extract(html, "
").strip()

    html = skip(html, "")
    title = extract(html, "").strip()
    html = skip(html, "")

    author = extract(html, "
").strip()
    while author[0] in ", ":
        author = author[1:].strip()

    prefixes = (
        "a",
        "book",
        "by",
        "directed",
        "movie",
        "other",
        "song",
        "speech",
        "television series",
    )
    while author.lower().split(" ")[0] in prefixes:
        author = " ".join(author.split(" ")[1:])

    author = normalize(author)
    title = normalize(title)
    quote = normalize(quote)

    return author, title, quote

def readquote(text_id):
        html = get_text(text_id)
        author, title, quote = process(text_id, html)

        print("id %s: '%s' by %s" % (text_id, title, author))
        print("   \"%s\"" % quote)

        return author, title, quote

def read_ids(url="http://www.typeracerdata.com/texts"):
    html = loadurl(url)

    ids = set()
    for line in html.split("\n"):
        find = "/text?id="
        if not find in line:
            continue
        start = line.index(find)
        stop = line.index('"', start + len(find))
        text_id = line[start + len(find):stop]
        ids.add(text_id)

    return ids

def main(verbose=False, threads=1):
    print("Reading list of quotes")
    ids = read_ids()

    print("Found %d quotes" % len(ids))
    print("Downloading each quote w/%d threads" % threads)

    if threads > 1:
        pool = ThreadPool(threads)
        results = pool.map(readquote, ids)
        pool.close()
        pool.join()
    else:
        try:
            results = []
            for count, id in enumerate(ids):
                print("%d/%d" % (count, len(ids)))
                results.append(readquote(id))
        except KeyboardInterrupt:
            pass

    print("Adding %d quotes to database" % len(results))

    quotes = []
    for author, title, quote in results:
        quotes.append({
            "author": author,
            "title": title,
            "text": quote})

    filename = "../wpm/data/examples.json"

    with open(filename, "rt") as f:
        examples = json.load(f)
        oldcount = len(examples)

    # Make quotes unique
    unique = set()
    for quote in examples + quotes:
        author = quote["author"]
        title = quote["title"]
        text = quote["text"]
        unique.add((author, title, text))

    examples = []
    for author, title, text in unique:
        examples.append({
            "author": author,
            "title": title,
            "text": text,
        })

    with open(filename, "wt") as f:
        json.dump(examples, f)

    print("Total quote count in database: %d" % len(examples))

if __name__ == "__main__":
    p = argparse.ArgumentParser()
    p.add_argument("-t", "--threads", default=1, type=int,
            help="Number of concurrent downloads")
    p.add_argument("-v", "--verbose", default=False,
            action="store_true")

    opts = p.parse_args()
    main(opts.verbose, opts.threads)
<|endoftext|>


<commit_before>#!/usr/bin/env python2

"""Find most interesting ROI's in a DWI image."""

import argparse
import glob
import numpy as np

import dwi.autoroi
import dwi.dicomfile
import dwi.mask
import dwi.patient
import dwi.util

PARAMS = ['ADCm']

IN_SUBREGION_DIR = 'bounding_box_100_10pad'
IN_MASK_DIR = 'dicom_masks'
IN_IMAGE_DIR = 'results_Mono_combinedDICOM'
IN_SAMPLELIST_FILE = 'samples_all.txt'
IN_SUBWINDOWS_FILE = 'subwindows.txt'
IN_PATIENTS_FILE = 'patients.txt'

OUT_MASK_DIR = 'masks_auto2'
OUT_IMAGE_DIR = 'find_roi2_images'

def parse_args():
    """Parse command-line arguments."""
    p = argparse.ArgumentParser(description = __doc__)
    p.add_argument('--verbose', '-v',
            action='count',
            help='increase verbosity')
    p.add_argument('--roidim', metavar='I', nargs=3, type=int, default=[1,5,5],
            help='dimensions of wanted ROI (3 integers; default 1 5 5)')
    p.add_argument('--cases', metavar='I', nargs='*', type=int, default=[],
            help='case numbers')
    p.add_argument('--scan',
            help='scan id')
    p.add_argument('--outmask',
            help='output mask file')
    p.add_argument('--outpic',
            help='output graphic file')
    args = p.parse_args()
    return args

def read_subregion(case, scan):
    """Read subregion definition."""
    d = dict(c=case, s=scan)
    s = IN_SUBREGION_DIR + '/{c}_*_{s}_*.txt'.format(**d)
    paths = glob.glob(s)
    if len(paths) != 1:
        raise Exception('Subregion file confusion: %s' % s)
    subregion = dwi.util.read_subregion_file(paths[0])
    return subregion

def read_dicom_masks(case, scan):
    """Read cancer and normal masks in DICOM format.

    Cancer mask path ends with "_ca", normal with "_n". Unless these names
    exist, first two are used. Some cases have a third mask which has cancer,
    they are ignored for now.
    """
    d = dict(c=case, s=scan)
    s = IN_MASK_DIR + '/{c}_*_{s}_D_*'.format(**d)
    cancer_path, normal_path, other_paths = None, None, []
    paths = sorted(glob.glob(s), key=str.lower)
    if len(paths) < 2:
        raise Exception('Not all masks were not found: %s' % s)
    for path in paths:
        if not cancer_path and path.lower().endswith('ca'):
            cancer_path = path
        elif not normal_path and path.lower().endswith('n'):
            normal_path = path
        else:
            other_paths.append(path)
    if not cancer_path:
        cancer_path = other_paths.pop(0)
    if not normal_path:
        normal_path = other_paths.pop(0)
    masks = map(dwi.mask.read_dicom_mask, [cancer_path, normal_path])
    return tuple(masks)

def read_prostate_mask(case, scan):
    """Read 3D prostate mask in DICOM format.

    The first multi-slice mask with proper pathname is used.
    """
    d = dict(c=case, s=scan)
    IN_PROSTATE_MASK_DIR = 'masks_prostate'
    s = IN_PROSTATE_MASK_DIR + '/{c}_*_{s}_*'.format(**d)
    paths = sorted(glob.glob(s))
    for path in paths:
        mask = dwi.mask.read_mask(path)
        if len(mask.selected_slices()) > 1:
            return mask
    raise Exception('Multi-slice prostate mask not found: %s' % s)

def read_image(case, scan, param):
    d = dict(c=case, s=scan, p=param)
    s = IN_IMAGE_DIR + '/{c}_*_{s}/{c}_*_{s}_{p}'.format(**d)
    paths = glob.glob(s)
    if len(paths) != 1:
        raise Exception('Image path confusion: %s' % s)
    d = dwi.dicomfile.read_dir(paths[0])
    image = d['image']
    #image = image.squeeze(axis=3) # Remove single subvalue dimension.
    return image

def clip_outliers(img):
    """Clip parameter-specific intensity outliers in-place."""
    for i in range(img.shape[-1]):
        if PARAMS[i].startswith('ADC'):
            img[...,i].clip(0, 0.002, out=img[...,i])
        elif PARAMS[i].startswith('K'):
            img[...,i].clip(0, 2, out=img[...,i])

def read_data(cases):
    samples = dwi.util.read_sample_list(IN_SAMPLELIST_FILE)
    subwindows = dwi.util.read_subwindows(IN_SUBWINDOWS_FILE)
    patientsinfo = dwi.patient.read_patients_file(IN_PATIENTS_FILE)
    data = []
    for sample in samples:
        case = sample['case']
        if cases and not case in cases:
            continue
        score = dwi.patient.get_patient(patientsinfo, case).score
        for scan in sample['scans']:
            subwindow = subwindows[(case, scan)]
            slice_index = subwindow[0] # Make it zero-based.
            subregion = read_subregion(case, scan)
            cancer_mask, normal_mask = read_dicom_masks(case, scan)
            prostate_mask = read_prostate_mask(case, scan)
            image = read_image(case, scan, PARAMS[0])
            cropped_cancer_mask = cancer_mask.crop(subregion)
            cropped_normal_mask = normal_mask.crop(subregion)
            cropped_prostate_mask = prostate_mask.crop(subregion)
            cropped_image = dwi.util.crop_image(image, subregion).copy()
            #cropped_image = cropped_image[[slice_index],...] # TODO: one slice
            clip_outliers(cropped_image)
            d = dict(case=case, scan=scan, score=score,
                    subwindow=subwindow,
                    slice_index=slice_index,
                    subregion=subregion,
                    cancer_mask=cropped_cancer_mask,
                    normal_mask=cropped_normal_mask,
                    prostate_mask=cropped_prostate_mask,
                    original_shape=image.shape,
                    image=cropped_image)
            data.append(d)
    return data

###

def draw_roi(img, pos, color):
    """Draw a rectangle ROI on a layer."""
    y, x = pos
    img[y:y+4:4, x] = color
    img[y:y+4:4, x+4] = color
    img[y, x:x+4:4] = color
    img[y+4, x:x+5:4] = color

def get_roi_layer(img, pos, color):
    """Get a layer with a rectangle ROI for drawing."""
    layer = np.zeros(img.shape + (4,))
    draw_roi(layer, pos, color)
    return layer

def draw(data):
    import matplotlib
    import matplotlib.pyplot as plt
    import pylab as pl

    plt.rcParams['image.cmap'] = 'gray'
    plt.rcParams['image.interpolation'] = 'nearest'
    n_cols, n_rows = 3, 1
    fig = plt.figure(figsize=(n_cols*6, n_rows*6))

    CANCER_COLOR = (1.0, 0.0, 0.0, 1.0)
    NORMAL_COLOR = (0.0, 1.0, 0.0, 1.0)
    AUTO_COLOR = (1.0, 1.0, 0.0, 1.0)

    slice_index = data['roi_corner'][0]
    cancer_pos = (-1, -1)
    normal_pos = (-1, -1)
    distance = -1
    auto_pos = (data['roi_coords'][1][0], data['roi_coords'][2][0])
    if 'cancer_mask' in data:
        cancer_pos = data['cancer_mask'].where()[0][1:3]
        distance = dwi.util.distance(cancer_pos, auto_pos)
    if 'normal_mask' in data:
        normal_pos = data['normal_mask'].where()[0][1:3]

    ax1 = fig.add_subplot(1, n_cols, 1)
    ax1.set_title('Slice %i %s' % (slice_index, PARAMS[0]))
    iview = data['image'][slice_index,:,:,0]
    plt.imshow(iview)

    ax2 = fig.add_subplot(1, n_cols, 2)
    ax2.set_title('Calculated score map')
    iview = data['image'][slice_index,...,0]
    pview = data['sum_scoremaps'][slice_index,...,0]
    pview /= pview.max()
    imgray = plt.imshow(iview, alpha=1)
    imjet = plt.imshow(pview, alpha=0.8, cmap='jet')

    ax3 = fig.add_subplot(1, n_cols, 3)
    ax3.set_title('ROIs: %s, %s, distance: %.2f' % (cancer_pos, auto_pos,
            distance))
    iview = data['image'][slice_index,:,:,0]
    #plt.imshow(iview)
    view = np.zeros(iview.shape + (3,), dtype=float)
    view[...,0] = iview / iview.max()
    view[...,1] = iview / iview.max()
    view[...,2] = iview / iview.max()
    for i, a in enumerate(iview):
        for j, v in enumerate(a):
            if v < dwi.autoroi.ADCM_MIN:
                view[i,j,:] = [0.5, 0, 0]
            elif v > dwi.autoroi.ADCM_MAX:
                view[i,j,:] = [0, 0.5, 0]
    plt.imshow(view)
    if 'cancer_mask' in data:
        plt.imshow(get_roi_layer(iview, cancer_pos, CANCER_COLOR), alpha=0.8)
    if 'normal_mask' in data:
        plt.imshow(get_roi_layer(iview, normal_pos, NORMAL_COLOR), alpha=0.8)
    plt.imshow(get_roi_layer(iview, auto_pos, AUTO_COLOR), alpha=0.8)

    fig.colorbar(imgray, ax=ax1, shrink=0.65)
    fig.colorbar(imjet, ax=ax2, shrink=0.65)
    fig.colorbar(imgray, ax=ax3, shrink=0.65)

    plt.tight_layout()
    filename = OUT_IMAGE_DIR + '/{case}_{scan}.png'.format(**data)
    plt.savefig(filename, bbox_inches='tight')
    fig.clf()

def write_mask(d):
    """Write mask. XXX: Here only single-slice ones."""
    slice_index = d['roi_corner'][0]
    a = np.zeros((d['original_shape'][1:3]), dtype=int)
    _, y, x = d['roi_coords']
    y_offset, x_offset = d['subregion'][2], d['subregion'][4]
    y = (y[0]+y_offset, y[1]+y_offset)
    x = (x[0]+x_offset, x[1]+x_offset)
    a[y[0]:y[1], x[0]:x[1]] = 1
    mask = dwi.mask.Mask(slice_index+1, a)
    filename = OUT_MASK_DIR + '/{case}_{scan}_auto2.mask'.format(**d)
    mask.write(filename)

args = parse_args()

print 'Reading data...'
data = read_data(args.cases)

for d in data:
    print
    print d['case'], d['scan'], d['score'], d['subwindow'], d['subregion']
    print d['image'].shape, d['cancer_mask'], d['normal_mask'], d['prostate_mask']
    d.update(dwi.autoroi.find_roi(d['image'], args.roidim, PARAMS))
    print 'Optimal ROI: {} at {}'.format(d['roi_coords'], d['roi_corner'])
    draw(d)
    write_mask(d)

#if args.verbose:
#    for i, p in enumerate(params):
#        z, y, x = coords
#        a = img[z[0]:z[1], y[0]:y[1], x[0]:x[1], i]
#        print p, a.min(), a.max(), np.median(a)
#        print dwi.util.fivenum(a.flatten())
#        a = img[..., i]
#        print p, a.min(), a.max(), np.median(a)
#        print dwi.util.fivenum(a.flatten())
<commit_msg>Assert for proper mask shapes<commit_after>#!/usr/bin/env python2

"""Find most interesting ROI's in a DWI image."""

import argparse
import glob
import numpy as np

import dwi.autoroi
import dwi.dicomfile
import dwi.mask
import dwi.patient
import dwi.util

PARAMS = ['ADCm']

IN_SUBREGION_DIR = 'bounding_box_100_10pad'
IN_MASK_DIR = 'dicom_masks'
IN_IMAGE_DIR = 'results_Mono_combinedDICOM'
IN_SAMPLELIST_FILE = 'samples_all.txt'
IN_SUBWINDOWS_FILE = 'subwindows.txt'
IN_PATIENTS_FILE = 'patients.txt'

OUT_MASK_DIR = 'masks_auto2'
OUT_IMAGE_DIR = 'find_roi2_images'

def parse_args():
    """Parse command-line arguments."""
    p = argparse.ArgumentParser(description = __doc__)
    p.add_argument('--verbose', '-v',
            action='count',
            help='increase verbosity')
    p.add_argument('--roidim', metavar='I', nargs=3, type=int, default=[1,5,5],
            help='dimensions of wanted ROI (3 integers; default 1 5 5)')
    p.add_argument('--cases', metavar='I', nargs='*', type=int, default=[],
            help='case numbers')
    p.add_argument('--scan',
            help='scan id')
    p.add_argument('--outmask',
            help='output mask file')
    p.add_argument('--outpic',
            help='output graphic file')
    args = p.parse_args()
    return args

def read_subregion(case, scan):
    """Read subregion definition."""
    d = dict(c=case, s=scan)
    s = IN_SUBREGION_DIR + '/{c}_*_{s}_*.txt'.format(**d)
    paths = glob.glob(s)
    if len(paths) != 1:
        raise Exception('Subregion file confusion: %s' % s)
    subregion = dwi.util.read_subregion_file(paths[0])
    return subregion

def read_dicom_masks(case, scan):
    """Read cancer and normal masks in DICOM format.

    Cancer mask path ends with "_ca", normal with "_n". Unless these names
    exist, first two are used. Some cases have a third mask which has cancer,
    they are ignored for now.
    """
    d = dict(c=case, s=scan)
    s = IN_MASK_DIR + '/{c}_*_{s}_D_*'.format(**d)
    cancer_path, normal_path, other_paths = None, None, []
    paths = sorted(glob.glob(s), key=str.lower)
    if len(paths) < 2:
        raise Exception('Not all masks were not found: %s' % s)
    for path in paths:
        if not cancer_path and path.lower().endswith('ca'):
            cancer_path = path
        elif not normal_path and path.lower().endswith('n'):
            normal_path = path
        else:
            other_paths.append(path)
    if not cancer_path:
        cancer_path = other_paths.pop(0)
    if not normal_path:
        normal_path = other_paths.pop(0)
    masks = map(dwi.mask.read_dicom_mask, [cancer_path, normal_path])
    return tuple(masks)

def read_prostate_mask(case, scan):
    """Read 3D prostate mask in DICOM format.

    The first multi-slice mask with proper pathname is used.
    """
    d = dict(c=case, s=scan)
    IN_PROSTATE_MASK_DIR = 'masks_prostate'
    s = IN_PROSTATE_MASK_DIR + '/{c}_*_{s}_*'.format(**d)
    paths = sorted(glob.glob(s))
    for path in paths:
        mask = dwi.mask.read_mask(path)
        if len(mask.selected_slices()) > 1:
            return mask
    raise Exception('Multi-slice prostate mask not found: %s' % s)

def read_image(case, scan, param):
    d = dict(c=case, s=scan, p=param)
    s = IN_IMAGE_DIR + '/{c}_*_{s}/{c}_*_{s}_{p}'.format(**d)
    paths = glob.glob(s)
    if len(paths) != 1:
        raise Exception('Image path confusion: %s' % s)
    d = dwi.dicomfile.read_dir(paths[0])
    image = d['image']
    #image = image.squeeze(axis=3) # Remove single subvalue dimension.
    return image

def clip_outliers(img):
    """Clip parameter-specific intensity outliers in-place."""
    for i in range(img.shape[-1]):
        if PARAMS[i].startswith('ADC'):
            img[...,i].clip(0, 0.002, out=img[...,i])
        elif PARAMS[i].startswith('K'):
            img[...,i].clip(0, 2, out=img[...,i])

def read_data(cases):
    samples = dwi.util.read_sample_list(IN_SAMPLELIST_FILE)
    subwindows = dwi.util.read_subwindows(IN_SUBWINDOWS_FILE)
    patientsinfo = dwi.patient.read_patients_file(IN_PATIENTS_FILE)
    data = []
    for sample in samples:
        case = sample['case']
        if cases and not case in cases:
            continue
        score = dwi.patient.get_patient(patientsinfo, case).score
        for scan in sample['scans']:
            subwindow = subwindows[(case, scan)]
            slice_index = subwindow[0] # Make it zero-based.
            subregion = read_subregion(case, scan)
            cancer_mask, normal_mask = read_dicom_masks(case, scan)
            prostate_mask = read_prostate_mask(case, scan)
            image = read_image(case, scan, PARAMS[0])
            cropped_cancer_mask = cancer_mask.crop(subregion)
            cropped_normal_mask = normal_mask.crop(subregion)
            cropped_prostate_mask = prostate_mask.crop(subregion)
            cropped_image = dwi.util.crop_image(image, subregion).copy()
            #cropped_image = cropped_image[[slice_index],...] # TODO: one slice
            clip_outliers(cropped_image)
            d = dict(case=case, scan=scan, score=score,
                    subwindow=subwindow,
                    slice_index=slice_index,
                    subregion=subregion,
                    cancer_mask=cropped_cancer_mask,
                    normal_mask=cropped_normal_mask,
                    prostate_mask=cropped_prostate_mask,
                    original_shape=image.shape,
                    image=cropped_image)
            data.append(d)
            assert d['cancer_mask'].array.shape ==\
                    d['normal_mask'].array.shape ==\
                    d['prostate_mask'].array.shape ==\
                    d['image'].shape[0:3]
    return data

###

def draw_roi(img, pos, color):
    """Draw a rectangle ROI on a layer."""
    y, x = pos
    img[y:y+4:4, x] = color
    img[y:y+4:4, x+4] = color
    img[y, x:x+4:4] = color
    img[y+4, x:x+5:4] = color

def get_roi_layer(img, pos, color):
    """Get a layer with a rectangle ROI for drawing."""
    layer = np.zeros(img.shape + (4,))
    draw_roi(layer, pos, color)
    return layer

def draw(data):
    import matplotlib
    import matplotlib.pyplot as plt
    import pylab as pl

    plt.rcParams['image.cmap'] = 'gray'
    plt.rcParams['image.interpolation'] = 'nearest'
    n_cols, n_rows = 3, 1
    fig = plt.figure(figsize=(n_cols*6, n_rows*6))

    CANCER_COLOR = (1.0, 0.0, 0.0, 1.0)
    NORMAL_COLOR = (0.0, 1.0, 0.0, 1.0)
    AUTO_COLOR = (1.0, 1.0, 0.0, 1.0)

    slice_index = data['roi_corner'][0]
    cancer_pos = (-1, -1)
    normal_pos = (-1, -1)
    distance = -1
    auto_pos = (data['roi_coords'][1][0], data['roi_coords'][2][0])
    if 'cancer_mask' in data:
        cancer_pos = data['cancer_mask'].where()[0][1:3]
        distance = dwi.util.distance(cancer_pos, auto_pos)
    if 'normal_mask' in data:
        normal_pos = data['normal_mask'].where()[0][1:3]

    ax1 = fig.add_subplot(1, n_cols, 1)
    ax1.set_title('Slice %i %s' % (slice_index, PARAMS[0]))
    iview = data['image'][slice_index,:,:,0]
    plt.imshow(iview)

    ax2 = fig.add_subplot(1, n_cols, 2)
    ax2.set_title('Calculated score map')
    iview = data['image'][slice_index,...,0]
    pview = data['sum_scoremaps'][slice_index,...,0]
    pview /= pview.max()
    imgray = plt.imshow(iview, alpha=1)
    imjet = plt.imshow(pview, alpha=0.8, cmap='jet')

    ax3 = fig.add_subplot(1, n_cols, 3)
    ax3.set_title('ROIs: %s, %s, distance: %.2f' % (cancer_pos, auto_pos,
            distance))
    iview = data['image'][slice_index,:,:,0]
    #plt.imshow(iview)
    view = np.zeros(iview.shape + (3,), dtype=float)
    view[...,0] = iview / iview.max()
    view[...,1] = iview / iview.max()
    view[...,2] = iview / iview.max()
    for i, a in enumerate(iview):
        for j, v in enumerate(a):
            if v < dwi.autoroi.ADCM_MIN:
                view[i,j,:] = [0.5, 0, 0]
            elif v > dwi.autoroi.ADCM_MAX:
                view[i,j,:] = [0, 0.5, 0]
    plt.imshow(view)
    if 'cancer_mask' in data:
        plt.imshow(get_roi_layer(iview, cancer_pos, CANCER_COLOR), alpha=0.8)
    if 'normal_mask' in data:
        plt.imshow(get_roi_layer(iview, normal_pos, NORMAL_COLOR), alpha=0.8)
    plt.imshow(get_roi_layer(iview, auto_pos, AUTO_COLOR), alpha=0.8)

    fig.colorbar(imgray, ax=ax1, shrink=0.65)
    fig.colorbar(imjet, ax=ax2, shrink=0.65)
    fig.colorbar(imgray, ax=ax3, shrink=0.65)

    plt.tight_layout()
    filename = OUT_IMAGE_DIR + '/{case}_{scan}.png'.format(**data)
    plt.savefig(filename, bbox_inches='tight')
    fig.clf()

def write_mask(d):
    """Write mask. XXX: Here only single-slice ones."""
    slice_index = d['roi_corner'][0]
    a = np.zeros((d['original_shape'][1:3]), dtype=int)
    _, y, x = d['roi_coords']
    y_offset, x_offset = d['subregion'][2], d['subregion'][4]
    y = (y[0]+y_offset, y[1]+y_offset)
    x = (x[0]+x_offset, x[1]+x_offset)
    a[y[0]:y[1], x[0]:x[1]] = 1
    mask = dwi.mask.Mask(slice_index+1, a)
    filename = OUT_MASK_DIR + '/{case}_{scan}_auto2.mask'.format(**d)
    mask.write(filename)

args = parse_args()

print 'Reading data...'
data = read_data(args.cases)

for d in data:
    print
    print d['case'], d['scan'], d['score'], d['subwindow'], d['subregion']
    print d['image'].shape, d['cancer_mask'], d['normal_mask'], d['prostate_mask']
    d.update(dwi.autoroi.find_roi(d['image'], args.roidim, PARAMS))
    print 'Optimal ROI: {} at {}'.format(d['roi_coords'], d['roi_corner'])
    draw(d)
    write_mask(d)

#if args.verbose:
#    for i, p in enumerate(params):
#        z, y, x = coords
#        a = img[z[0]:z[1], y[0]:y[1], x[0]:x[1], i]
#        print p, a.min(), a.max(), np.median(a)
#        print dwi.util.fivenum(a.flatten())
#        a = img[..., i]
#        print p, a.min(), a.max(), np.median(a)
#        print dwi.util.fivenum(a.flatten())
<|endoftext|>