<commit_before>#!/usr/bin/python
""" Classes and functions for fitting ivim model """
from __future__ import division, print_function, absolute_import
import warnings
import functools
import numpy as np
from dipy.core.optimize import Optimizer

from scipy.optimize import leastsq

from dipy.core.gradients import gradient_table
from dipy.reconst.base import ReconstModel
from dipy.reconst.dti import _min_positive_signal, apparent_diffusion_coef
from dipy.reconst.dti import TensorModel, mean_diffusivity
from .vec_val_sum import vec_val_vect
from dipy.core.sphere import Sphere

def ivim_function(params, bvals):
    """The ivim function
    Parameters
    ----------
    params : array
             The IVIM parameters [S0, f, D_star, D]

    bvals  : array
             bvalues
    """
    S0, f, D_star, D = params
    S = S0 * (f * np.exp(-bvals * D_star) + (1 - f) * np.exp(-bvals * D))
    return S

def _ivim_error(params, bvals, signal):
    """Error function to be used in fitting the model
    """
    return (signal - ivim_function(params, bvals))

class IvimModel(ReconstModel):
    """Ivim model
    """

    def __init__(self, gtab, split_b=200.0, min_signal=None):
        """
        An IVIM model

        Parameters
        ----------
        gtab : GradientTable class instance
        split_b : split the data at this b-value for a two stage fit

        References
        ----------
        .. [1] Le Bihan, Denis, et al. "Separation of diffusion
                   and perfusion in intravoxel incoherent motion MR
                   imaging." Radiology 168.2 (1988): 497-505.
        .. [2] Federau, Christian, et al. "Quantitative measurement
                   of brain perfusion with intravoxel incoherent motion
                   MR imaging." Radiology 265.3 (2012): 874-881.
        """
        ReconstModel.__init__(self, gtab)
        self.split_b = split_b
        self.min_signal = min_signal
        if self.min_signal is not None and self.min_signal <= 0:
            e_s = "The `min_signal` key-word argument needs to be strictly"
            e_s += " positive."
            raise ValueError(e_s)

    def fit(self, data, mask=None, x0=None, fit_method="one_stage", routine="minimize",
            jac=False, bounds=((0, 1.), (0, 1.), (0, 1.), (0, 1.)), tol=1e-25, algorithm='L-BFGS-B',
            gtol=1e-25, ftol=1e-25, eps=1e-15):
        """ Fit method of the Ivim model class

        Parameters
        ----------
        data : array
            The measured signal from one voxel.
        mask : array
            A boolean array used to mark the coordinates in the data that
            should be analyzed that has the shape data.shape[:-1]
        x0 : array
            Initial guesses for the parameters S0, f, D_star and D
            Default x0 = [1.0, 0.10, 0.001, 0.0009]
            These guess parameters are taken from the Federau paper
            Dimension can either be 1 or (N, 4) where N is the number of
            voxels in the data.
        fit_method: str
            Use one-stage fitting or two-stage fitting.
        jac : Boolean
            Use the Jacobian
        bounds : tuple
            Bounds for the various parameters
        tol : float
            tolerance for convergence
        method : str
            Fitting algorithm to use from scipy.optimize

        Returns
        -------
        IvimFit object
        """
        if mask is None:
            # Flatten it to 2D either way:
            data_in_mask = np.reshape(data, (-1, data.shape[-1]))
        else:
            # Check for valid shape of the mask
            if mask.shape != data.shape[:-1]:
                raise ValueError("Mask is not the same shape as data.")
            mask = np.array(mask, dtype=bool, copy=False)
            data_in_mask = np.reshape(data[mask], (-1, data.shape[-1]))

        if self.min_signal is None:
            min_signal = _min_positive_signal(data)
        else:
            min_signal = self.min_signal

        # Generate guess_parameters for all voxels
        if x0 is None:
            x0 = np.ones(
                (data.shape[:-1] + (4,))) * [1.0, 0.10, 0.001, 0.0009]
        else:
            if x0.shape != (data.shape[0], 4):
                raise ValueError(
                    "Guess params should be of shape (num_voxels,4)")
        data_in_mask = np.maximum(data_in_mask, min_signal)

        if fit_method == "one_stage":
            params_in_mask = one_stage(data_in_mask, self.gtab, x0,
                                       jac, bounds, tol,
                                       routine,
                                       algorithm, gtol, ftol, eps)
        elif fit_method == "two_stage":
            params_in_mask = two_stage(data_in_mask, self.gtab, x0,
                                       self.split_b, jac, bounds, tol,
                                       routine,
                                       algorithm, gtol, ftol, eps)
        else:
            raise ValueError("""Fit method must be either
                             'one_stage' or 'two_stage'""")

        if mask is None:
            out_shape = data.shape[:-1] + (-1, )
            ivim_params = params_in_mask.reshape(out_shape)
        else:
            ivim_params = np.zeros(data.shape[:-1] + (4,))
            ivim_params[mask, :] = params_in_mask

        return IvimFit(self, ivim_params)

    def predict(self, ivim_params):
        """
        Predict a signal for this IvimModel class instance given parameters.

        Parameters
        ----------
        ivim_params : ndarray
            The last dimension should have 4 parameters: S0, f, D_star and D
        """

        return ivim_function(ivim_params, self.gtab.bvals)

class IvimFit(object):

    def __init__(self, model, model_params):
        """ Initialize a IvimFit class instance.
            Parameters
            ----------
            The model parameters are S0, f, D_star, D
        """
        self.model = model
        self.model_params = model_params

    def __getitem__(self, index):
        model_params = self.model_params
        N = model_params.ndim
        if type(index) is not tuple:
            index = (index,)
        elif len(index) >= model_params.ndim:
            raise IndexError("IndexError: invalid index")
        index = index + (slice(None),) * (N - len(index))
        return type(self)(self.model, model_params[index])

    @property
    def shape(self):
        return self.model_params.shape[:-1]

    def predict(self, gtab, step=None, S0=1.0):
        r"""
        Given a model fit, predict the signal.

        Parameters
        ----------
        gtab : a GradientTable class instance
            This encodes the directions for which a prediction is made

        Notes
        -----
        The predicted signal is given by:

        .. math ::

        Where:
        .. math ::

        """
        return ivim_function(self.model_params, gtab.bvals)

def one_stage(data, gtab, x0, jac, bounds, tol, routine, algorithm,
              gtol, ftol, eps):
    """
    Fit the ivim params using minimize

    Parameters
    ----------
    data : array ([X, Y, Z, ...], g)
        Data or response variables holding the data. Note that the last
        dimension should contain the data. It makes no copies of data.

    jac : bool
        Use the Jacobian? Default: False

    Returns
    -------
    ivim_params: the S0, f, D_star, D value for each voxel.

    """
    flat_data = data.reshape((-1, data.shape[-1]))
    flat_x0 = x0.reshape(-1, x0.shape[-1])
    # Flatten for the iteration over voxels:
    bvals = gtab.bvals
    ivim_params = np.empty((flat_data.shape[0], 4))

    if routine == 'minimize':
        result = _minimize(flat_data, bvals, flat_x0, ivim_params,
                           bounds, tol, jac, algorithm, gtol, ftol, eps)
    elif routine == "leastsq":
        result = _leastsq(flat_data, bvals, flat_x0, ivim_params)
    ivim_params.shape = data.shape[:-1] + (4,)
    return ivim_params

def _minimize(flat_data, bvals, flat_x0, ivim_params,
              bounds, tol, jac, algorithm, gtol, ftol, eps):
    """Use minimize for finding ivim_params"""
    sum_sq = lambda params, bvals, signal: np.sum(
        _ivim_error(params, bvals, signal)**2)

    result = []
    for vox in range(flat_data.shape[0]):
        res = Optimizer(sum_sq,
                        flat_x0[vox],
                        args=(bvals, flat_data[vox]), bounds=bounds,
                        tol=tol, method=algorithm, jac=jac,
                        options={'gtol': gtol, 'ftol': ftol, 'eps': eps})
        ivim_params[vox, :4] = res.xopt
        result += [res]
    return result

def _leastsq(flat_data, bvals, flat_x0, ivim_params):
    """Use minimize for finding ivim_params"""
    result = []
    for vox in range(flat_data.shape[0]):
        res = leastsq(_ivim_error,
                      flat_x0[vox],
                      args=(bvals, flat_data[vox]))
        ivim_params[vox, :4] = res[0]
        result += [res]
    return result

def two_stage(data, gtab, x0,
              split_b, jac, bounds, tol,
              routine, algorithm, gtol, ftol, eps):
    """
    Fit the ivim params using a two stage fit

    Parameters
    ----------
    data : array ([X, Y, Z, ...], g)
        Data or response variables holding the data. Note that the last
        dimension should contain the data. It makes no copies of data.

    jac : bool
        Use the Jacobian? Default: False

    Returns
    -------
    ivim_params: the S0, f, D_star, D value for each voxel.

    """
    flat_data = data.reshape((-1, data.shape[-1]))
    # Flatten for the iteration over voxels:
    bvals_ge_split = gtab.bvals[gtab.bvals > split_b]
    bvecs_ge_split = gtab.bvecs[gtab.bvals > split_b]
    gtab_ge_split = gradient_table(bvals_ge_split, bvecs_ge_split.T)

    tensor_model = TensorModel(gtab_ge_split)
    tenfit = tensor_model.fit(data[..., gtab.bvals > split_b])

    D_guess = mean_diffusivity(tenfit.evals)

    x0[..., 3] = D_guess

    dti_params = tenfit.model_params

    S0_hat = get_S0_guess(dti_params, data, gtab_ge_split)
    f_guess = 1.0 - S0_hat / data[..., 0]
    x0[..., 1] = f_guess
    return one_stage(data, gtab, x0, jac, bounds, tol, routine, algorithm,
                     gtol, ftol, eps)

def get_S0_guess(dti_params, data, gtab):
    evecs = dti_params[..., 3:12].reshape((dti_params.shape[:-1] + (3, 3)))
    evals = dti_params[..., :3]
    qform = vec_val_vect(evecs, evals)

    sphere = Sphere(xyz=gtab.bvecs[~gtab.b0s_mask])
    ADC = apparent_diffusion_coef(qform, sphere)

    S0_hat = np.mean(data[..., ~gtab.b0s_mask] /
                     np.exp(-gtab.bvals[~gtab.b0s_mask] * ADC),
                     -1)
    return S0_hat
<commit_msg>Upper bound for S0 is not 1<commit_after>#!/usr/bin/python
""" Classes and functions for fitting ivim model """
from __future__ import division, print_function, absolute_import
import warnings
import functools
import numpy as np
from dipy.core.optimize import Optimizer

from scipy.optimize import leastsq

from dipy.core.gradients import gradient_table
from dipy.reconst.base import ReconstModel
from dipy.reconst.dti import _min_positive_signal, apparent_diffusion_coef
from dipy.reconst.dti import TensorModel, mean_diffusivity
from .vec_val_sum import vec_val_vect
from dipy.core.sphere import Sphere

def ivim_function(params, bvals):
    """The ivim function
    Parameters
    ----------
    params : array
             The IVIM parameters [S0, f, D_star, D]

    bvals  : array
             bvalues
    """
    S0, f, D_star, D = params
    S = S0 * (f * np.exp(-bvals * D_star) + (1 - f) * np.exp(-bvals * D))
    return S

def _ivim_error(params, bvals, signal):
    """Error function to be used in fitting the model
    """
    return (signal - ivim_function(params, bvals))

class IvimModel(ReconstModel):
    """Ivim model
    """

    def __init__(self, gtab, split_b=200.0, min_signal=None):
        """
        An IVIM model

        Parameters
        ----------
        gtab : GradientTable class instance
        split_b : split the data at this b-value for a two stage fit

        References
        ----------
        .. [1] Le Bihan, Denis, et al. "Separation of diffusion
                   and perfusion in intravoxel incoherent motion MR
                   imaging." Radiology 168.2 (1988): 497-505.
        .. [2] Federau, Christian, et al. "Quantitative measurement
                   of brain perfusion with intravoxel incoherent motion
                   MR imaging." Radiology 265.3 (2012): 874-881.
        """
        ReconstModel.__init__(self, gtab)
        self.split_b = split_b
        self.min_signal = min_signal
        if self.min_signal is not None and self.min_signal <= 0:
            e_s = "The `min_signal` key-word argument needs to be strictly"
            e_s += " positive."
            raise ValueError(e_s)

    def fit(self, data, mask=None, x0=None, fit_method="one_stage", routine="minimize",
            jac=False, bounds=((0, None), (0, 1.), (0, 1.), (0, 1.)), tol=1e-25, algorithm='L-BFGS-B',
            gtol=1e-25, ftol=1e-25, eps=1e-15):
        """ Fit method of the Ivim model class

        Parameters
        ----------
        data : array
            The measured signal from one voxel.
        mask : array
            A boolean array used to mark the coordinates in the data that
            should be analyzed that has the shape data.shape[:-1]
        x0 : array
            Initial guesses for the parameters S0, f, D_star and D
            Default x0 = [1.0, 0.10, 0.001, 0.0009]
            These guess parameters are taken from the Federau paper
            Dimension can either be 1 or (N, 4) where N is the number of
            voxels in the data.
        fit_method: str
            Use one-stage fitting or two-stage fitting.
        jac : Boolean
            Use the Jacobian
        bounds : tuple
            Bounds for the various parameters
        tol : float
            tolerance for convergence
        method : str
            Fitting algorithm to use from scipy.optimize

        Returns
        -------
        IvimFit object
        """
        if mask is None:
            # Flatten it to 2D either way:
            data_in_mask = np.reshape(data, (-1, data.shape[-1]))
        else:
            # Check for valid shape of the mask
            if mask.shape != data.shape[:-1]:
                raise ValueError("Mask is not the same shape as data.")
            mask = np.array(mask, dtype=bool, copy=False)
            data_in_mask = np.reshape(data[mask], (-1, data.shape[-1]))

        if self.min_signal is None:
            min_signal = _min_positive_signal(data)
        else:
            min_signal = self.min_signal

        # Generate guess_parameters for all voxels
        if x0 is None:
            x0 = np.ones(
                (data.shape[:-1] + (4,))) * [1.0, 0.10, 0.001, 0.0009]
        else:
            if x0.shape != (data.shape[0], 4):
                raise ValueError(
                    "Guess params should be of shape (num_voxels,4)")
        data_in_mask = np.maximum(data_in_mask, min_signal)

        if fit_method == "one_stage":
            params_in_mask = one_stage(data_in_mask, self.gtab, x0,
                                       jac, bounds, tol,
                                       routine,
                                       algorithm, gtol, ftol, eps)
        elif fit_method == "two_stage":
            params_in_mask = two_stage(data_in_mask, self.gtab, x0,
                                       self.split_b, jac, bounds, tol,
                                       routine,
                                       algorithm, gtol, ftol, eps)
        else:
            raise ValueError("""Fit method must be either
                             'one_stage' or 'two_stage'""")

        if mask is None:
            out_shape = data.shape[:-1] + (-1, )
            ivim_params = params_in_mask.reshape(out_shape)
        else:
            ivim_params = np.zeros(data.shape[:-1] + (4,))
            ivim_params[mask, :] = params_in_mask

        return IvimFit(self, ivim_params)

    def predict(self, ivim_params):
        """
        Predict a signal for this IvimModel class instance given parameters.

        Parameters
        ----------
        ivim_params : ndarray
            The last dimension should have 4 parameters: S0, f, D_star and D
        """

        return ivim_function(ivim_params, self.gtab.bvals)

class IvimFit(object):

    def __init__(self, model, model_params):
        """ Initialize a IvimFit class instance.
            Parameters
            ----------
            The model parameters are S0, f, D_star, D
        """
        self.model = model
        self.model_params = model_params

    def __getitem__(self, index):
        model_params = self.model_params
        N = model_params.ndim
        if type(index) is not tuple:
            index = (index,)
        elif len(index) >= model_params.ndim:
            raise IndexError("IndexError: invalid index")
        index = index + (slice(None),) * (N - len(index))
        return type(self)(self.model, model_params[index])

    @property
    def shape(self):
        return self.model_params.shape[:-1]

    def predict(self, gtab, step=None, S0=1.0):
        r"""
        Given a model fit, predict the signal.

        Parameters
        ----------
        gtab : a GradientTable class instance
            This encodes the directions for which a prediction is made

        Notes
        -----
        The predicted signal is given by:

        .. math ::

        Where:
        .. math ::

        """
        return ivim_function(self.model_params, gtab.bvals)

def one_stage(data, gtab, x0, jac, bounds, tol, routine, algorithm,
              gtol, ftol, eps):
    """
    Fit the ivim params using minimize

    Parameters
    ----------
    data : array ([X, Y, Z, ...], g)
        Data or response variables holding the data. Note that the last
        dimension should contain the data. It makes no copies of data.

    jac : bool
        Use the Jacobian? Default: False

    Returns
    -------
    ivim_params: the S0, f, D_star, D value for each voxel.

    """
    flat_data = data.reshape((-1, data.shape[-1]))
    flat_x0 = x0.reshape(-1, x0.shape[-1])
    # Flatten for the iteration over voxels:
    bvals = gtab.bvals
    ivim_params = np.empty((flat_data.shape[0], 4))

    if routine == 'minimize':
        result = _minimize(flat_data, bvals, flat_x0, ivim_params,
                           bounds, tol, jac, algorithm, gtol, ftol, eps)
    elif routine == "leastsq":
        result = _leastsq(flat_data, bvals, flat_x0, ivim_params)
    ivim_params.shape = data.shape[:-1] + (4,)
    return ivim_params

def _minimize(flat_data, bvals, flat_x0, ivim_params,
              bounds, tol, jac, algorithm, gtol, ftol, eps):
    """Use minimize for finding ivim_params"""
    sum_sq = lambda params, bvals, signal: np.sum(
        _ivim_error(params, bvals, signal)**2)

    result = []
    for vox in range(flat_data.shape[0]):
        res = Optimizer(sum_sq,
                        flat_x0[vox],
                        args=(bvals, flat_data[vox]), bounds=bounds,
                        tol=tol, method=algorithm, jac=jac,
                        options={'gtol': gtol, 'ftol': ftol, 'eps': eps})
        ivim_params[vox, :4] = res.xopt
        result += [res]
    return result

def _leastsq(flat_data, bvals, flat_x0, ivim_params):
    """Use minimize for finding ivim_params"""
    result = []
    for vox in range(flat_data.shape[0]):
        res = leastsq(_ivim_error,
                      flat_x0[vox],
                      args=(bvals, flat_data[vox]))
        ivim_params[vox, :4] = res[0]
        result += [res]
    return result

def two_stage(data, gtab, x0,
              split_b, jac, bounds, tol,
              routine, algorithm, gtol, ftol, eps):
    """
    Fit the ivim params using a two stage fit

    Parameters
    ----------
    data : array ([X, Y, Z, ...], g)
        Data or response variables holding the data. Note that the last
        dimension should contain the data. It makes no copies of data.

    jac : bool
        Use the Jacobian? Default: False

    Returns
    -------
    ivim_params: the S0, f, D_star, D value for each voxel.

    """
    flat_data = data.reshape((-1, data.shape[-1]))
    # Flatten for the iteration over voxels:
    bvals_ge_split = gtab.bvals[gtab.bvals > split_b]
    bvecs_ge_split = gtab.bvecs[gtab.bvals > split_b]
    gtab_ge_split = gradient_table(bvals_ge_split, bvecs_ge_split.T)

    tensor_model = TensorModel(gtab_ge_split)
    tenfit = tensor_model.fit(data[..., gtab.bvals > split_b])

    D_guess = mean_diffusivity(tenfit.evals)

    x0[..., 3] = D_guess

    dti_params = tenfit.model_params

    S0_hat = get_S0_guess(dti_params, data, gtab_ge_split)
    f_guess = 1.0 - S0_hat / data[..., 0]
    x0[..., 1] = f_guess
    return one_stage(data, gtab, x0, jac, bounds, tol, routine, algorithm,
                     gtol, ftol, eps)

def get_S0_guess(dti_params, data, gtab):
    evecs = dti_params[..., 3:12].reshape((dti_params.shape[:-1] + (3, 3)))
    evals = dti_params[..., :3]
    qform = vec_val_vect(evecs, evals)

    sphere = Sphere(xyz=gtab.bvecs[~gtab.b0s_mask])
    ADC = apparent_diffusion_coef(qform, sphere)

    S0_hat = np.mean(data[..., ~gtab.b0s_mask] /
                     np.exp(-gtab.bvals[~gtab.b0s_mask] * ADC),
                     -1)
    return S0_hat
<|endoftext|>


<commit_before>import json
from uuid import uuid1

from sqlalchemy import func
from sqlalchemy.exc import IntegrityError
from flask import Blueprint, request, jsonify, make_response
from bson.errors import InvalidId

from mhn import db
from mhn.api import errors
from mhn.api.models import (
        Sensor, Rule, DeployScript as Script,
        DeployScript, RuleSource)
from mhn.api.decorators import deploy_auth, sensor_auth
from mhn.common.utils import error_response
from mhn.common.clio import Clio
from mhn.auth import current_user, login_required

api = Blueprint('api', __name__, url_prefix='/api')

# Endpoints for the Sensor resource.
@api.route('/sensor/', methods=['POST'])
@deploy_auth
def create_sensor():
    missing = Sensor.check_required(request.json)
    if missing:
        return error_response(
                errors.API_FIELDS_MISSING.format(missing), 400)
    else:
        sensor = Sensor(**request.json)
        sensor.uuid = str(uuid1())
        Clio().authkey.new(**sensor.new_auth_dict()).post()
        try:
            db.session.add(sensor)
            db.session.commit()
        except IntegrityError:
            return error_response(
                    errors.API_SENSOR_EXISTS.format(request.json['name']), 400)
        else:
            return jsonify(sensor.to_dict())

@api.route('/sensor/<uuid>/', methods=['PUT'])
def update_sensor(uuid):
    sensor = Sensor.query.filter_by(uuid=uuid).first_or_404()
    for field in request.json.keys():
        if field in Sensor.editable_fields():
            setattr(sensor, field, request.json[field])
        elif field in Sensor.fields():
            return error_response(
                    errors.API_FIELD_NOT_EDITABLE.format(field), 400)
        else:
            return error_response(
                    errors.API_FIELD_INVALID.format(field), 400)
    else:
        try:
            db.session.commit()
        except IntegrityError:
            return error_response(
                    errors.API_SENSOR_EXISTS.format(request.json['name']), 400)
        return jsonify(sensor.to_dict())

@api.route('/sensor/<uuid>/', methods=['DELETE'])
@login_required
def delete_sensor(uuid):
    sensor = Sensor.query.filter_by(uuid=uuid).first_or_404()
    db.session.delete(sensor)
    db.session.commit()
    return jsonify({})

@api.route('/sensor/<uuid>/connect/', methods=['POST'])
@sensor_auth
def connect_sensor(uuid):
    sensor = Sensor.query.filter_by(uuid=uuid).first_or_404()
    sensor.ip = request.remote_addr
    db.session.commit()
    return jsonify(sensor.to_dict())

# Utility functions that generalize the GET
# requests of resources from Mnemosyne.
def _get_one_resource(resource, res_id):
    try:
        res = resource.get(_id=res_id)
    except InvalidId:
        res = None
    if not res:
        return error_response(errors.API_RESOURCE_NOT_FOUND, 404)
    else:
        return jsonify(res.to_dict())

def _get_query_resource(resource, query):
    return jsonify(data=[r.to_dict() for r in resource.get(**query)])
# Now let's make use these methods in the views.

@api.route('/feed/<feed_id>/', methods=['GET'])
@login_required
def get_feed(feed_id):
    return _get_one_resource(Clio().hpfeed, feed_id)

@api.route('/session/<session_id>/', methods=['GET'])
@login_required
def get_session(session_id):
    return _get_one_resource(Clio().session, session_id)

@api.route('/feed/', methods=['GET'])
@login_required
def get_feeds():
    return _get_query_resource(Clio().hpfeed, request.args.to_dict())

@api.route('/session/', methods=['GET'])
@login_required
def get_sessions():
    return _get_query_resource(Clio().session, request.args.to_dict())

@api.route('/rule/<rule_id>/', methods=['PUT'])
@login_required
def update_rule(rule_id):
    rule = Rule.query.filter_by(id=rule_id).first_or_404()
    for field in request.json.keys():
        if field in Rule.editable_fields():
            setattr(rule, field, request.json[field])
        elif field in Rule.fields():
            return error_response(
                    errors.API_FIELD_NOT_EDITABLE.format(field), 400)
        else:
            return error_response(
                    errors.API_FIELD_INVALID.format(field), 400)
    else:
        db.session.commit()
        return jsonify(rule.to_dict())

@api.route('/rule/', methods=['GET'])
@sensor_auth
def get_rules():
    # Getting active rules.
    if request.args.get('plaintext') in ['1', 'true']:
        # Requested rendered rules in plaintext.
        resp = make_response(Rule.renderall())
        resp.headers['Content-Disposition'] = "attachment; filename=mhn.rules"
        return resp
    else:
        # Responding with active rules.
        rules = Rule.query.filter_by(is_active=True).\
                    group_by(Rule.sid).\
                    having(func.max(Rule.rev))
        resp = make_response(json.dumps([ru.to_dict() for ru in rules]))
        resp.headers['Content-Type'] = "application/json"
        return resp

@api.route('/rulesources/', methods=['POST'])
@login_required
def create_rule_source():
    missing = RuleSource.check_required(request.json)
    if missing:
        return error_response(
                errors.API_FIELDS_MISSING.format(missing), 400)
    else:
        rsource = RuleSource(**request.json)
        try:
            db.session.add(rsource)
            db.session.commit()
        except IntegrityError:
            return error_response(
                    errors.API_SOURCE_EXISTS.format(request.json['uri']), 400)
        else:
            return jsonify(rsource.to_dict())

@api.route('/rulesources/<rs_id>/', methods=['DELETE'])
@login_required
def delete_rule_source(rs_id):
    source = RuleSource.query.filter_by(id=rs_id).first_or_404()
    db.session.delete(source)
    db.session.commit()
    return jsonify({})

@api.route('/script/', methods=['POST'])
@login_required
def create_script():
    missing = Script.check_required(request.json)
    if missing:
        return error_response(
                errors.API_FIELDS_MISSING.format(missing), 400)
    else:
        script = Script(**request.json)
        script.user = current_user
        db.session.add(script)
        db.session.commit()
        return jsonify(script.to_dict())

@api.route('/script/', methods=['PUT', 'PATCH'])
@login_required
def update_script():
    script = Script.query.get(request.json.get('id'))
    script.user = current_user
    for editable in Script.editable_fields():
        if editable in request.json:
            setattr(script, editable, request.json[editable])
    db.session.add(script)
    db.session.commit()
    return jsonify(script.to_dict())

@api.route('/script/', methods=['GET'])
def get_script():
    if request.args.get('script_id'):
        script = DeployScript.query.get(request.args.get('script_id'))
    else:
        script = DeployScript.query.order_by(DeployScript.date.desc()).first()
    if request.args.get('text') in ['1', 'true']:
        resp = make_response(script.script)
        resp.headers['Content-Disposition'] = "attachment; filename=deploy.sh"
        return resp
    else:
        return jsonify(script.to_dict())
<commit_msg>Take ip at registration<commit_after>import json
from uuid import uuid1

from sqlalchemy import func
from sqlalchemy.exc import IntegrityError
from flask import Blueprint, request, jsonify, make_response
from bson.errors import InvalidId

from mhn import db
from mhn.api import errors
from mhn.api.models import (
        Sensor, Rule, DeployScript as Script,
        DeployScript, RuleSource)
from mhn.api.decorators import deploy_auth, sensor_auth
from mhn.common.utils import error_response
from mhn.common.clio import Clio
from mhn.auth import current_user, login_required

api = Blueprint('api', __name__, url_prefix='/api')

# Endpoints for the Sensor resource.
@api.route('/sensor/', methods=['POST'])
@deploy_auth
def create_sensor():
    missing = Sensor.check_required(request.json)
    if missing:
        return error_response(
                errors.API_FIELDS_MISSING.format(missing), 400)
    else:
        sensor = Sensor(**request.json)
        sensor.uuid = str(uuid1())
        sensor.ip = request.remote_addr
        Clio().authkey.new(**sensor.new_auth_dict()).post()
        try:
            db.session.add(sensor)
            db.session.commit()
        except IntegrityError:
            return error_response(
                    errors.API_SENSOR_EXISTS.format(request.json['name']), 400)
        else:
            return jsonify(sensor.to_dict())

@api.route('/sensor/<uuid>/', methods=['PUT'])
def update_sensor(uuid):
    sensor = Sensor.query.filter_by(uuid=uuid).first_or_404()
    for field in request.json.keys():
        if field in Sensor.editable_fields():
            setattr(sensor, field, request.json[field])
        elif field in Sensor.fields():
            return error_response(
                    errors.API_FIELD_NOT_EDITABLE.format(field), 400)
        else:
            return error_response(
                    errors.API_FIELD_INVALID.format(field), 400)
    else:
        try:
            db.session.commit()
        except IntegrityError:
            return error_response(
                    errors.API_SENSOR_EXISTS.format(request.json['name']), 400)
        return jsonify(sensor.to_dict())

@api.route('/sensor/<uuid>/', methods=['DELETE'])
@login_required
def delete_sensor(uuid):
    sensor = Sensor.query.filter_by(uuid=uuid).first_or_404()
    db.session.delete(sensor)
    db.session.commit()
    return jsonify({})

@api.route('/sensor/<uuid>/connect/', methods=['POST'])
@sensor_auth
def connect_sensor(uuid):
    sensor = Sensor.query.filter_by(uuid=uuid).first_or_404()
    sensor.ip = request.remote_addr
    db.session.commit()
    return jsonify(sensor.to_dict())

# Utility functions that generalize the GET
# requests of resources from Mnemosyne.
def _get_one_resource(resource, res_id):
    try:
        res = resource.get(_id=res_id)
    except InvalidId:
        res = None
    if not res:
        return error_response(errors.API_RESOURCE_NOT_FOUND, 404)
    else:
        return jsonify(res.to_dict())

def _get_query_resource(resource, query):
    return jsonify(data=[r.to_dict() for r in resource.get(**query)])
# Now let's make use these methods in the views.

@api.route('/feed/<feed_id>/', methods=['GET'])
@login_required
def get_feed(feed_id):
    return _get_one_resource(Clio().hpfeed, feed_id)

@api.route('/session/<session_id>/', methods=['GET'])
@login_required
def get_session(session_id):
    return _get_one_resource(Clio().session, session_id)

@api.route('/feed/', methods=['GET'])
@login_required
def get_feeds():
    return _get_query_resource(Clio().hpfeed, request.args.to_dict())

@api.route('/session/', methods=['GET'])
@login_required
def get_sessions():
    return _get_query_resource(Clio().session, request.args.to_dict())

@api.route('/rule/<rule_id>/', methods=['PUT'])
@login_required
def update_rule(rule_id):
    rule = Rule.query.filter_by(id=rule_id).first_or_404()
    for field in request.json.keys():
        if field in Rule.editable_fields():
            setattr(rule, field, request.json[field])
        elif field in Rule.fields():
            return error_response(
                    errors.API_FIELD_NOT_EDITABLE.format(field), 400)
        else:
            return error_response(
                    errors.API_FIELD_INVALID.format(field), 400)
    else:
        db.session.commit()
        return jsonify(rule.to_dict())

@api.route('/rule/', methods=['GET'])
@sensor_auth
def get_rules():
    # Getting active rules.
    if request.args.get('plaintext') in ['1', 'true']:
        # Requested rendered rules in plaintext.
        resp = make_response(Rule.renderall())
        resp.headers['Content-Disposition'] = "attachment; filename=mhn.rules"
        return resp
    else:
        # Responding with active rules.
        rules = Rule.query.filter_by(is_active=True).\
                    group_by(Rule.sid).\
                    having(func.max(Rule.rev))
        resp = make_response(json.dumps([ru.to_dict() for ru in rules]))
        resp.headers['Content-Type'] = "application/json"
        return resp

@api.route('/rulesources/', methods=['POST'])
@login_required
def create_rule_source():
    missing = RuleSource.check_required(request.json)
    if missing:
        return error_response(
                errors.API_FIELDS_MISSING.format(missing), 400)
    else:
        rsource = RuleSource(**request.json)
        try:
            db.session.add(rsource)
            db.session.commit()
        except IntegrityError:
            return error_response(
                    errors.API_SOURCE_EXISTS.format(request.json['uri']), 400)
        else:
            return jsonify(rsource.to_dict())

@api.route('/rulesources/<rs_id>/', methods=['DELETE'])
@login_required
def delete_rule_source(rs_id):
    source = RuleSource.query.filter_by(id=rs_id).first_or_404()
    db.session.delete(source)
    db.session.commit()
    return jsonify({})

@api.route('/script/', methods=['POST'])
@login_required
def create_script():
    missing = Script.check_required(request.json)
    if missing:
        return error_response(
                errors.API_FIELDS_MISSING.format(missing), 400)
    else:
        script = Script(**request.json)
        script.user = current_user
        db.session.add(script)
        db.session.commit()
        return jsonify(script.to_dict())

@api.route('/script/', methods=['PUT', 'PATCH'])
@login_required
def update_script():
    script = Script.query.get(request.json.get('id'))
    script.user = current_user
    for editable in Script.editable_fields():
        if editable in request.json:
            setattr(script, editable, request.json[editable])
    db.session.add(script)
    db.session.commit()
    return jsonify(script.to_dict())

@api.route('/script/', methods=['GET'])
def get_script():
    if request.args.get('script_id'):
        script = DeployScript.query.get(request.args.get('script_id'))
    else:
        script = DeployScript.query.order_by(DeployScript.date.desc()).first()
    if request.args.get('text') in ['1', 'true']:
        resp = make_response(script.script)
        resp.headers['Content-Disposition'] = "attachment; filename=deploy.sh"
        return resp
    else:
        return jsonify(script.to_dict())
<|endoftext|>


<commit_before>from django.core.exceptions import ObjectDoesNotExist
from django.db.models.signals import pre_save
from django.dispatch import receiver
from directory.models import *
from lxml import html
import requests
import re

@receiver(pre_save, sender=Rfc)
def complete_rfc_instance(sender, instance, *args, **kwargs):
    """Automatically fetches general document information
    from ietf.org before saving RFC instance."""

    def get_year(response):
        tree = html.fromstring(response.content)
        docinfo = " ".join(
            tree.xpath('//pre[1]/text()')
        )
        month_list = ['January', 'February', 'March', 'April', 'May', 'June',
            'July', 'August', 'September', 'October', 'November', 'December']
        month_and_year = re.compile(
            r'\b(?:%s)\b.*?(\d{4})' % '|'.join(month_list)
        )
        match = month_and_year.search(docinfo)
        return int(match.group(1))

    def get_title(response):
        tree = html.fromstring(response.content)
        headers = tree.xpath('//span[@class="h1"]/text()')
        return " ".join(headers)

    def get_status(response):
        tree = html.fromstring(response.content)
        # concat all fields possibly containing doc status
        docinfo = " ".join(
            tree.xpath('//span[@class="pre noprint docinfo"]/text()')
        )

        # search for predefined options
        if re.search('INTERNET STANDARD', docinfo):
            return 'IST'
        elif re.search('PROPOSED STANDARD', docinfo):
            return 'PST'
        elif re.search('DRAFT STANDARD', docinfo):
            return 'DST'
        elif re.search('BEST CURRENT PRACTISE', docinfo):
            return 'BCP'
        elif re.search('INFORMATIONAL', docinfo):
            return 'INF'
        elif re.search('EXPERIMENTAL', docinfo):
            return 'EXP'
        elif re.search('HISTORIC', docinfo):
            return 'HST'
        else:
            return 'UND'

    if instance.is_draft:
        url = f"https://tools.ietf.org/html/draft-ietf-tls-rfc{instance.number}"
    else:
        url = f"https://tools.ietf.org/html/rfc{instance.number}"
    resp = requests.get(url)
    if resp.status_code == 200:
        instance.url  = url
        instance.title = get_title(resp)
        instance.status = get_status(resp)
        instance.release_year = get_year(resp)
    else:
        # cancel saving the instance if unable to receive web page
        raise Exception('RFC not found')

@receiver(pre_save, sender=CipherSuite)
def complete_cs_instance(sender, instance, *args, **kwargs):
    '''Derives related algorithms form instance.name of the cipher suites.'''

    # TLS1.3 ciphers start with 0x13
    if instance.hex_byte_1 == '0x13':
        name = instance.name
        (prt,_,rst) = name.replace("_", " ").partition(" ")
        (enc,_,hsh) = rst.rpartition(" ")
        aut = "-"
        kex = "-"

    else:
        # EXPORT substring does not describe any algorithm, so we remove it
        # substring is later appended to the protocol_version
        if re.search("EXPORT", instance.name):
            name = instance.name.replace('EXPORT_', '')
            export_cipher = True
        else:
            name = instance.name
            export_cipher = False

        (prt,_,rst) = name.replace("_", " ").partition(" ")
        (kex,_,rst) = rst.partition("WITH")

        # add information about export-grade cipher to protocol version
        if export_cipher:
            prt += " EXPORT"

        # split kex again, potentially yielding auth algorithm
        # otherwise this variable will remain unchanged
        (kex,_,aut) = kex.partition(" ")
        (enc,_,hsh) = rst.rpartition(" ")

        # split enc again if we only got a number for hsh
        # specifically needed for 'CCM 8' hash algorithm
        if re.match(r'\d+', hsh.strip()):
            enc += " " + hsh
            hsh = "SHA256"

    # if aut is not excplicitly defined, set it equal to kex
    if not aut:
        instance.auth_algorithm, _ = AuthAlgorithm.objects.get_or_create(
            short_name=kex.strip()
        )
    else:
        instance.auth_algorithm, _ = AuthAlgorithm.objects.get_or_create(
            short_name=aut.strip()
        )

    # connect foreign keys from other models
    instance.kex_algorithm, _ = KexAlgorithm.objects.get_or_create(
        short_name=kex.strip()
    )

    instance.protocol_version, _ = ProtocolVersion.objects.get_or_create(
        short_name=prt.strip()
    )
    instance.enc_algorithm, _ = EncAlgorithm.objects.get_or_create(
        short_name=enc.strip()
    )
    instance.hash_algorithm, _ = HashAlgorithm.objects.get_or_create(
        short_name=hsh.strip()
    )

@receiver(pre_save, sender=CipherSuite)
def complete_cs_names(sender, instance, *args, **kwargs):
    try:
        related_gnutls = GnutlsCipher.objects.get(hex_byte_1__iexact=instance.hex_byte_1,
                                                  hex_byte_2__iexact=instance.hex_byte_2)
        instance.gnutls_name = related_gnutls.name
    except ObjectDoesNotExist:
        pass

    try:
        related_openssl = OpensslCipher.objects.get(hex_byte_1__iexact=instance.hex_byte_1,
                                                    hex_byte_2__iexact=instance.hex_byte_2)
        instance.openssl_name = related_openssl.name
    except ObjectDoesNotExist:
        pass

@receiver(pre_save, sender=TlsVersion)
def complete_tls_version(sender, instance, *args, **kwargs):
    instance.short = f"{instance.major}{instance.minor}"
<commit_msg>fixes ccm parsing to include ccm-only ciphers<commit_after>from django.core.exceptions import ObjectDoesNotExist
from django.db.models.signals import pre_save
from django.dispatch import receiver
from directory.models import *
from lxml import html
import requests
import re

@receiver(pre_save, sender=Rfc)
def complete_rfc_instance(sender, instance, *args, **kwargs):
    """Automatically fetches general document information
    from ietf.org before saving RFC instance."""

    def get_year(response):
        tree = html.fromstring(response.content)
        docinfo = " ".join(
            tree.xpath('//pre[1]/text()')
        )
        month_list = ['January', 'February', 'March', 'April', 'May', 'June',
            'July', 'August', 'September', 'October', 'November', 'December']
        month_and_year = re.compile(
            r'\b(?:%s)\b.*?(\d{4})' % '|'.join(month_list)
        )
        match = month_and_year.search(docinfo)
        return int(match.group(1))

    def get_title(response):
        tree = html.fromstring(response.content)
        headers = tree.xpath('//span[@class="h1"]/text()')
        return " ".join(headers)

    def get_status(response):
        tree = html.fromstring(response.content)
        # concat all fields possibly containing doc status
        docinfo = " ".join(
            tree.xpath('//span[@class="pre noprint docinfo"]/text()')
        )

        # search for predefined options
        if re.search('INTERNET STANDARD', docinfo):
            return 'IST'
        elif re.search('PROPOSED STANDARD', docinfo):
            return 'PST'
        elif re.search('DRAFT STANDARD', docinfo):
            return 'DST'
        elif re.search('BEST CURRENT PRACTISE', docinfo):
            return 'BCP'
        elif re.search('INFORMATIONAL', docinfo):
            return 'INF'
        elif re.search('EXPERIMENTAL', docinfo):
            return 'EXP'
        elif re.search('HISTORIC', docinfo):
            return 'HST'
        else:
            return 'UND'

    if instance.is_draft:
        url = f"https://tools.ietf.org/html/draft-ietf-tls-rfc{instance.number}"
    else:
        url = f"https://tools.ietf.org/html/rfc{instance.number}"
    resp = requests.get(url)
    if resp.status_code == 200:
        instance.url  = url
        instance.title = get_title(resp)
        instance.status = get_status(resp)
        instance.release_year = get_year(resp)
    else:
        # cancel saving the instance if unable to receive web page
        raise Exception('RFC not found')

@receiver(pre_save, sender=CipherSuite)
def complete_cs_instance(sender, instance, *args, **kwargs):
    '''Derives related algorithms form instance.name of the cipher suites.'''

    # TLS1.3 ciphers start with 0x13
    if instance.hex_byte_1 == '0x13':
        name = instance.name
        (prt,_,rst) = name.replace("_", " ").partition(" ")
        (enc,_,hsh) = rst.rpartition(" ")
        aut = "-"
        kex = "-"

    else:
        # EXPORT substring does not describe any algorithm, so we remove it
        # substring is later appended to the protocol_version
        if re.search("EXPORT", instance.name):
            name = instance.name.replace('EXPORT_', '')
            export_cipher = True
        else:
            name = instance.name
            export_cipher = False

        (prt,_,rst) = name.replace("_", " ").partition(" ")
        (kex,_,rst) = rst.partition("WITH")

        # add information about export-grade cipher to protocol version
        if export_cipher:
            prt += " EXPORT"

        # split kex again, potentially yielding auth algorithm
        # otherwise this variable will remain unchanged
        (kex,_,aut) = kex.partition(" ")
        (enc,_,hsh) = rst.rpartition(" ")

        # split enc again if we only got a number for hsh
        # specifically needed for CCM/CCM8 ciphers
        if re.match(r'\d+', hsh.strip()) or re.match(r'CCM\Z', hsh.strip()):
            enc += " " + hsh
            hsh = "SHA256"

    # if aut is not excplicitly defined, set it equal to kex
    if not aut:
        instance.auth_algorithm, _ = AuthAlgorithm.objects.get_or_create(
            short_name=kex.strip()
        )
    else:
        instance.auth_algorithm, _ = AuthAlgorithm.objects.get_or_create(
            short_name=aut.strip()
        )

    # connect foreign keys from other models
    instance.kex_algorithm, _ = KexAlgorithm.objects.get_or_create(
        short_name=kex.strip()
    )

    instance.protocol_version, _ = ProtocolVersion.objects.get_or_create(
        short_name=prt.strip()
    )
    instance.enc_algorithm, _ = EncAlgorithm.objects.get_or_create(
        short_name=enc.strip()
    )
    instance.hash_algorithm, _ = HashAlgorithm.objects.get_or_create(
        short_name=hsh.strip()
    )

@receiver(pre_save, sender=CipherSuite)
def complete_cs_names(sender, instance, *args, **kwargs):
    try:
        related_gnutls = GnutlsCipher.objects.get(hex_byte_1__iexact=instance.hex_byte_1,
                                                  hex_byte_2__iexact=instance.hex_byte_2)
        instance.gnutls_name = related_gnutls.name
    except ObjectDoesNotExist:
        pass

    try:
        related_openssl = OpensslCipher.objects.get(hex_byte_1__iexact=instance.hex_byte_1,
                                                    hex_byte_2__iexact=instance.hex_byte_2)
        instance.openssl_name = related_openssl.name
    except ObjectDoesNotExist:
        pass

@receiver(pre_save, sender=TlsVersion)
def complete_tls_version(sender, instance, *args, **kwargs):
    instance.short = f"{instance.major}{instance.minor}"
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

###############################################################################
#  Copyright Kitware Inc.
#
#  Licensed under the Apache License, Version 2.0 ( the "License" );
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
###############################################################################

import json

from girder.api import access
from girder.api.describe import Description
from girder.api.rest import Resource, RestException

from girder.plugins.minerva.utility.minerva_utility import (findAnalysisFolder,
                                                            findAnalysisByName,
                                                            addJobOutput)
from girder.plugins.minerva.rest.dataset import Dataset

class Analysis(Resource):
    def __init__(self):
        self.resourceName = 'minerva_analysis'
        self.route('GET', ('folder',), self.getAnalysisFolder)
        self.route('POST', ('folder',), self.createAnalysisFolder)
        self.route('POST', ('bsve_search',), self.bsveSearchAnalysis)
        self.route('POST', ('mmwr_import',), self.bsveMMWRAnalysis)

    @access.user
    def getAnalysisFolder(self, params):
        folder = findAnalysisFolder(self.getCurrentUser())
        return {'folder': folder}
    getAnalysisFolder.description = (
        Description('Get the minerva analysis folder.'))

    @access.user
    def createAnalysisFolder(self, params):
        folder = findAnalysisFolder(self.getCurrentUser(), create=True)
        return {'folder': folder}
    createAnalysisFolder.description = (
        Description('Create the minerva analysis folder, a global resource.'))

    @access.user
    def bsveSearchAnalysis(self, params):
        currentUser = self.getCurrentUser()
        datasetName = params['datasetName']
        bsveSearchParams = params['bsveSearchParams']
        analysis = findAnalysisByName(currentUser, 'bsve search')
        # TODO in case can't find analysis?

        try:
            bsveSearchParams = json.loads(bsveSearchParams)
        except ValueError:
            raise RestException('bsveSearchParams is invalid JSON.')

        minerva_metadata = {
            'dataset_type': 'geojson',
            'source': 'bsve_search',
            'bsve_search_params': bsveSearchParams,
            'original_type': 'json'
        }

        datasetResource = Dataset()
        dataset = datasetResource.constructDataset(datasetName,
                                                   minerva_metadata,
                                                   'created by bsve search')
        params = {
            'bsveSearchParams': bsveSearchParams
        }

        # TODO change token to job token
        user, token = self.getCurrentUser(returnToken=True)
        kwargs = {
            'params': params,
            'user': currentUser,
            'dataset': dataset,
            'analysis': analysis,
            'token': token
        }

        job = self.model('job', 'jobs').createLocalJob(
            title='bsve search: %s' % datasetName,
            user=currentUser,
            type='bsve.search',
            public=False,
            kwargs=kwargs,
            module='girder.plugins.minerva.jobs.bsve_search_worker',
            async=True)
        addJobOutput(job, dataset)
        self.model('job', 'jobs').scheduleJob(job)
        return job

    bsveSearchAnalysis.description = (
        Description('Create the minerva analysis folder, a global resource.')
        .param('datasetName', 'Name of the dataset created by this analysis.')
        .param('bsveSearchParams', 'JSON search parameters to send to bsve.'))

    @access.user
    def bsveMMWRAnalysis(self, params):
        currentUser = self.getCurrentUser()
        datasetName = params['datasetName']
        count = int(params.get('count', 1000))
        analysis = findAnalysisByName(currentUser, 'MMWR data import')
        # TODO in case can't find analysis?

        minerva_metadata = {
            'dataset_type': 'geojson',
            'source': 'mmwr_data_import',
            'original_type': 'json'
        }

        datasetResource = Dataset()
        dataset = datasetResource.constructDataset(
            datasetName,
            minerva_metadata,
            'created by MMWR data import'
        )
        params = {
            'count': count
        }

        # TODO change token to job token
        user, token = self.getCurrentUser(returnToken=True)
        kwargs = {
            'params': params,
            'user': currentUser,
            'dataset': dataset,
            'analysis': analysis,
            'token': token
        }

        job = self.model('job', 'jobs').createLocalJob(
            title='MMWR import: %s' % datasetName,
            user=currentUser,
            type='bsve.mmwr',
            public=False,
            kwargs=kwargs,
            module='girder.plugins.minerva.jobs.soda_import_worker',
            async=True)
        addJobOutput(job, dataset)
        self.model('job', 'jobs').scheduleJob(job)
        return job

    bsveMMWRAnalysis.description = (
        Description('Create a new accumulated MMWR dataset from the BSVE.')
        .param('datasetName', 'Name of the dataset created by this analysis.')
        .param(
            'count', 'The number of items to get from the server',
            required=False
        )
    )
<commit_msg>Change mmwr analysis results 'source' to 'source_type'<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

###############################################################################
#  Copyright Kitware Inc.
#
#  Licensed under the Apache License, Version 2.0 ( the "License" );
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
###############################################################################

import json

from girder.api import access
from girder.api.describe import Description
from girder.api.rest import Resource, RestException

from girder.plugins.minerva.utility.minerva_utility import (findAnalysisFolder,
                                                            findAnalysisByName,
                                                            addJobOutput)
from girder.plugins.minerva.rest.dataset import Dataset

class Analysis(Resource):
    def __init__(self):
        self.resourceName = 'minerva_analysis'
        self.route('GET', ('folder',), self.getAnalysisFolder)
        self.route('POST', ('folder',), self.createAnalysisFolder)
        self.route('POST', ('bsve_search',), self.bsveSearchAnalysis)
        self.route('POST', ('mmwr_import',), self.bsveMMWRAnalysis)

    @access.user
    def getAnalysisFolder(self, params):
        folder = findAnalysisFolder(self.getCurrentUser())
        return {'folder': folder}
    getAnalysisFolder.description = (
        Description('Get the minerva analysis folder.'))

    @access.user
    def createAnalysisFolder(self, params):
        folder = findAnalysisFolder(self.getCurrentUser(), create=True)
        return {'folder': folder}
    createAnalysisFolder.description = (
        Description('Create the minerva analysis folder, a global resource.'))

    @access.user
    def bsveSearchAnalysis(self, params):
        currentUser = self.getCurrentUser()
        datasetName = params['datasetName']
        bsveSearchParams = params['bsveSearchParams']
        analysis = findAnalysisByName(currentUser, 'bsve search')
        # TODO in case can't find analysis?

        try:
            bsveSearchParams = json.loads(bsveSearchParams)
        except ValueError:
            raise RestException('bsveSearchParams is invalid JSON.')

        minerva_metadata = {
            'dataset_type': 'geojson',
            'source_type': 'bsve_search',
            'bsve_search_params': bsveSearchParams,
            'original_type': 'json'
        }

        datasetResource = Dataset()
        dataset = datasetResource.constructDataset(datasetName,
                                                   minerva_metadata,
                                                   'created by bsve search')
        params = {
            'bsveSearchParams': bsveSearchParams
        }

        # TODO change token to job token
        user, token = self.getCurrentUser(returnToken=True)
        kwargs = {
            'params': params,
            'user': currentUser,
            'dataset': dataset,
            'analysis': analysis,
            'token': token
        }

        job = self.model('job', 'jobs').createLocalJob(
            title='bsve search: %s' % datasetName,
            user=currentUser,
            type='bsve.search',
            public=False,
            kwargs=kwargs,
            module='girder.plugins.minerva.jobs.bsve_search_worker',
            async=True)
        addJobOutput(job, dataset)
        self.model('job', 'jobs').scheduleJob(job)
        return job

    bsveSearchAnalysis.description = (
        Description('Create the minerva analysis folder, a global resource.')
        .param('datasetName', 'Name of the dataset created by this analysis.')
        .param('bsveSearchParams', 'JSON search parameters to send to bsve.'))

    @access.user
    def bsveMMWRAnalysis(self, params):
        currentUser = self.getCurrentUser()
        datasetName = params['datasetName']
        count = int(params.get('count', 1000))
        analysis = findAnalysisByName(currentUser, 'MMWR data import')
        # TODO in case can't find analysis?

        minerva_metadata = {
            'dataset_type': 'geojson',
            'source_type': 'mmwr_data_import',
            'original_type': 'json'
        }

        datasetResource = Dataset()
        dataset = datasetResource.constructDataset(
            datasetName,
            minerva_metadata,
            'created by MMWR data import'
        )
        params = {
            'count': count
        }

        # TODO change token to job token
        user, token = self.getCurrentUser(returnToken=True)
        kwargs = {
            'params': params,
            'user': currentUser,
            'dataset': dataset,
            'analysis': analysis,
            'token': token
        }

        job = self.model('job', 'jobs').createLocalJob(
            title='MMWR import: %s' % datasetName,
            user=currentUser,
            type='bsve.mmwr',
            public=False,
            kwargs=kwargs,
            module='girder.plugins.minerva.jobs.soda_import_worker',
            async=True)
        addJobOutput(job, dataset)
        self.model('job', 'jobs').scheduleJob(job)
        return job

    bsveMMWRAnalysis.description = (
        Description('Create a new accumulated MMWR dataset from the BSVE.')
        .param('datasetName', 'Name of the dataset created by this analysis.')
        .param(
            'count', 'The number of items to get from the server',
            required=False
        )
    )
<|endoftext|>


<commit_before>#!/usr/bin/python -O
import argparse
import socket

#
# This is needed to force ipv4 on ipv6 devices. It's sometimes needed
# if there isn't a clean ipv6 route to get to the big wild internet.
# In these cases, a pure ipv6 route simply will not work.  People aren't
# always in full control of every hop ... so it's much safer to force
# ipv4 then optimistically cross our fingers.
#
origGetAddrInfo = socket.getaddrinfo

def getAddrInfoWrapper(host, port, family=0, socktype=0, proto=0, flags=0):
  return origGetAddrInfo(host, port, socket.AF_INET, socktype, proto, flags)

# Replace the original socket.getaddrinfo by our version
socket.getaddrinfo = getAddrInfoWrapper

import urllib2
import sqlite3
import time

conn = sqlite3.connect('../db/main.db')
db = {'conn': conn, 'c': conn.cursor()}

CYCLE_TIME = 60 * 60 * 12

# this trick robbed from http://stackoverflow.com/questions/702834/whats-the-common-practice-for-enums-in-python
ID, CALLSIGN, DESCRIPTION, BASE_URL, LAST_SEEN, FIRST_SEEN, PINGS, DROPS, LATENCY, ACTIVE, LOG, NOTES = range(12)

parser = argparse.ArgumentParser()
parser.add_argument("-q", "--query", default="heartbeat", help="query to send to the servers (if heartbeat then this daemonizes)")
parser.add_argument("-c", "--callsign", default="all", help="station to query (default all)")
args = parser.parse_args()

while True:

  # retrieve a list of the active stations
  if args.callsign == 'all':
    station_list = db['c'].execute('select * from stations where active = 1')

  else:
    station_list = db['c'].execute('select * from stations where active = 1 and callsign = "%s"' % args.callsign)

  for station in station_list.fetchall():
    url = station[BASE_URL]

    try:
      start = time.time()

      stream = urllib2.urlopen("http://%s/%s" % (url, args.query))
      data = stream.read()

      stop = time.time()

      print url
      print data
      db['c'].execute('update stations set latency = latency + ?, pings = pings + 1, last_seen = current_timestamp where id = ?', ( str(stop - start), str(station[ID]) ))

    except urllib2.HTTPError:
      # Say that we couldn't see this station
      db['c'].execute('update stations set drops = drops + 1 where id = ?', str(station[ID]))

  if args.query == 'heartbeat':
    db['conn'].commit()
    time.sleep(CYCLE_TIME)

  else:
    break

<commit_msg>catching more errors<commit_after>#!/usr/bin/python -O
import argparse
import socket

#
# This is needed to force ipv4 on ipv6 devices. It's sometimes needed
# if there isn't a clean ipv6 route to get to the big wild internet.
# In these cases, a pure ipv6 route simply will not work.  People aren't
# always in full control of every hop ... so it's much safer to force
# ipv4 then optimistically cross our fingers.
#
origGetAddrInfo = socket.getaddrinfo

def getAddrInfoWrapper(host, port, family=0, socktype=0, proto=0, flags=0):
  return origGetAddrInfo(host, port, socket.AF_INET, socktype, proto, flags)

# Replace the original socket.getaddrinfo by our version
socket.getaddrinfo = getAddrInfoWrapper

import urllib2
import sqlite3
import time

conn = sqlite3.connect('../db/main.db')
db = {'conn': conn, 'c': conn.cursor()}

CYCLE_TIME = 60 * 60 * 12

# this trick robbed from http://stackoverflow.com/questions/702834/whats-the-common-practice-for-enums-in-python
ID, CALLSIGN, DESCRIPTION, BASE_URL, LAST_SEEN, FIRST_SEEN, PINGS, DROPS, LATENCY, ACTIVE, LOG, NOTES = range(12)

parser = argparse.ArgumentParser()
parser.add_argument("-q", "--query", default="heartbeat", help="query to send to the servers (if heartbeat then this daemonizes)")
parser.add_argument("-c", "--callsign", default="all", help="station to query (default all)")
args = parser.parse_args()

while True:

  # retrieve a list of the active stations
  if args.callsign == 'all':
    station_list = db['c'].execute('select * from stations where active = 1')

  else:
    station_list = db['c'].execute('select * from stations where active = 1 and callsign = "%s"' % args.callsign)

  for station in station_list.fetchall():
    url = station[BASE_URL]

    try:
      start = time.time()

      stream = urllib2.urlopen("http://%s/%s" % (url, args.query))
      data = stream.read()

      stop = time.time()

      print "---------\n%s\n%s" % (url, data)

      db['c'].execute('update stations set latency = latency + ?, pings = pings + 1, last_seen = current_timestamp where id = ?', ( str(stop - start), str(station[ID]) ))

    except urllib2.URLError:
      db['c'].execute('update stations set drops = drops + 1 where id = ?', str(station[ID]))

    except urllib2.HTTPError:
      # Say that we couldn't see this station
      db['c'].execute('update stations set drops = drops + 1 where id = ?', str(station[ID]))

  if args.query == 'heartbeat':
    db['conn'].commit()
    time.sleep(CYCLE_TIME)

  else:
    break

<|endoftext|>


<commit_before>'''
Attempt to automate steps necessary for uploading stuff
to Google Drive. Based on steps listed on:
https://developers.google.com/drive/quickstart-python

1. Enable the Drive API
  1. Create API project
     https://code.google.com/apis/console/?pli=1
  2. Enable the Drive API in Services
  3. Open API Access tab and create an OAuth 2.0 client ID
  4. Set application name (e.g. "Drive Quickstart Sample")
  5. Client ID Settings section:
    1. Installed application
    2. Other
  5. From the API Access page, you'll need Client ID
     and Client Secret info.

Author: anatoly techtonik <techtonik@gmail.com>
License: Public Domain
'''

from config import CLIENT_ID, CLIENT_SECRET

# ---[ bootstrap section ]---
# check dependencies and fetch them if neeeded
try:
  import httplib2
  import apiclient
except ImportError as exc:
  print('Dependencies are not importable. Bootstrapping locally.')
  speccy = {
    'httplib2': (
      'httplib2-0.8.zip',
      'https://httplib2.googlecode.com/files/httplib2-0.8.zip',
      'bb79080bc94d5e01f8ca3b79ba449a5b169c5e72 159827'),
    'apiclient': (
      'google-api-python-client-1.2.zip',
      'https://google-api-python-client.googlecode.com/files/google-api-python-client-1.2.zip',
      '4e82bbc87e17acf70606ba45b379ea6c5095dd63 97080')
  }
  if 'httplib2' not in exc.message:
    speccy.pop('httplib2')

  # [ ] create .locally if not exists
  # [ ] wgetsecure(.locally, speccy)

# ---[ /bootstrap ]---

# ---[ boilerplate from quickstart.py example ]---
# ---[ /boilerplate ]---

if __name__ == '__main__':
  # [ ] command line usage: drive.py upload <filename>
  pass
<commit_msg>disk.google: Add dependency download code<commit_after>'''
Attempt to automate steps necessary for uploading stuff
to Google Drive. Based on steps listed on:
https://developers.google.com/drive/quickstart-python

1. Enable the Drive API
  1. Create API project
     https://code.google.com/apis/console/?pli=1
  2. Enable the Drive API in Services
  3. Open API Access tab and create an OAuth 2.0 client ID
  4. Set application name (e.g. "Drive Quickstart Sample")
  5. Client ID Settings section:
    1. Installed application
    2. Other
  5. From the API Access page, you'll need Client ID
     and Client Secret info.

Author: anatoly techtonik <techtonik@gmail.com>
License: Public Domain
'''

from config import CLIENT_ID, CLIENT_SECRET

# ---[ bootstrap section ]---
# check dependencies and fetch them if needed

# --[cut locally.py]--
# ---[ download utilities ]---

from hashlib import sha1
import os.path as osp
import urllib
import os

def hashsize(path):
  '''
  Generate SHA-1 hash + file size string for the given
  filename path. Used to check integrity of downloads.
  Resulting string is space separated 'hash size':

    >>> hashsize('locally.py')
    'fbb498a1d3a3a47c8c1ad5425deb46b635fac2eb 2006'
  '''
  size = osp.getsize(path)
  h = sha1()
  with open(path, 'rb') as source:
    while True:
      c = source.read(64*1024)  # read in 64k blocks
      if not c:
        break
      h.update(c)
  return '%s %s' % (h.hexdigest(), size)

class HashSizeCheckFailed(Exception):
  '''
  Exception to throw when downloaded file fails hash
  and size check.
  '''
  pass

def wgetsecure(targetdir, filespec, quiet=False):
  '''
  Using description in `filespec` list, download
  files from specified URL (if they don't exist)
  and check that their size and sha-1 hash matches.

  Files are downloaded into `targetdir`. `filespec`
  is list of tuples (filename, 'hash size', URL).

    filespec = [
     ('wget.py',
      '4eb39538d9e9f360643a0a0b17579f6940196fe4 12262',
      'https://bitbucket.org/techtonik/python-wget/raw/2.0/wget.py'),
    ]

  Raises HashSizeCheckFailed if hash/size check
  fails. Set quiet to false to skip printing
  progress messages.
  '''
  # [-] no rollback
  def check(filepath, shize, quiet):
    if not quiet:
      print('Checking hash/size for %s' % filepath)
    if hashsize(filepath) != shize:
      raise HashSizeCheckFailed(
                'Hash/Size mismatch\n  exp: %s\n  act: %s'
                % (shize, hashsize(filepath)))

  for f, shize, url in filespec:
    filepath = osp.join(targetdir, f)
    # [x] download file if not exists
    if osp.exists(filepath):
      if not quiet:
        print("Downloading " + f + " skipped (already downloaded).")
      # [x] do not remove file if it exists and check failed
      check(filepath, shize, quiet)
    else:
      if not quiet:
        print("Downloading %s into %s" % (f, targetdir))
      urllib.urlretrieve(url, filepath)
      try:
        check(filepath, shize, quiet)
      except HashSizeCheckFailed:
        # [x] remove file if it was downloaded and check failed
        os.remove(filepath)
        raise

# ---[ /download utils ]---
# --[/cut locally.py]--

try:
  import httplib2
  import apiclient
except ImportError as exc:
  # --- bootstrap .locally ---
  #
  # this creates .locally/ subdirectory and sets few global
  # variables for convenience:
  #
  #   ROOT  - absolute path to this dir
  #   LOOT  - absolute path to the .locally/ subdir, added to
  #           to sys.path, always ends with /

  import os
  import sys
  ROOT = os.path.abspath(os.path.dirname(__file__))
  LOOT = os.path.join(ROOT, '.locally/')

  # try to bootstrap from .locally/
  sys.path.insert(0, LOOT)

  try:
    import httplib2
    import apiclient
  except ImportError as exc:
    print('Dependencies are not importable.')
    print('Bootstrapping dependencies locally..')

    if not os.path.exists(LOOT):
      print('..creating subdir')
      os.makedirs(LOOT)

    # downloading files
    speccy = {
      'httplib2': (
        'httplib2-0.8.zip',
        'bb79080bc94d5e01f8ca3b79ba449a5b169c5e72 159827',
        'https://httplib2.googlecode.com/files/httplib2-0.8.zip'),
      'apiclient': (
        'google-api-python-client-1.2.zip',
        '4e82bbc87e17acf70606ba45b379ea6c5095dd63 97080',
        'https://google-api-python-client.googlecode.com/files/google-api-python-client-1.2.zip'),
    }
    if 'httplib2' not in exc.message:
      speccy.pop('httplib2')

    wgetsecure(LOOT, speccy.values())

  # [ ] create .locally if not exists
  # [ ] wgetsecure(.locally, speccy)

# ---[ /bootstrap ]---

# ---[ boilerplate from quickstart.py example ]---
# ---[ /boilerplate ]---

if __name__ == '__main__':
  # [ ] command line usage: drive.py upload <filename>
  pass
<|endoftext|>


<commit_before>#! /usr/bin/python

from docstore import connect as docstore_connect
from installer import install_docstore
from json import loads
from pubsub import connect as pubsub_connect
from sys import argv

def start_game(game_info, docstore_url):
    name, id = game_info['name'], game_info['id']
    sources = ['/games/%s' % (name,),
               '/services/emcee/run_components',
               '/libraries']
    install_docstore('run_components.%s' % (id,), sources,
                     [id, name, docstore_url], docstore_url)

docstore_url = argv[1]
docstore = docstore_connect(docstore_url)
pubsub_url = docstore.wait_and_get('/services/pubsub')
if pubsub_url is None:
    print 'No pubsub data on docstore, exiting'
    exit(1)
pubsub = pubsub_connect(pubsub_url, 'emcee', unmarshal=loads)
pubsub.consume_topic('game.wanted', lambda m: start_game(m, docstore_url))
<commit_msg>emcee uses installer service~<commit_after>#! /usr/bin/python

from docstore import connect as docstore_connect
from json import loads, dumps
from pubsub import connect as pubsub_connect
from sys import argv

def start_game(game_info, docstore_url):
    name, id = game_info['name'], game_info['id']
    sources = ['/games/%s' % (name,),
               '/services/emcee/run_components',
               '/libraries']
    pubsub_url = docstore_connect(docstore_url).get('/services/pubsub')
    pubsub = pubsub_connect(pubsub_url, 'process', marshal=dumps)

    install_message = {'name': 'run_components.%s' % (id,),
                       'sources': sources,
                       'options': [id, name, docstore_url]}
    pubsub.publish('install', install_message)

docstore_url = argv[1]
docstore = docstore_connect(docstore_url)
pubsub_url = docstore.wait_and_get('/services/pubsub')
if pubsub_url is None:
    print 'No pubsub data on docstore, exiting'
    exit(1)
pubsub = pubsub_connect(pubsub_url, 'emcee', unmarshal=loads)
pubsub.consume_topic('game.wanted', lambda m: start_game(m, docstore_url))
<|endoftext|>


<commit_before>import sys
import os
from path import path

from fabric.context_managers import warn_only, quiet, prefix, hide
from fabric.api import env, local, lcd
from fabric.colors import red, yellow, green

# The top-level project directory path
PROJECT_ROOT = None

# The name of the django project module
DJANGO_PROJECT_NAME = None

# The path containing a manage.py file (PROJECT_ROOT / DJANGO_PROJECT_NAME)
SITE_ROOT = None

_env_already_read = None
_package_already_read = None
_django_settings = None
_django_project_module = None

def configured():
    return PROJECT_ROOT is not None

def _wrap_path(possible_path):
    if not isinstance(possible_path, path):
        possible_path = path(possible_path)
    return possible_path

def _require_configured(fn):
    def real_fn(*args, **kwargs):
        if not configured():
            raise Exception("Call configure() first.")
        fn(*args, **kwargs)

    return real_fn

def configure(project_root, django_project_name):
    if configured():
        return

    global PROJECT_ROOT, SITE_ROOT, DJANGO_PROJECT_NAME
    PROJECT_ROOT = _wrap_path(project_root)
    SITE_ROOT = PROJECT_ROOT / django_project_name
    DJANGO_PROJECT_NAME = django_project_name

    sys.path.append(SITE_ROOT)

@_require_configured
def django_module():
    import importlib

    global _django_project_module
    _django_project_module = importlib.import_module(DJANGO_PROJECT_NAME)

@_require_configured
def dot_env():
    """Parse the .env file and return a dictionary"""

    global _env_already_read

    if not _env_already_read:
        from fabutils import env_file

        _env_already_read = env_file.read(PROJECT_ROOT / '.env')

    return _env_already_read

@_require_configured
def package_json():
    """Parse the package.json file"""

    global _package_already_read

    if not _package_already_read:
        import json

        with open(PROJECT_ROOT / 'package.json') as packfile:
            _package_already_read = json.load(packfile)

    return _package_already_read

@_require_configured
def django_settings(default_settings_module='settings'):
    """Return the django settings object."""

    denv = dot_env()

    env.django_settings_module = denv.get('DJANGO_SETTINGS_MODULE', default_settings_module)
    os.environ.setdefault('DJANGO_SETTINGS_MODULE', env.django_settings_module)

    from django.conf import settings

    return settings

def symlink_supported():
    """Return True if the system can sustain symlinks."""
    with quiet():
        result = local('ln -s __linktest__target __linktest__source').succeeded
        local('rm -f __linktest__source __linktest__target')
        return result

@_require_configured
def manage_py(args):
    """Run a manage.py task"""

    with lcd(SITE_ROOT):
        if _wrap_path('manage.py').exists():
            local('python manage.py %s' % args)
            return True
        else:
            print yellow("Django script manage.py doesn't exist")
            return False

@_require_configured
def pip_install(requirements):
    """Install some pip requirements"""

    if isinstance(requirements, basestring):
        requirements = (requirements,)

    pip_path = path(local('which pip', capture=True))
    if not pip_path.exists():
        print pip_path
        print red("Cannot find pip!")
        return False

    use_sudo = False
    if pip_path.get_owner() == 'root':
        use_sudo = True

    print "Installing python requirements..."

    with lcd(PROJECT_ROOT):

        for req in requirements:
            if use_sudo:
                result = local('sudo pip install %s' % req)
            else:
                result = local('pip install %s' % req)

            if not result.succeeded:
                print red("Failed to install %s" % req)
                return False

    return True

@_require_configured
def npm_install():
    """Install npm modules"""

    print "Installing npm modules..."

    with lcd(PROJECT_ROOT):
        if path('package.json').exists():
            if symlink_supported():
                local('ls && pwd')
                local('npm install')
            else:
                print yellow("Symbolic links not supported. Using no-bin-link option.")
                local('npm install --no-bin-link')

            return True

        else:
            print yellow("Node.js package.json doesn't exist")
            return False

@_require_configured
def bower_install():
    """Install bower packages"""
    print "Installing bower packages..."

    with lcd(PROJECT_ROOT):
        if path('bower.json').exists():
            print "Installing bower requirements..."
            local('bower prune --config.interactive=false')
            local('bower install --config.interactive=false')

            return True
        else:
            print yellow("File bower.json doesn't exist")
            return False

def django_render(template_file, output_file, context):
    """Use the Django template engine to render a template with a context dict"""
    from django.template import Template, Context

    from django.conf import settings

    if not settings.configured:
        settings.configure()

    output_file = _wrap_path(output_file)
    template_file = _wrap_path(template_file)

    if output_file.exists():
        local('mv %s %s.bak' % (output_file, output_file))
        print "Backed up %s file as %s.bak" % (output_file, output_file)

    with open(template_file, 'rb') as infile:
        template = Template(infile.read())

        with open(output_file, 'wb') as outfile:
            outfile.write(template.render(Context(context)))

<commit_msg>fix fab issue<commit_after>import sys
import os
from path import path

from fabric.context_managers import warn_only, quiet, prefix, hide
from fabric.api import env, local, lcd
from fabric.colors import red, yellow, green

# The top-level project directory path
PROJECT_ROOT = None

# The name of the django project module
DJANGO_PROJECT_NAME = None

# The path containing a manage.py file (PROJECT_ROOT / DJANGO_PROJECT_NAME)
SITE_ROOT = None

_env_already_read = None
_package_already_read = None
_django_settings = None
_django_project_module = None

def configured():
    return PROJECT_ROOT is not None

def _wrap_path(possible_path):
    if not isinstance(possible_path, path):
        possible_path = path(possible_path)
    return possible_path

def _require_configured(fn):
    def real_fn(*args, **kwargs):
        if not configured():
            raise Exception("Call configure() first.")
        return fn(*args, **kwargs)

    return real_fn

def configure(project_root, django_project_name):
    if configured():
        return

    global PROJECT_ROOT, SITE_ROOT, DJANGO_PROJECT_NAME
    PROJECT_ROOT = _wrap_path(project_root)
    SITE_ROOT = PROJECT_ROOT / django_project_name
    DJANGO_PROJECT_NAME = django_project_name

    sys.path.append(SITE_ROOT)

@_require_configured
def django_module():
    import importlib

    global _django_project_module
    _django_project_module = importlib.import_module(DJANGO_PROJECT_NAME)

@_require_configured
def dot_env():
    """Parse the .env file and return a dictionary"""

    global _env_already_read

    if not _env_already_read:
        from fabutils import env_file

        _env_already_read = env_file.read(PROJECT_ROOT / '.env')

    return _env_already_read

@_require_configured
def package_json():
    """Parse the package.json file"""

    global _package_already_read

    if not _package_already_read:
        import json

        with open(PROJECT_ROOT / 'package.json') as packfile:
            _package_already_read = json.load(packfile)

    return _package_already_read

@_require_configured
def django_settings(default_settings_module='settings'):
    """Return the django settings object."""

    denv = dot_env()

    env.django_settings_module = denv.get('DJANGO_SETTINGS_MODULE', default_settings_module)
    os.environ.setdefault('DJANGO_SETTINGS_MODULE', env.django_settings_module)

    from django.conf import settings

    return settings

def symlink_supported():
    """Return True if the system can sustain symlinks."""
    with quiet():
        result = local('ln -s __linktest__target __linktest__source').succeeded
        local('rm -f __linktest__source __linktest__target')
        return result

@_require_configured
def manage_py(args):
    """Run a manage.py task"""

    with lcd(SITE_ROOT):
        if _wrap_path('manage.py').exists():
            local('python manage.py %s' % args)
            return True
        else:
            print yellow("Django script manage.py doesn't exist")
            return False

@_require_configured
def pip_install(requirements):
    """Install some pip requirements"""

    if isinstance(requirements, basestring):
        requirements = (requirements,)

    pip_path = path(local('which pip', capture=True))
    if not pip_path.exists():
        print pip_path
        print red("Cannot find pip!")
        return False

    use_sudo = False
    if pip_path.get_owner() == 'root':
        use_sudo = True

    print "Installing python requirements..."

    with lcd(PROJECT_ROOT):

        for req in requirements:
            if use_sudo:
                result = local('sudo pip install %s' % req)
            else:
                result = local('pip install %s' % req)

            if not result.succeeded:
                print red("Failed to install %s" % req)
                return False

    return True

@_require_configured
def npm_install():
    """Install npm modules"""

    print "Installing npm modules..."

    with lcd(PROJECT_ROOT):
        if path('package.json').exists():
            if symlink_supported():
                local('ls && pwd')
                local('npm install')
            else:
                print yellow("Symbolic links not supported. Using no-bin-link option.")
                local('npm install --no-bin-link')

            return True

        else:
            print yellow("Node.js package.json doesn't exist")
            return False

@_require_configured
def bower_install():
    """Install bower packages"""
    print "Installing bower packages..."

    with lcd(PROJECT_ROOT):
        if path('bower.json').exists():
            print "Installing bower requirements..."
            local('bower prune --config.interactive=false')
            local('bower install --config.interactive=false')

            return True
        else:
            print yellow("File bower.json doesn't exist")
            return False

def django_render(template_file, output_file, context):
    """Use the Django template engine to render a template with a context dict"""
    from django.template import Template, Context

    from django.conf import settings

    if not settings.configured:
        settings.configure()

    output_file = _wrap_path(output_file)
    template_file = _wrap_path(template_file)

    if output_file.exists():
        local('mv %s %s.bak' % (output_file, output_file))
        print "Backed up %s file as %s.bak" % (output_file, output_file)

    with open(template_file, 'rb') as infile:
        template = Template(infile.read())

        with open(output_file, 'wb') as outfile:
            outfile.write(template.render(Context(context)))

<|endoftext|>


<commit_before>"""
utils
"""
from __future__ import print_function
import sys
import shlex
import subprocess
import os
import io

DEBUG = bool(os.environ.get("SETUPTOOLS_SCM_DEBUG"))

def trace(*k):
    if DEBUG:
        print(*k)
        sys.stdout.flush()

def ensure_stripped_str(str_or_bytes):
    if isinstance(str_or_bytes, str):
        return str_or_bytes.strip()
    else:
        return str_or_bytes.decode('utf-8', 'surogate_escape').strip()

def do_ex(cmd, cwd='.'):
    trace('cmd', repr(cmd))
    p = subprocess.Popen(
        shlex.split(cmd),
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        cwd=cwd,
        env=dict(
            os.environ,
            # disable hgrc processing other than .hg/hgrc
            HGRCPATH='',
            # try to disable i18n
            LC_ALL='C',
            LANGUAGE='',
            HGPLAIN='1',
        )
    )

    out, err = p.communicate()
    if out:
        trace('out', repr(out))
    if err:
        trace('err', repr(err))
    if p.returncode:
        trace('ret', p.returncode)
    return ensure_stripped_str(out), ensure_stripped_str(err), p.returncode

def do(cmd, cwd='.'):
    out, err, ret = do_ex(cmd, cwd)
    if ret:
        trace('ret', ret)
        print(err)
    return out

def data_from_mime(path):
    with io.open(path, encoding='utf-8') as fp:
        content = fp.read()
    trace('content', repr(content))
    # the complex conditions come from reading pseudo-mime-messages
    data = dict(
        x.split(': ', 1)
        for x in content.splitlines()
        if ': ' in x)
    trace('data', data)
    return data
<commit_msg>Use unicode literals in utils.py<commit_after>"""
utils
"""
from __future__ import print_function, unicode_literals
import sys
import shlex
import subprocess
import os
import io

DEBUG = bool(os.environ.get("SETUPTOOLS_SCM_DEBUG"))

def trace(*k):
    if DEBUG:
        print(*k)
        sys.stdout.flush()

def ensure_stripped_str(str_or_bytes):
    if isinstance(str_or_bytes, str):
        return str_or_bytes.strip()
    else:
        return str_or_bytes.decode('utf-8', 'surogate_escape').strip()

def do_ex(cmd, cwd='.'):
    trace('cmd', repr(cmd))
    p = subprocess.Popen(
        shlex.split(cmd),
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        cwd=cwd,
        env=dict(
            os.environ,
            # disable hgrc processing other than .hg/hgrc
            HGRCPATH='',
            # try to disable i18n
            LC_ALL='C',
            LANGUAGE='',
            HGPLAIN='1',
        )
    )

    out, err = p.communicate()
    if out:
        trace('out', repr(out))
    if err:
        trace('err', repr(err))
    if p.returncode:
        trace('ret', p.returncode)
    return ensure_stripped_str(out), ensure_stripped_str(err), p.returncode

def do(cmd, cwd='.'):
    out, err, ret = do_ex(cmd, cwd)
    if ret:
        trace('ret', ret)
        print(err)
    return out

def data_from_mime(path):
    with io.open(path, encoding='utf-8') as fp:
        content = fp.read()
    trace('content', repr(content))
    # the complex conditions come from reading pseudo-mime-messages
    data = dict(
        x.split(': ', 1)
        for x in content.splitlines()
        if ': ' in x)
    trace('data', data)
    return data
<|endoftext|>


<commit_before>## Please use make index to update this file
all = """
CreateSliver
DeleteSliver
GetCredential
GetGids
GetSelfCredential
GetTicket
GetVersion
List
ListResources
ListSlices
RedeemTicket
Register
RegisterPeerObject
Remove
RemovePeerObject
RenewSliver
Resolve
ResolveGENI
Shutdown
SliverStatus
Start
Stop
Update
UpdateSliver
get_aggregates
get_key
get_registries
get_trusted_certs
register_peer_object
remove_peer_object
reset_slice
""".split()
<commit_msg>Added CreateGid() method to Registry interface<commit_after>## Please use make index to update this file
all = """
CreateSliver
CreateGid
DeleteSliver
GetCredential
GetGids
GetSelfCredential
GetTicket
GetVersion
List
ListResources
ListSlices
RedeemTicket
Register
RegisterPeerObject
Remove
RemovePeerObject
RenewSliver
Resolve
ResolveGENI
Shutdown
SliverStatus
Start
Stop
Update
UpdateSliver
get_aggregates
get_key
get_registries
get_trusted_certs
register_peer_object
remove_peer_object
reset_slice
""".split()
<|endoftext|>


<commit_before># * require certificate as an argument
# * lookup gid in db
# * get pubkey from gid
# * if certifacate matches pubkey from gid, return gid, else raise exception
#  if not peer.is_pubkey(gid.get_pubkey()):
#            raise ConnectionKeyGIDMismatch(gid.get_subject())

from sfa.util.faults import *
from sfa.util.misc import *
from sfa.util.method import Method
from sfa.util.parameter import Parameter, Mixed
from sfa.trust.auth import Auth
from sfa.trust.gid import GID
from sfa.trust.certificate import Certificate
from sfa.util.genitable import GeniTable

class get_gids(Method):
    """
    Get a list of record information (hrn, gid and type) for
    the specified hrns.

    @param cred credential string
    @param cert certificate string
    @return
    """

    interfaces = ['registry']

    accepts = [
        Parameter(str, "Certificate string"),
        Mixed(Paramter(str, "Human readable name (hrn)"),
              Parameter([str], "List of Human readable names (hrn)")),
        Mixed(Parameter(str, "Request hash"),
              Parameter(None, "Request hash not specified"))
        ]

    returns = [Parameter(dict, "Dictionary of gids keyed on hrn")]

    def call(self, cred, hrns, request_hash=None):
        self.api.auth.authenticateCred(cred, [cred, hrns], request_hash)
        self.api.auth.check(cred, 'getgids')
        table = GeniTable()
        records = table.find({'hrn': [hrns]}, columns=['hrn','type','gid'])

        return records
<commit_msg>fix spelling error<commit_after># * require certificate as an argument
# * lookup gid in db
# * get pubkey from gid
# * if certifacate matches pubkey from gid, return gid, else raise exception
#  if not peer.is_pubkey(gid.get_pubkey()):
#            raise ConnectionKeyGIDMismatch(gid.get_subject())

from sfa.util.faults import *
from sfa.util.misc import *
from sfa.util.method import Method
from sfa.util.parameter import Parameter, Mixed
from sfa.trust.auth import Auth
from sfa.trust.gid import GID
from sfa.trust.certificate import Certificate
from sfa.util.genitable import GeniTable

class get_gids(Method):
    """
    Get a list of record information (hrn, gid and type) for
    the specified hrns.

    @param cred credential string
    @param cert certificate string
    @return
    """

    interfaces = ['registry']

    accepts = [
        Parameter(str, "Certificate string"),
        Mixed(Parameter(str, "Human readable name (hrn)"),
              Parameter([str], "List of Human readable names (hrn)")),
        Mixed(Parameter(str, "Request hash"),
              Parameter(None, "Request hash not specified"))
        ]

    returns = [Parameter(dict, "Dictionary of gids keyed on hrn")]

    def call(self, cred, hrns, request_hash=None):
        self.api.auth.authenticateCred(cred, [cred, hrns], request_hash)
        self.api.auth.check(cred, 'getgids')
        table = GeniTable()
        records = table.find({'hrn': [hrns]}, columns=['hrn','type','gid'])

        return records
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8
"""
Code for fitting individual components of the lens+source system

"""

<commit_msg>update component scripts<commit_after>#!/usr/bin/env python
# encoding: utf-8
"""
Code for fitting individual components of the lens+source system

"""

# The ID of the *lens* in my segmentation file, could be different in
# yours
LENS_SEG_ID = 253

def extract_beams():
    """
    Get beam cutouts from the grism exposures
    """
    import glob
    import drizzlepac # to avoid memory issues with GroupFLT...
    import numpy as np

    import grizli
    import grizli.prep

    # Find the grism exposures
    files=glob.glob('*_flt.fits')
    info = grizli.utils.get_flt_info(files)
    grism_files = list(info['FILE'][info['FILTER'] == 'G141'])
    np.save('all_grism_files.npy', [grism_files])

    # Initialize GroupFLT object
    # This should read GrismFLT files and static contam model
    # you generated in grism.py
    grp = grizli.multifit.GroupFLT(grism_files=grism_files,
                                   direct_files=[],
                                 ref_file='sdssj0851+3331-f160w_drz_sci.fits',
                                   seg_file='sdssj0851+3331-f160w_seg.fits',
                                   catalog='sdssj0851+3331-f160w.cat',
                                   cpu_count=4)

    beams = grp.get_beams(LENS_SEG_ID, size=80, beam_id='A')

    mb = grizli.multifit.MultiBeam(beams, fcontam=0., group_name='sgas-lens')

    # Save them to a file for quick I/O
    mb.write_beam_fits()

    # id661acmq_flt_00253.g141.A.fits
    # id661acnq_flt_00253.g141.A.fits
    # id661bcuq_flt_00253.g141.A.fits
    # id661bcvq_flt_00253.g141.A.fits
    # etc.

def load_beams():
    """
    Load the beams from the saved files
    """
    import glob
    import grizli

    beam_files = glob.glob('*{0}.g141.A.fits'.format(LENS_SEG_ID))
    # Load from FITS files
    mb = grizli.multifit.MultiBeam(beam_files, fcontam=0., group_name='sgas-lens')

    return mb

def make_component_segments(mb, region_file='sgas-lens-components.reg', force_recompute=False):
    """
    Make custom segmentation file for each beam in the `mb` list.

    The `region_file` file is in the sgas-lens/sgas/data repository
    """
    import os
    import numpy as np

    import astropy.io.fits as pyfits
    import astropy.wcs as pywcs
    import pyregion

    import sgas

    # Load the segment file if already exists and if not forced to recompute
    if os.path.exists('component_segments.npy') & (not force_recompute):
        component_segments, old_region_file = np.load('component_segments.npy')
        if (old_region_file == region_file):
            for ib, beam in enumerate(mb.beams):
                beam.beam.seg = component_segments[ib]

            return mb

    # Read the regions file
    regions = pyregion.open(os.path.join(sgas.get_data_path(), region_file))
    Nreg = len(regions)

    component_segments = []

    # Loop through beams, make segments for each one of them
    for beam in mb.beams:
        regs = regions.as_imagecoord(beam.direct.header)

        #initialize a new segmentation array
        seg = beam.beam.seg*0

        # update with region segments
        for i in range(Nreg):
            mask_i = regs[i:i+1].get_mask(header=beam.direct.header, shape=beam.beam.sh)

            # update nonzero segment pixels
            clip = seg == 0
            seg[clip] += mask_i[clip]*(i+1)

        beam.beam.seg = seg
        component_segments.append(seg)

    np.save('component_segments.npy', [component_segments, region_file])
    return mb

def test():
    # SHow an example
    model = mb.beams[0].beam.model*0.
    for i in range(6):
        set_id(mb, id=i+1)
        mb.compute_model()
        model += mb.beams[0].beam.model
        ds9.view(model)

def set_id(mb, id=1):
    # Set the id on the MultiBeam object for a given component
    mb.id = id
    for beam in mb.beams:
        beam.id = id
        beam.beam.id = id

        ### Initialize for fits
        beam.flat_flam = beam.compute_model(in_place=False)

        ### OK data where the 2D model has non-zero flux
        beam.fit_mask = (~beam.mask.flatten()) & (beam.ivar.flatten() != 0)
        beam.fit_mask &= (beam.flat_flam > 0.01*beam.flat_flam.max())

        try:
            delattr(beam, 'optimal_profile')
        except:
            pass

    mb.flat_flam = np.hstack([b.flat_flam for b in mb.beams])
    mb.fit_mask = np.hstack([b.fit_mask for b in mb.beams])
    mb.DoF = mb.fit_mask.sum()

    <|endoftext|>