<commit_before># encoding: utf-8
# ---------------------------------------------------------------------------
#  Copyright (C) 2008-2014, IPython Development Team and Enthought, Inc.
#  Distributed under the terms of the BSD License.  See COPYING.rst.
# ---------------------------------------------------------------------------

"""
Plotting functions for distarrays.
"""

from matplotlib import pyplot, colors, cm
from numpy import arange, concatenate, empty, linspace, resize

from distarray.externals.six.moves import range
from distarray.decorators import local

@local
def _get_ranks(arr):
    """
    Given a distarray arr, return a distarray with the same shape, but
    with the elements equal to the rank of the process the element is
    on.
    """
    out = arr.copy()
    out.ndarray[:] = arr.comm_rank
    out.ndarray= out.ndarray.astype(int)
    return out

def cmap_discretize(cmap, N):
    """Create a discrete colormap from the continuous colormap cmap.

    Parameters
    ----------
    cmap : colormap instance, or string
        The continuous colormap, as object or name, to make discrete.
        For example, matplotlib.cm.jet, or 'jet'.
    N : int
        The number of discrete colors desired.

    Returns
    -------
    colormap
        The desired discrete colormap.

    Example
    -------
    >>> x = resize(arange(100), (5,100))
    >>> djet = cmap_discretize(cm.jet, 5)
    >>> pyplot.imshow(x, cmap=djet)
    """
    # This is copied from:
    # http://wiki.scipy.org/Cookbook/Matplotlib/ColormapTransformations
    if type(cmap) == str:
        cmap = cm.get_cmap(cmap)
    colors_i = concatenate((linspace(0, 1., N), (0., 0., 0., 0.)))
    colors_rgba = cmap(colors_i)
    indices = linspace(0, 1., N + 1)
    cdict = {}
    for ki, key in enumerate(('red', 'green', 'blue')):
        cdict[key] = [
            (indices[i], colors_rgba[i - 1, ki], colors_rgba[i, ki])
            for i in range(N + 1)
        ]
    # Return colormap object.
    return colors.LinearSegmentedColormap(cmap.name + "_%d" % N, cdict, 1024)

def create_discrete_colormaps(num_values):
    """ Create colormap objects for a discrete colormap.

    Parameters
    ----------
    num_values : The number of distinct colors to use.

    Returns
    -------
    cmap, norm, text_colors : tuple
        The matplotlib colormap, norm, and recommended text colors.
        text_colors is an array of length num_values,
        with each entry being a nice color for text drawn
        on top of the colormap selection.
    """
    # Create discrete colormap for matplotlib.
    cmap = cmap_discretize(cm.jet, num_values)
    bounds = range(num_values + 1)
    norm = colors.BoundaryNorm(bounds, cmap.N)

    # Choose a text color for each discrete color.
    # The idea is to pick black for colors near white.
    # This is not sophisticated but ok for this use.
    text_colors = []
    for j in range(num_values):
        # Get rgb color that matshow() will use.
        jj = float(j + 0.5) / float(num_values)
        cj = cmap(jj)
        # Get average of rgb values.
        avg = (cj[0] + cj[1] + cj[2]) / 3.0
        # With 4-color jet, avg cyan=0.6111, yellow=0.6337.
        # Choose empirically reasonable cutoff.
        if avg >= 0.5:
            text_color = 'black'
        else:
            text_color = 'white'
        text_colors.append(text_color)

    # Return a tuple with all the parts.
    colormaps = (cmap, norm, text_colors)
    return colormaps

def plot_local_array_subfigure(subfig,
                               local_array,
                               process,
                               coord,
                               colormap_objects,
                               *args, **kwargs):
    """ Plot a single local_array into a matplotlib subfigure. """
    title = 'Process %r' % (coord,)
    subfig.set_title(title, fontsize=10)

    # Coerce to 2D if needed.
    if len(local_array.shape) == 1:
        local_array.shape = (1, local_array.shape[0])

    # Fill array with the process number.
    # (Then it will color the same as in the global plot.)
    shape = local_array.shape
    plot_array = empty(shape, dtype=int)
    plot_array.fill(process)

    cmap, norm, text_colors = colormap_objects
    text_color = text_colors[process]

    # I tried to adjust the size of the subplots carefully, with
    # the idea that the size should be proportional to the local array
    # size, but I was not able to work that out.
    # So this makes all the plots the same size which at least
    # does not look too strange.
    extent = [-0.5, shape[1] - 0.5, -0.5, shape[0] - 0.5]
    subfig.imshow(plot_array,
                  extent=extent,
                  interpolation='nearest',
                  aspect='auto',
                  cmap=cmap, norm=norm,
                  *args, **kwargs)

    # Note that y limits are flipped to get the first row
    # of the arrays at the top of the plot.
    subfig.set_xlim(0 - 0.5, shape[1] - 0.5)
    subfig.set_ylim(shape[0] - 0.5, 0 - 0.5)

    # Configure a grid but otherwise hide the tickmarks.
    x_ticks = [i - 0.5 for i in range(shape[1] + 1)]
    y_ticks = [i - 0.5 for i in range(shape[0] + 1)]
    subfig.xaxis.set_ticks(x_ticks)
    subfig.yaxis.set_ticks(y_ticks)
    subfig.grid(True, linestyle='-', color=text_color)
    all_ticks = []
    all_ticks.extend(subfig.xaxis.iter_ticks())
    all_ticks.extend(subfig.yaxis.iter_ticks())
    for tick in all_ticks:
        tick[0].label1On = False
        tick[0].label2On = False
        tick[0].tick1On = False
        tick[0].tick2On = False

    # Label each cell.
    for row in range(shape[0]):
        for col in range(shape[1]):
            value = local_array[row, col]
            label = '%d' % (value)
            subfig.text(
                col, row, label,
                horizontalalignment='center',
                verticalalignment='center',
                color=text_color)

def plot_local_arrays(darray,
                      process_coords,
                      colormap_objects,
                      filename):
    """ Plot the local arrays as a multi-figure matplotlib plot. """
    # Get the local arrays that are not empty.
    ndarrays = darray.get_ndarrays()
    local_arrays = []
    for processor, local_array in enumerate(ndarrays):
        processor_coord = process_coords[processor]
        if local_array.size > 0:
            local_arrays.append((processor, processor_coord, local_array))

    pyplot.clf()

    num_local_arrays = len(local_arrays)
    if (num_local_arrays % 2) == 0:
        # 2 X N grid
        subplot_grid = (2, num_local_arrays // 2)
    else:
        # N x 1 grid
        subplot_grid = (num_local_arrays, 1)

    _, subfigs = pyplot.subplots(*subplot_grid)
    for i, (process, coord, local_array) in enumerate(local_arrays):
        if subplot_grid[1] > 1:
            N = subplot_grid[1]
            row, col = i // N, i % N
            subfig = subfigs[row, col]
        else:
            subfig = subfigs[i]
        plot_local_array_subfigure(subfig,
                                   local_array,
                                   process,
                                   coord,
                                   colormap_objects)

    # Add main title and adjust size.
    figure = pyplot.gcf()
    figure.suptitle('Local Arrays', fontsize=14)
    figure.set_size_inches(10.0, 5.0)

    if filename is not None:
        pyplot.savefig(filename, dpi=100)

def plot_array_distribution(darray,
                            process_coords,
                            title=None,
                            xlabel=None,
                            ylabel=None,
                            yflip=False,
                            cell_label=True,
                            legend=False,
                            global_plot_filename=None,
                            local_plot_filename=None,
                            *args, **kwargs):
    """
    Plot a distarray's memory layout. It can be 1D or 2D.
    Elements are colored according to the process they are on.

    Parameters
    ----------
    darray : DistArray
        The distributed array to plot.
    process_coords : List of tuples.
        The process grid coordinates.
    title : string
        Text label for the plot title, or None.
    xlabel : string
        Text label for the x-axis, or None.
    ylabel : string
        Text label for the y-axis, or None.
    yflip : bool
        If True, then the y-axis increases downwards, to match the layout
        when printing the array itself.
    cell_label : bool
        If True, then each cell in the plot is labeled with the array value.
        This can look cluttered for large arrays.
    legend : bool
        If True, then a colorbar legend is drawn to label the colors.
    global_plot_filename : string
        Output filename for the global array plot image.
    local_plot_filename : string
        Output filename for the local array plot image.

    Returns
    -------
    out
        The process assignment array, as a DistArray.
    """
    # This is based somewhat on:
    #   http://matplotlib.org/examples/api/colorbar_only.html

    # Process per element.
    process_darray = _get_ranks(darray)
    process_array = process_darray.toarray()

    # Values per element.
    values_array = darray.toarray()

    # Coerce to 2D if needed.
    if len(process_array.shape) == 1:
        process_array.shape = (1, process_array.shape[0])
        values_array.shape = process_array.shape

    # Create discrete colormap.
    num_processors = process_array.max() + 1
    colormap_objects = create_discrete_colormaps(num_processors)
    cmap, norm, text_colors = colormap_objects

    # Plot the array.
    img = pyplot.matshow(process_array, cmap=cmap, norm=norm, *args, **kwargs)

    # Add title and labels.
    if title is not None:
        pyplot.title(title)
    if xlabel is not None:
        pyplot.xlabel(xlabel)
    if ylabel is not None:
        pyplot.ylabel(ylabel)

    # Either invert y-axis, and put tick labels at bottom,
    # or put the x-axis label at the top.
    axis = pyplot.gca()
    if yflip:
        axis.invert_yaxis()
        for tick in axis.xaxis.iter_ticks():
            tick[0].label1On = True
            tick[0].label2On = False
    else:
        axis.xaxis.set_label_position('top')

    # Label each cell.
    if cell_label:
        for row in range(values_array.shape[0]):
            for col in range(values_array.shape[1]):
                process = process_array[row, col]
                value = values_array[row, col]
                label = '%d' % (value)
                color = text_colors[process]
                pyplot.text(
                    col, row, label,
                    horizontalalignment='center',
                    verticalalignment='center',
                    color=color)

    # Add colorbar legend.
    if legend:
        cbar = pyplot.colorbar(img)
        cbar_ticks = [0.5 + p for p in range(num_processors)]
        cbar_labels = [str(p) for p in range(num_processors)]
        cbar.set_ticks(cbar_ticks)
        cbar.set_ticklabels(cbar_labels)
        cbar.set_label('Processor')

    # Adjust size.
    figure = pyplot.gcf()
    figure.set_size_inches(10.0, 5.0)

    # Save to output file.
    if global_plot_filename is not None:
        pyplot.savefig(global_plot_filename, dpi=100)

    # Make similar plots for the local arrays...
    if local_plot_filename is not None:
        plot_local_arrays(darray,
                          process_coords,
                          colormap_objects,
                          local_plot_filename)

    return process_darray
<commit_msg>Fix import in plotting.py<commit_after># encoding: utf-8
# ---------------------------------------------------------------------------
#  Copyright (C) 2008-2014, IPython Development Team and Enthought, Inc.
#  Distributed under the terms of the BSD License.  See COPYING.rst.
# ---------------------------------------------------------------------------

"""
Plotting functions for distarrays.
"""

from matplotlib import pyplot, colors, cm
from numpy import arange, concatenate, empty, linspace, resize

from distarray.externals.six.moves import range
from distarray.dist.decorators import local

@local
def _get_ranks(arr):
    """
    Given a distarray arr, return a distarray with the same shape, but
    with the elements equal to the rank of the process the element is
    on.
    """
    out = arr.copy()
    out.ndarray[:] = arr.comm_rank
    out.ndarray= out.ndarray.astype(int)
    return out

def cmap_discretize(cmap, N):
    """Create a discrete colormap from the continuous colormap cmap.

    Parameters
    ----------
    cmap : colormap instance, or string
        The continuous colormap, as object or name, to make discrete.
        For example, matplotlib.cm.jet, or 'jet'.
    N : int
        The number of discrete colors desired.

    Returns
    -------
    colormap
        The desired discrete colormap.

    Example
    -------
    >>> x = resize(arange(100), (5,100))
    >>> djet = cmap_discretize(cm.jet, 5)
    >>> pyplot.imshow(x, cmap=djet)
    """
    # This is copied from:
    # http://wiki.scipy.org/Cookbook/Matplotlib/ColormapTransformations
    if type(cmap) == str:
        cmap = cm.get_cmap(cmap)
    colors_i = concatenate((linspace(0, 1., N), (0., 0., 0., 0.)))
    colors_rgba = cmap(colors_i)
    indices = linspace(0, 1., N + 1)
    cdict = {}
    for ki, key in enumerate(('red', 'green', 'blue')):
        cdict[key] = [
            (indices[i], colors_rgba[i - 1, ki], colors_rgba[i, ki])
            for i in range(N + 1)
        ]
    # Return colormap object.
    return colors.LinearSegmentedColormap(cmap.name + "_%d" % N, cdict, 1024)

def create_discrete_colormaps(num_values):
    """ Create colormap objects for a discrete colormap.

    Parameters
    ----------
    num_values : The number of distinct colors to use.

    Returns
    -------
    cmap, norm, text_colors : tuple
        The matplotlib colormap, norm, and recommended text colors.
        text_colors is an array of length num_values,
        with each entry being a nice color for text drawn
        on top of the colormap selection.
    """
    # Create discrete colormap for matplotlib.
    cmap = cmap_discretize(cm.jet, num_values)
    bounds = range(num_values + 1)
    norm = colors.BoundaryNorm(bounds, cmap.N)

    # Choose a text color for each discrete color.
    # The idea is to pick black for colors near white.
    # This is not sophisticated but ok for this use.
    text_colors = []
    for j in range(num_values):
        # Get rgb color that matshow() will use.
        jj = float(j + 0.5) / float(num_values)
        cj = cmap(jj)
        # Get average of rgb values.
        avg = (cj[0] + cj[1] + cj[2]) / 3.0
        # With 4-color jet, avg cyan=0.6111, yellow=0.6337.
        # Choose empirically reasonable cutoff.
        if avg >= 0.5:
            text_color = 'black'
        else:
            text_color = 'white'
        text_colors.append(text_color)

    # Return a tuple with all the parts.
    colormaps = (cmap, norm, text_colors)
    return colormaps

def plot_local_array_subfigure(subfig,
                               local_array,
                               process,
                               coord,
                               colormap_objects,
                               *args, **kwargs):
    """ Plot a single local_array into a matplotlib subfigure. """
    title = 'Process %r' % (coord,)
    subfig.set_title(title, fontsize=10)

    # Coerce to 2D if needed.
    if len(local_array.shape) == 1:
        local_array.shape = (1, local_array.shape[0])

    # Fill array with the process number.
    # (Then it will color the same as in the global plot.)
    shape = local_array.shape
    plot_array = empty(shape, dtype=int)
    plot_array.fill(process)

    cmap, norm, text_colors = colormap_objects
    text_color = text_colors[process]

    # I tried to adjust the size of the subplots carefully, with
    # the idea that the size should be proportional to the local array
    # size, but I was not able to work that out.
    # So this makes all the plots the same size which at least
    # does not look too strange.
    extent = [-0.5, shape[1] - 0.5, -0.5, shape[0] - 0.5]
    subfig.imshow(plot_array,
                  extent=extent,
                  interpolation='nearest',
                  aspect='auto',
                  cmap=cmap, norm=norm,
                  *args, **kwargs)

    # Note that y limits are flipped to get the first row
    # of the arrays at the top of the plot.
    subfig.set_xlim(0 - 0.5, shape[1] - 0.5)
    subfig.set_ylim(shape[0] - 0.5, 0 - 0.5)

    # Configure a grid but otherwise hide the tickmarks.
    x_ticks = [i - 0.5 for i in range(shape[1] + 1)]
    y_ticks = [i - 0.5 for i in range(shape[0] + 1)]
    subfig.xaxis.set_ticks(x_ticks)
    subfig.yaxis.set_ticks(y_ticks)
    subfig.grid(True, linestyle='-', color=text_color)
    all_ticks = []
    all_ticks.extend(subfig.xaxis.iter_ticks())
    all_ticks.extend(subfig.yaxis.iter_ticks())
    for tick in all_ticks:
        tick[0].label1On = False
        tick[0].label2On = False
        tick[0].tick1On = False
        tick[0].tick2On = False

    # Label each cell.
    for row in range(shape[0]):
        for col in range(shape[1]):
            value = local_array[row, col]
            label = '%d' % (value)
            subfig.text(
                col, row, label,
                horizontalalignment='center',
                verticalalignment='center',
                color=text_color)

def plot_local_arrays(darray,
                      process_coords,
                      colormap_objects,
                      filename):
    """ Plot the local arrays as a multi-figure matplotlib plot. """
    # Get the local arrays that are not empty.
    ndarrays = darray.get_ndarrays()
    local_arrays = []
    for processor, local_array in enumerate(ndarrays):
        processor_coord = process_coords[processor]
        if local_array.size > 0:
            local_arrays.append((processor, processor_coord, local_array))

    pyplot.clf()

    num_local_arrays = len(local_arrays)
    if (num_local_arrays % 2) == 0:
        # 2 X N grid
        subplot_grid = (2, num_local_arrays // 2)
    else:
        # N x 1 grid
        subplot_grid = (num_local_arrays, 1)

    _, subfigs = pyplot.subplots(*subplot_grid)
    for i, (process, coord, local_array) in enumerate(local_arrays):
        if subplot_grid[1] > 1:
            N = subplot_grid[1]
            row, col = i // N, i % N
            subfig = subfigs[row, col]
        else:
            subfig = subfigs[i]
        plot_local_array_subfigure(subfig,
                                   local_array,
                                   process,
                                   coord,
                                   colormap_objects)

    # Add main title and adjust size.
    figure = pyplot.gcf()
    figure.suptitle('Local Arrays', fontsize=14)
    figure.set_size_inches(10.0, 5.0)

    if filename is not None:
        pyplot.savefig(filename, dpi=100)

def plot_array_distribution(darray,
                            process_coords,
                            title=None,
                            xlabel=None,
                            ylabel=None,
                            yflip=False,
                            cell_label=True,
                            legend=False,
                            global_plot_filename=None,
                            local_plot_filename=None,
                            *args, **kwargs):
    """
    Plot a distarray's memory layout. It can be 1D or 2D.
    Elements are colored according to the process they are on.

    Parameters
    ----------
    darray : DistArray
        The distributed array to plot.
    process_coords : List of tuples.
        The process grid coordinates.
    title : string
        Text label for the plot title, or None.
    xlabel : string
        Text label for the x-axis, or None.
    ylabel : string
        Text label for the y-axis, or None.
    yflip : bool
        If True, then the y-axis increases downwards, to match the layout
        when printing the array itself.
    cell_label : bool
        If True, then each cell in the plot is labeled with the array value.
        This can look cluttered for large arrays.
    legend : bool
        If True, then a colorbar legend is drawn to label the colors.
    global_plot_filename : string
        Output filename for the global array plot image.
    local_plot_filename : string
        Output filename for the local array plot image.

    Returns
    -------
    out
        The process assignment array, as a DistArray.
    """
    # This is based somewhat on:
    #   http://matplotlib.org/examples/api/colorbar_only.html

    # Process per element.
    process_darray = _get_ranks(darray)
    process_array = process_darray.toarray()

    # Values per element.
    values_array = darray.toarray()

    # Coerce to 2D if needed.
    if len(process_array.shape) == 1:
        process_array.shape = (1, process_array.shape[0])
        values_array.shape = process_array.shape

    # Create discrete colormap.
    num_processors = process_array.max() + 1
    colormap_objects = create_discrete_colormaps(num_processors)
    cmap, norm, text_colors = colormap_objects

    # Plot the array.
    img = pyplot.matshow(process_array, cmap=cmap, norm=norm, *args, **kwargs)

    # Add title and labels.
    if title is not None:
        pyplot.title(title)
    if xlabel is not None:
        pyplot.xlabel(xlabel)
    if ylabel is not None:
        pyplot.ylabel(ylabel)

    # Either invert y-axis, and put tick labels at bottom,
    # or put the x-axis label at the top.
    axis = pyplot.gca()
    if yflip:
        axis.invert_yaxis()
        for tick in axis.xaxis.iter_ticks():
            tick[0].label1On = True
            tick[0].label2On = False
    else:
        axis.xaxis.set_label_position('top')

    # Label each cell.
    if cell_label:
        for row in range(values_array.shape[0]):
            for col in range(values_array.shape[1]):
                process = process_array[row, col]
                value = values_array[row, col]
                label = '%d' % (value)
                color = text_colors[process]
                pyplot.text(
                    col, row, label,
                    horizontalalignment='center',
                    verticalalignment='center',
                    color=color)

    # Add colorbar legend.
    if legend:
        cbar = pyplot.colorbar(img)
        cbar_ticks = [0.5 + p for p in range(num_processors)]
        cbar_labels = [str(p) for p in range(num_processors)]
        cbar.set_ticks(cbar_ticks)
        cbar.set_ticklabels(cbar_labels)
        cbar.set_label('Processor')

    # Adjust size.
    figure = pyplot.gcf()
    figure.set_size_inches(10.0, 5.0)

    # Save to output file.
    if global_plot_filename is not None:
        pyplot.savefig(global_plot_filename, dpi=100)

    # Make similar plots for the local arrays...
    if local_plot_filename is not None:
        plot_local_arrays(darray,
                          process_coords,
                          colormap_objects,
                          local_plot_filename)

    return process_darray
<|endoftext|>


<commit_before>import unittest
import numpy as np
from IPython.parallel import Client
from distarray.client import DistArrayContext

class TestDistArrayContext(unittest.TestCase):

    def setUp(self):
        self.client = Client()
        self.dv = self.client[:]

    def test_create_DAC(self):
        '''Can we create a plain vanilla context?'''
        dac = DistArrayContext(self.dv)
        self.assertIs(dac.view, self.dv)

    def test_create_DAC_with_targets(self):
        '''Can we create a context with a subset of engines?'''
        dac = DistArrayContext(self.dv, targets=[0, 1])
        self.assertIs(dac.view, self.dv)

class TestDistArrayProxy(unittest.TestCase):

    def setUp(self):
        self.client = Client()
        self.dv = self.client[:]
        self.dac = DistArrayContext(self.dv)

        self.dap = self.dac.fromndarray(np.arange(100))

    def test_getitem(self):
        self.assertEqual(self.dap[55], 55)

if __name__ == '__main__':
    unittest.main(verbosity=2)
<commit_msg>Add failing test for DistArrayProxy.__setitem__<commit_after>import unittest
import numpy as np
from IPython.parallel import Client
from distarray.client import DistArrayContext

class TestDistArrayContext(unittest.TestCase):

    def setUp(self):
        self.client = Client()
        self.dv = self.client[:]

    def test_create_DAC(self):
        '''Can we create a plain vanilla context?'''
        dac = DistArrayContext(self.dv)
        self.assertIs(dac.view, self.dv)

    def test_create_DAC_with_targets(self):
        '''Can we create a context with a subset of engines?'''
        dac = DistArrayContext(self.dv, targets=[0, 1])
        self.assertIs(dac.view, self.dv)

class TestDistArrayProxy(unittest.TestCase):

    def setUp(self):
        self.client = Client()
        self.dv = self.client[:]
        self.dac = DistArrayContext(self.dv)

        self.dap = self.dac.fromndarray(np.arange(100))

    def test_getitem(self):
        self.assertEqual(self.dap[55], 55)

    def test_setitem(self):
        self.dap[35] = 9999
        print self.dap[35]

if __name__ == '__main__':
    unittest.main(verbosity=2)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# coding: utf-8

from django.core.urlresolvers import reverse, NoReverseMatch
from django.utils.safestring import mark_safe
from table.utils import Accessor
from .base import Column

class LinkColumn(Column):
    def __init__(self, links, delimiter=u' ', *args, **kwargs):
        super(LinkColumn, self).__init__(*args, **kwargs)
        self.links = links
        self.delimiter = delimiter
        self.searchable = False
        self.safe = False

    def render(self, obj):
        return self.delimiter.join([link.render(obj) for link in self.links])

class Link(object):
    """ Represents a link element in html.
    """
    def __init__(self, text, viewname, args=None, kwargs=None, urlconf=None,
                 current_app=None):
        self.text = text
        self.viewname = viewname
        self.args = args or []
        self.kwargs = kwargs or {}
        self.urlconf = urlconf
        self.current_app = current_app

    def resolve(self, obj):
        """ Resolving URL paths to the corresponding object. See:
            https://docs.djangoproject.com/en/dev/ref/urlresolvers/#reverse
        """
        viewname = self.viewname
        params = {}

        if self.args:
            params['args'] = [getattr(obj, attr) for attr in self.args]
        if self.kwargs:
            params['kwargs'] = {}
            for key, value in self.kwargs.itmes:
                params['kwargs'][key] = getattr(obj, value)
        if self.urlconf:
            params['urlconf'] = self.urlconf
        if self.current_app:
            params['current_app'] = self.current_app

        try:
            url = reverse(viewname, **params)
        except NoReverseMatch:
            url = ''
        return url

    def render(self, obj):
        """ Render link as HTML output tag .
        """
        text = self.text.resolve(obj) if isinstance(self.text, Accessor) else self.text
        return mark_safe(u'%s' % (self.resolve(obj), text))

<commit_msg>onclick option<commit_after>#!/usr/bin/env python
# coding: utf-8

from django.core.urlresolvers import reverse, NoReverseMatch
from django.utils.safestring import mark_safe
from table.utils import Accessor
from .base import Column

class LinkColumn(Column):
    def __init__(self, links, delimiter=u' ', *args, **kwargs):
        super(LinkColumn, self).__init__(*args, **kwargs)
        self.links = links
        self.delimiter = delimiter
        self.searchable = False
        self.safe = False

    def render(self, obj):
        return self.delimiter.join([link.render(obj) for link in self.links])

class Link(object):
    """ Represents a link element in html.
    """
    def __init__(self, text, viewname, args=None, kwargs=None, urlconf=None,
                 current_app=None, onclick=None):
        self.text = text
        self.viewname = viewname
        self.args = args or []
        self.kwargs = kwargs or {}
        self.urlconf = urlconf
        self.current_app = current_app
        self.onclick = onclick

    def resolve(self, obj):
        """ Resolving URL paths to the corresponding object. See:
            https://docs.djangoproject.com/en/dev/ref/urlresolvers/#reverse
        """
        viewname = self.viewname
        params = {}

        if self.args:
            params['args'] = [getattr(obj, attr) for attr in self.args]
        if self.kwargs:
            params['kwargs'] = {}
            for key, value in self.kwargs.itmes:
                params['kwargs'][key] = getattr(obj, value)
        if self.urlconf:
            params['urlconf'] = self.urlconf
        if self.current_app:
            params['current_app'] = self.current_app

        try:
            url = reverse(viewname, **params)
        except NoReverseMatch:
            url = ''
        return url

    def render(self, obj):
        """ Render link as HTML output tag .
        """
        url = self.resolve(obj)
        text = self.text.resolve(obj) if isinstance(self.text, Accessor) else self.text
        if self.onclick:
            onclick = self.onclick + '(' + url + ')'
            return mark_safe(u'%s' % (onclick, text))
        else:
            return mark_safe(u'%s' % (url, text))

<|endoftext|>


<commit_before>from __future__ import print_function, division, absolute_import

from .serialize import register_serialization, serialize, deserialize

def serialize_sparse(x):
    coords_header, coords_frames = serialize(x.coords)
    data_header, data_frames = serialize(x.data)

    header = {'coords-header': coords_header,
              'data-header': data_header,
              'shape': x.shape,
              'nframes': [len(coords_frames), len(data_frames)]}
    return header, coords_frames + data_frames

def deserialize_sparse(header, frames):
    import sparse

    coords_frames = frames[:header['nframes'][0]]
    data_frames = frames[header['nframes'][0]:]

    coords = deserialize(header['coords-header'], coords_frames)
    data = deserialize(header['data-header'], data_frames)

    shape = header['shape']

    return sparse.COO(coords, data, shape=shape)

register_serialization('sparse.core.COO', serialize_sparse, deserialize_sparse)
<commit_msg>Register sparse.coo due to version change (#1709)<commit_after>from __future__ import print_function, division, absolute_import

from .serialize import register_serialization, serialize, deserialize

def serialize_sparse(x):
    coords_header, coords_frames = serialize(x.coords)
    data_header, data_frames = serialize(x.data)

    header = {'coords-header': coords_header,
              'data-header': data_header,
              'shape': x.shape,
              'nframes': [len(coords_frames), len(data_frames)]}
    return header, coords_frames + data_frames

def deserialize_sparse(header, frames):
    import sparse

    coords_frames = frames[:header['nframes'][0]]
    data_frames = frames[header['nframes'][0]:]

    coords = deserialize(header['coords-header'], coords_frames)
    data = deserialize(header['data-header'], data_frames)

    shape = header['shape']

    return sparse.COO(coords, data, shape=shape)

register_serialization('sparse.core.COO', serialize_sparse, deserialize_sparse)  # version 0.1
register_serialization('sparse.coo.COO', serialize_sparse, deserialize_sparse)  # version 0.2
<|endoftext|>


<commit_before>from deploy import lazy_file, pipe
import ftplib
import os
import paramiko
import tarfile
import urllib
import urlparse
import sh

class location( object ):
    def __init__( self, url ):
        if isinstance( url, basestring ):
            url = urlparse.urlparse( url )
        self.url = url

    def source( self ):
        pass

    def destination( self, files ):
        pass

class local_location( location ):
    def source( self, recursive = False ):
        for fpath in os.listdir( self.url.path ):
            fpath = os.path.relpath( os.path.join( self.url.path, fpath ) )
            if os.path.isfile( fpath ):
                yield lazy_file( fpath )
            elif os.path.isdir( fpath ) and recursive:
                for f in local_location( fpath ).source( True ):
                    yield f

    def destination( self, files ):
        for f in files:
            dfn = os.path.join( self.url.path, f.name )

            # Ensure dest paths exist
            dpath = os.path.dirname( dfn )
            if not os.path.exists( dpath ):
                os.makedirs( dpath )

            if not f.delete:
                print 'local:%s' % dfn
                open( dfn, 'wb' ).write( f.read() )
            else:
                print 'local DELETE', dfn
                try:
                    os.remove( dfn )
                except OSError:
                    pass

class remote_location( location ):
    def __init__( self, url ):
        super( remote_location, self ).__init__( url )

        self.con = None

    def connect( self ):
        pass

    def destination( self, files ):
        for f in files:
            if not self.con:
                self.connect()

            if f.delete:
                self.rm( f )
            else:
                self.put( f )

class ftp_location( remote_location ):
    def connect( self ):
        self.con = ftplib.FTP()
        self.con.connect( self.url.hostname, self.url.port )
        self.con.login( urllib.unquote( self.url.username ), urllib.unquote( self.url.password ) )

    def _remote_path( self, f ):
        return os.path.join( self.url.path, os.path.dirname( f.name ) )

    def get( self, path ):
        p = pipe( path )
        self.con.cwd( self._remote_path( p ) )

        self.con.retrbinary( 'RETR %s' % os.path.basename( path ), p.write )
        p.reset()

        return p

    def put( self, f ):
        print f.name

        fpath = self._remote_path( f )

        try:
            self.con.cwd( fpath )
        except ftplib.error_perm as e:
            if e.message.startswith( '550' ):
                self.mkdirs( fpath )
                self.con.cwd( fpath )

        print '%s:%s' % ( self.url.hostname, f.name )
        self.con.storbinary( 'STOR %s' % os.path.basename( f.name ), f )

    def rm( self, f ):
        try:
            self.con.cwd( _remote_path( f ) )
        except Exception, e:
            print 'FTP-ERROR: Could not change directory to', os.path.dirname( f.name )
            print e
            print 'FTP-ERROR: Could not delete', f.name
            return

        try:
            print '%s DELETE %s' % ( self.url.hostname, f.name )
            self.con.delete( os.path.basename( f.name ) )
        except Exception, e:
            print 'FTP-ERROR: Could not delete', f.name
            print e

    def mkdirs( self, path ):
        if path.startswith( '/' ):
            path = path[1:]

        if not path:
            return

        self.con.cwd( '/' )

        last_existed = True
        for p in path.split( os.sep ):
            if not last_existed or p not in self.con.nlst():
                self.con.mkd( p )
                last_existed = False
            self.con.cwd( p )

class ssh_location( remote_location ):
    def connect( self ):
        self.sshcli = paramiko.SSHClient()
        self.sshcli.set_missing_host_key_policy( paramiko.AutoAddPolicy() )
        self.sshcli.connect( self.url.hostname, port = self.url.port )
        self.con = self.sshcli.open_sftp()

        try:
            self.con.chdir( self.url.path )
        except IOError:
            self.mkdirs( self.url.path )
            self.con.chdir( self.url.path )

    def put( self, f ):
        print '%s:%s' % ( self.url.hostname, f.name )
        try:
            self.con.putfo( f, f.name, callback = self.report_progress )
        except IOError:
            # Most likely that dir does not exist, create and retry
            self.mkdirs( os.path.dirname( f.name ) )
            self.put( f )

    def report_progress( self, a, b ):
        print a, b

    def rm( self, f ):
        print '%s DELETE %s' % ( self.url.hostname, f.name )
        self.con.remove( f.name )

    def mkdirs( self, path ):
        cur_path = ''
        last_existed = True

        for p in path.split( os.sep ):
            cur_path = os.path.join( cur_path, p )

            if last_existed:
                try:
                    self.con.stat( cur_path )
                    continue
                except IOError:
                    pass

            print 'creating', cur_path
            self.con.mkdir( cur_path )

class git_location( local_location ):
    def __init__( self, url = '' ):
        super( git_location, self ).__init__( url )

    def get_ref_commit( self, ref = 'HEAD' ):
        return str( sh.git( 'rev-parse', ref ) ).strip()

    def source( self, from_commit = None, to_commit = None ):
        if from_commit is not None:
            if to_commit is not None:
                files = sh.git.diff( '--name-status', '--no-renames', '--color=never', from_commit, to_commit, _iter = True, _tty_out = False )
            else:
                files = sh.git.diff( '--name-status', '--no-renames', '--color=never', from_commit, _iter = True, _tty_out = False )

            all_files = False
        else:
            files = sh.git( "ls-files", _iter = True, _tty_out = False )
            all_files = True

        for l in files:
            mode = 'A'
            if not all_files:
                mode, fname = l.strip().split( '\t' )
            else:
                fname = l.strip()

            f = lazy_file( fname )
            if mode == 'D':
                f.delete = True

            yield f

class tar_location( location ):
    def __init__( self, f ):
        self.f = f

    def source( self ):
        tar = tarfile.open( fileobj = self.f, mode = 'r|*' )

        for tarinfo in tar:
            yield tar.extractfile( tarinfo )

    def destination( self, files ):
        tar = tarfile.open( fileobj = self.f, mode = 'w|gz' )

        for f in files:
            f.read(0)
            tarinfo = tarfile.TarInfo( f.name )
            tarinfo.size = f.size
            tar.addfile( tarinfo, f )

class zip_location( location ):
    # Use zip
    # https://docs.python.org/2/library/zip.html
    pass
<commit_msg>handle misconfigured ftp servers that return local ip for pasv calls<commit_after>from deploy import lazy_file, pipe
import ftplib
import os
import paramiko
import tarfile
import urllib
import urlparse
import sh
import socket

class location( object ):
    def __init__( self, url ):
        if isinstance( url, basestring ):
            url = urlparse.urlparse( url )
        self.url = url

    def source( self ):
        pass

    def destination( self, files ):
        pass

class local_location( location ):
    def source( self, recursive = False ):
        for fpath in os.listdir( self.url.path ):
            fpath = os.path.relpath( os.path.join( self.url.path, fpath ) )
            if os.path.isfile( fpath ):
                yield lazy_file( fpath )
            elif os.path.isdir( fpath ) and recursive:
                for f in local_location( fpath ).source( True ):
                    yield f

    def destination( self, files ):
        for f in files:
            dfn = os.path.join( self.url.path, f.name )

            # Ensure dest paths exist
            dpath = os.path.dirname( dfn )
            if not os.path.exists( dpath ):
                os.makedirs( dpath )

            if not f.delete:
                print 'local:%s' % dfn
                open( dfn, 'wb' ).write( f.read() )
            else:
                print 'local DELETE', dfn
                try:
                    os.remove( dfn )
                except OSError:
                    pass

class remote_location( location ):
    def __init__( self, url ):
        super( remote_location, self ).__init__( url )

        self.con = None

    def connect( self ):
        pass

    def destination( self, files ):
        for f in files:
            if not self.con:
                self.connect()

            if f.delete:
                self.rm( f )
            else:
                self.put( f )

class BadPassiveFTP( ftplib.FTP ):
    def makepasv(self):
        host, port = ftplib.FTP.makepasv( self )

        return socket.gethostbyname( self.host ), port

class ftp_location( remote_location ):
    def __init__( self, url, bad_passive_server = False ):
        super( ftp_location, self ).__init__( url )
        self.bad_passive_server = bad_passive_server

    def connect( self ):
        if self.bad_passive_server:
            self.con = ftplib.FTP()
        else:
            self.con = BadPassiveFTP()

        self.con.connect( self.url.hostname, self.url.port )
        self.con.login( urllib.unquote( self.url.username ), urllib.unquote( self.url.password ) )

    def _remote_path( self, f ):
        return os.path.join( self.url.path, os.path.dirname( f.name ) )

    def get( self, path ):
        p = pipe( path )
        self.con.cwd( self._remote_path( p ) )

        self.con.retrbinary( 'RETR %s' % os.path.basename( path ), p.write )
        p.reset()

        return p

    def put( self, f ):
        print f.name

        fpath = self._remote_path( f )

        try:
            self.con.cwd( fpath )
        except ftplib.error_perm as e:
            if e.message.startswith( '550' ):
                self.mkdirs( fpath )
                self.con.cwd( fpath )

        print '%s:%s' % ( self.url.hostname, f.name )
        self.con.storbinary( 'STOR %s' % os.path.basename( f.name ), f )

    def rm( self, f ):
        try:
            self.con.cwd( _remote_path( f ) )
        except Exception, e:
            print 'FTP-ERROR: Could not change directory to', os.path.dirname( f.name )
            print e
            print 'FTP-ERROR: Could not delete', f.name
            return

        try:
            print '%s DELETE %s' % ( self.url.hostname, f.name )
            self.con.delete( os.path.basename( f.name ) )
        except Exception, e:
            print 'FTP-ERROR: Could not delete', f.name
            print e

    def mkdirs( self, path ):
        if path.startswith( '/' ):
            path = path[1:]

        if not path:
            return

        self.con.cwd( '/' )

        last_existed = True
        for p in path.split( os.sep ):
            if not last_existed or p not in self.con.nlst():
                self.con.mkd( p )
                last_existed = False
            self.con.cwd( p )

class ssh_location( remote_location ):
    def connect( self ):
        self.sshcli = paramiko.SSHClient()
        self.sshcli.set_missing_host_key_policy( paramiko.AutoAddPolicy() )
        self.sshcli.connect( self.url.hostname, port = self.url.port )
        self.con = self.sshcli.open_sftp()

        try:
            self.con.chdir( self.url.path )
        except IOError:
            self.mkdirs( self.url.path )
            self.con.chdir( self.url.path )

    def put( self, f ):
        print '%s:%s' % ( self.url.hostname, f.name )
        try:
            self.con.putfo( f, f.name, callback = self.report_progress )
        except IOError:
            # Most likely that dir does not exist, create and retry
            self.mkdirs( os.path.dirname( f.name ) )
            self.put( f )

    def report_progress( self, a, b ):
        print a, b

    def rm( self, f ):
        print '%s DELETE %s' % ( self.url.hostname, f.name )
        self.con.remove( f.name )

    def mkdirs( self, path ):
        cur_path = ''
        last_existed = True

        for p in path.split( os.sep ):
            cur_path = os.path.join( cur_path, p )

            if last_existed:
                try:
                    self.con.stat( cur_path )
                    continue
                except IOError:
                    pass

            print 'creating', cur_path
            self.con.mkdir( cur_path )

class git_location( local_location ):
    def __init__( self, url = '' ):
        super( git_location, self ).__init__( url )

    def get_ref_commit( self, ref = 'HEAD' ):
        return str( sh.git( 'rev-parse', ref ) ).strip()

    def source( self, from_commit = None, to_commit = None ):
        if from_commit is not None:
            if to_commit is not None:
                files = sh.git.diff( '--name-status', '--no-renames', '--color=never', from_commit, to_commit, _iter = True, _tty_out = False )
            else:
                files = sh.git.diff( '--name-status', '--no-renames', '--color=never', from_commit, _iter = True, _tty_out = False )

            all_files = False
        else:
            files = sh.git( "ls-files", _iter = True, _tty_out = False )
            all_files = True

        for l in files:
            mode = 'A'
            if not all_files:
                mode, fname = l.strip().split( '\t' )
            else:
                fname = l.strip()

            f = lazy_file( fname )
            if mode == 'D':
                f.delete = True

            yield f

class tar_location( location ):
    def __init__( self, f ):
        self.f = f

    def source( self ):
        tar = tarfile.open( fileobj = self.f, mode = 'r|*' )

        for tarinfo in tar:
            yield tar.extractfile( tarinfo )

    def destination( self, files ):
        tar = tarfile.open( fileobj = self.f, mode = 'w|gz' )

        for f in files:
            f.read(0)
            tarinfo = tarfile.TarInfo( f.name )
            tarinfo.size = f.size
            tar.addfile( tarinfo, f )

class zip_location( location ):
    # Use zip
    # https://docs.python.org/2/library/zip.html
    pass
<|endoftext|>


<commit_before>from __future__ import division

from django.db import models

from .reports import YearBasedInstitutionStatModel, AdmissionsManager
from ..instachart.models import SimpleChart

__all__ = ['PublicEnrollment', 'PublicGraduationRates', 'PublicAdmissions']

"""
RACES = [
    ,
    'American Indian/Alaskan Native',
    'Asian',
    'Hispanic',
    'International',
    'Multiracial',
    'Native Hawaiian/Pacific Island',
    'Unknown or Not Reported',
    'White',
]
"""

class PublicEnrollment(YearBasedInstitutionStatModel, SimpleChart):
    total = models.IntegerField(null=True)

    african_american_count = models.IntegerField(null=True)
    asian_count = models.IntegerField(null=True)
    hispanic_count = models.IntegerField(null=True)
    international_count = models.IntegerField(null=True)
    multiracial_count = models.IntegerField(null=True)
    native_american_count = models.IntegerField(null=True)
    pacific_islander_count = models.IntegerField(null=True)
    unknown_count = models.IntegerField(null=True)
    white_count = models.IntegerField(null=True)

    african_american_percent = models.FloatField(null=True,
        verbose_name='% Black')
    asian_percent = models.FloatField(null=True,
        verbose_name='% Asian')
    hispanic_percent = models.FloatField(null=True,
        verbose_name='% Hispanic')
    international_percent = models.FloatField(null=True,
        verbose_name='% International')
    multiracial_percent = models.FloatField(null=True,
        verbose_name='% Multiracial')
    native_american_percent = models.FloatField(null=True,
        verbose_name='% Native Am.')
    pacific_islander_percent = models.FloatField(null=True,
        verbose_name='% Pacific Islander')
    unknown_percent = models.FloatField(null=True,
        verbose_name='% N/A')
    white_percent = models.FloatField(null=True,
        verbose_name='% White')

    @property
    def total_percent_white(self):
        return self.white_percent

    @property
    def total_percent_black(self):
        return self.african_american_percent

    @property
    def total_percent_hispanic(self):
        return self.hispanic_percent

    @property
    def total_percent_native(self):
        return self.native_american_percent

    @property
    def total_percent_asian(self):
        return self.asian_percent

    @property
    def total_percent_unknown(self):
        return self.unknown_percent

    def race_data(self):
        race_attrs = [f.attname for f in self._meta.fields
                      if '_percent' in f.attname]
        data = []
        for race_attr in race_attrs:
            value = getattr(self, race_attr, None)
            if value is not None:
                data.append({
                    'year': self.year,
                    'metric': race_attr,
                    'race': self._meta.get_field(race_attr).verbose_name.title(),
                    'enrollment': self.total,
                    'value': value,
                })

        return data

    race_attrs = ['total_percent_%s' % race for race in
            ('white', 'black', 'hispanic', 'native', 'asian', 'unknown')]

    chart_series = (('year', 'total') +
                    tuple(race_attrs) +
                    ('international_percent', 'multiracial_percent',
                     'pacific_islander_percent'))

class PublicGraduationRates(YearBasedInstitutionStatModel, SimpleChart):
    associate_3yr = models.DecimalField(null=True,
        max_digits=5, decimal_places=2,
        verbose_name=u"Associate degree within three years",
        help_text=u"Only available for community colleges")
    associate_4yr = models.DecimalField(null=True,
        max_digits=5, decimal_places=2,
        verbose_name=u"Associate degree within four years",
        help_text=u"Only available for community colleges")
    associate_6yr = models.DecimalField(null=True,
        max_digits=5, decimal_places=2,
        verbose_name=u"Associate degree within six years",
        help_text=u"Only available for community colleges")

    bachelor_3yr = models.DecimalField(null=True,
        max_digits=5, decimal_places=2,
        verbose_name=u"Bachelor degree within three years",
        help_text=u"Only available for community colleges")
    bachelor_4yr = models.DecimalField(null=True,
        max_digits=5, decimal_places=2,
        verbose_name=u"Bachelor degree within four years")
    bachelor_5yr = models.DecimalField(null=True,
        max_digits=5, decimal_places=2,
        verbose_name=u"Bachelor degree within five years")
    bachelor_6yr = models.DecimalField(null=True,
        max_digits=5, decimal_places=2,
        verbose_name=u"Bachelor degree within six years")

    def __unicode__(self):
        return "Graduation Rates %s %s" % (self.display_year, self.institution)

    chart_series = ('display_year',
                    ('bachelor_4yr', "%s%%"),
                    ('bachelor_5yr', "%s%%"),
                    ('bachelor_6yr', "%s%%"))

class PublicAdmissions(YearBasedInstitutionStatModel, SimpleChart):
    number_of_applicants = models.IntegerField(null=True, blank=True)
    number_admitted = models.IntegerField(null=True, blank=True)
    number_admitted_who_enrolled = models.IntegerField(null=True, blank=True)
    percent_of_applicants_admitted = models.DecimalField(max_digits=4, decimal_places=1, null=True, blank=True, verbose_name=u"%admitted")
    percent_of_admitted_who_enrolled = models.DecimalField(max_digits=4, decimal_places=1, null=True, blank=True, verbose_name=u"%admitted who enrolled")

    objects = AdmissionsManager()
<commit_msg>fix % Unknown was reported as % N/A<commit_after>from __future__ import division

from django.db import models

from .reports import YearBasedInstitutionStatModel, AdmissionsManager
from ..instachart.models import SimpleChart

__all__ = ['PublicEnrollment', 'PublicGraduationRates', 'PublicAdmissions']

"""
RACES = [
    ,
    'American Indian/Alaskan Native',
    'Asian',
    'Hispanic',
    'International',
    'Multiracial',
    'Native Hawaiian/Pacific Island',
    'Unknown or Not Reported',
    'White',
]
"""

class PublicEnrollment(YearBasedInstitutionStatModel, SimpleChart):
    total = models.IntegerField(null=True)

    african_american_count = models.IntegerField(null=True)
    asian_count = models.IntegerField(null=True)
    hispanic_count = models.IntegerField(null=True)
    international_count = models.IntegerField(null=True)
    multiracial_count = models.IntegerField(null=True)
    native_american_count = models.IntegerField(null=True)
    pacific_islander_count = models.IntegerField(null=True)
    unknown_count = models.IntegerField(null=True)
    white_count = models.IntegerField(null=True)

    african_american_percent = models.FloatField(null=True,
        verbose_name='% Black')
    asian_percent = models.FloatField(null=True,
        verbose_name='% Asian')
    hispanic_percent = models.FloatField(null=True,
        verbose_name='% Hispanic')
    international_percent = models.FloatField(null=True,
        verbose_name='% International')
    multiracial_percent = models.FloatField(null=True,
        verbose_name='% Multiracial')
    native_american_percent = models.FloatField(null=True,
        verbose_name='% Native Am.')
    pacific_islander_percent = models.FloatField(null=True,
        verbose_name='% Pacific Islander')
    unknown_percent = models.FloatField(null=True,
        verbose_name='% Unknown')
    white_percent = models.FloatField(null=True,
        verbose_name='% White')

    @property
    def total_percent_white(self):
        return self.white_percent

    @property
    def total_percent_black(self):
        return self.african_american_percent

    @property
    def total_percent_hispanic(self):
        return self.hispanic_percent

    @property
    def total_percent_native(self):
        return self.native_american_percent

    @property
    def total_percent_asian(self):
        return self.asian_percent

    @property
    def total_percent_unknown(self):
        return self.unknown_percent

    def race_data(self):
        race_attrs = [f.attname for f in self._meta.fields
                      if '_percent' in f.attname]
        data = []
        for race_attr in race_attrs:
            value = getattr(self, race_attr, None)
            if value is not None:
                data.append({
                    'year': self.year,
                    'metric': race_attr,
                    'race': self._meta.get_field(race_attr).verbose_name.title(),
                    'enrollment': self.total,
                    'value': value,
                })

        return data

    race_attrs = ['total_percent_%s' % race for race in
            ('white', 'black', 'hispanic', 'native', 'asian', 'unknown')]

    chart_series = (('year', 'total') +
                    tuple(race_attrs) +
                    ('international_percent', 'multiracial_percent',
                     'pacific_islander_percent'))

class PublicGraduationRates(YearBasedInstitutionStatModel, SimpleChart):
    associate_3yr = models.DecimalField(null=True,
        max_digits=5, decimal_places=2,
        verbose_name=u"Associate degree within three years",
        help_text=u"Only available for community colleges")
    associate_4yr = models.DecimalField(null=True,
        max_digits=5, decimal_places=2,
        verbose_name=u"Associate degree within four years",
        help_text=u"Only available for community colleges")
    associate_6yr = models.DecimalField(null=True,
        max_digits=5, decimal_places=2,
        verbose_name=u"Associate degree within six years",
        help_text=u"Only available for community colleges")

    bachelor_3yr = models.DecimalField(null=True,
        max_digits=5, decimal_places=2,
        verbose_name=u"Bachelor degree within three years",
        help_text=u"Only available for community colleges")
    bachelor_4yr = models.DecimalField(null=True,
        max_digits=5, decimal_places=2,
        verbose_name=u"Bachelor degree within four years")
    bachelor_5yr = models.DecimalField(null=True,
        max_digits=5, decimal_places=2,
        verbose_name=u"Bachelor degree within five years")
    bachelor_6yr = models.DecimalField(null=True,
        max_digits=5, decimal_places=2,
        verbose_name=u"Bachelor degree within six years")

    def __unicode__(self):
        return "Graduation Rates %s %s" % (self.display_year, self.institution)

    chart_series = ('display_year',
                    ('bachelor_4yr', "%s%%"),
                    ('bachelor_5yr', "%s%%"),
                    ('bachelor_6yr', "%s%%"))

class PublicAdmissions(YearBasedInstitutionStatModel, SimpleChart):
    number_of_applicants = models.IntegerField(null=True, blank=True)
    number_admitted = models.IntegerField(null=True, blank=True)
    number_admitted_who_enrolled = models.IntegerField(null=True, blank=True)
    percent_of_applicants_admitted = models.DecimalField(max_digits=4, decimal_places=1, null=True, blank=True, verbose_name=u"%admitted")
    percent_of_admitted_who_enrolled = models.DecimalField(max_digits=4, decimal_places=1, null=True, blank=True, verbose_name=u"%admitted who enrolled")

    objects = AdmissionsManager()
<|endoftext|>


<commit_before>from __future__ import division

from django.db import models

from .reports import YearBasedInstitutionStatModel

__all__ = ['PublicEnrollment']

"""
RACES = [
    ,
    'American Indian/Alaskan Native',
    'Asian',
    'Hispanic',
    'International',
    'Multiracial',
    'Native Hawaiian/Pacific Island',
    'Unknown or Not Reported',
    'White',
]
"""

class PublicEnrollment(YearBasedInstitutionStatModel):
    total = models.IntegerField(null=True)

    african_american_count = models.IntegerField(null=True)
    asian_count = models.IntegerField(null=True)
    hispanic_count = models.IntegerField(null=True)
    international_count = models.IntegerField(null=True)
    multiracial_count = models.IntegerField(null=True)
    native_american_count = models.IntegerField(null=True)
    pacific_islander_count = models.IntegerField(null=True)
    unknown_count = models.IntegerField(null=True)
    white_count = models.IntegerField(null=True)

    african_american_percent = models.FloatField(null=True)
    asian_percent = models.FloatField(null=True)
    hispanic_percent = models.FloatField(null=True)
    international_percent = models.FloatField(null=True)
    multiracial_percent = models.FloatField(null=True)
    native_american_percent = models.FloatField(null=True)
    pacific_islander_percent = models.FloatField(null=True)
    unknown_percent = models.FloatField(null=True)
    white_percent = models.FloatField(null=True)
<commit_msg>Add verbose names to race percent fields<commit_after>from __future__ import division

from django.db import models

from .reports import YearBasedInstitutionStatModel

__all__ = ['PublicEnrollment']

"""
RACES = [
    ,
    'American Indian/Alaskan Native',
    'Asian',
    'Hispanic',
    'International',
    'Multiracial',
    'Native Hawaiian/Pacific Island',
    'Unknown or Not Reported',
    'White',
]
"""

class PublicEnrollment(YearBasedInstitutionStatModel):
    total = models.IntegerField(null=True)

    african_american_count = models.IntegerField(null=True)
    asian_count = models.IntegerField(null=True)
    hispanic_count = models.IntegerField(null=True)
    international_count = models.IntegerField(null=True)
    multiracial_count = models.IntegerField(null=True)
    native_american_count = models.IntegerField(null=True)
    pacific_islander_count = models.IntegerField(null=True)
    unknown_count = models.IntegerField(null=True)
    white_count = models.IntegerField(null=True)

    african_american_percent = models.FloatField(null=True,
        verbose_name='% Black')
    asian_percent = models.FloatField(null=True,
        verbose_name='% Asian')
    hispanic_percent = models.FloatField(null=True,
        verbose_name='% Hispanic')
    international_percent = models.FloatField(null=True,
        verbose_name='% International')
    multiracial_percent = models.FloatField(null=True,
        verbose_name='% Multiracial')
    native_american_percent = models.FloatField(null=True,
        verbose_name='% Native Am.')
    pacific_islander_percent = models.FloatField(null=True,
        verbose_name='% Pacific Islander')
    unknown_percent = models.FloatField(null=True,
        verbose_name='% N/A')
    white_percent = models.FloatField(null=True,
        verbose_name='% White')
<|endoftext|>


<commit_before># coding: utf-8
import os

from tests.util import temp_dir, build_grab
from tests.util import BaseGrabTestCase

from grab import GrabMisuseError

class GrabSimpleTestCase(BaseGrabTestCase):
    def setUp(self):
        self.server.reset()

    def test_body_inmemory_false(self):
        with temp_dir() as tmp_dir:
            grab = build_grab()
            grab.setup(body_inmemory=False)
            with self.assertRaises(GrabMisuseError):
                grab.go(self.server.get_url())

            self.server.response['get.data'] = b'foo'
            grab = build_grab()
            grab.setup(body_inmemory=False)
            grab.setup(body_storage_dir=tmp_dir)
            grab.go(self.server.get_url())
            self.assertTrue(os.path.exists(grab.doc.body_path))
            self.assertTrue(tmp_dir in grab.doc.body_path)
            with open(grab.doc.body_path, 'rb') as inp:
                self.assertEqual(b'foo', inp.read())
            # pylint: disable=protected-access
            self.assertEqual(grab.doc._bytes_body, None)
            # pylint: enable=protected-access
            old_path = grab.doc.body_path

            grab.go(self.server.get_url())
            self.assertTrue(old_path != grab.doc.body_path)

        with temp_dir() as tmp_dir:
            self.server.response['get.data'] = 'foo'
            grab = build_grab()
            grab.setup(body_inmemory=False)
            grab.setup(body_storage_dir=tmp_dir)
            grab.setup(body_storage_filename='music.mp3')
            grab.go(self.server.get_url())
            self.assertTrue(os.path.exists(grab.doc.body_path))
            self.assertTrue(tmp_dir in grab.doc.body_path)
            with open(grab.doc.body_path, 'rb') as inp:
                self.assertEqual(b'foo', inp.read())
            self.assertEqual(os.path.join(tmp_dir, 'music.mp3'),
                             grab.doc.body_path)
            self.assertEqual(grab.doc.body, b'foo')
            # pylint: disable=protected-access
            self.assertEqual(grab.doc._bytes_body, None)
            # pylint: enable=protected-access

    def test_body_inmemory_true(self):
        grab = build_grab()
        self.server.response['data'] = b'bar'
        grab.go(self.server.get_url())
        # pylint: disable=protected-access
        self.assertEqual(grab.doc._bytes_body, b'bar')
        # pylint: enable=protected-access

    def test_assign_unicode_to_body(self):
        grab = build_grab()
        grab.doc.body = b'abc'
        grab.doc.body = b'def'

        with self.assertRaises(GrabMisuseError):
            grab.doc.body = u'Спутник'

    def test_empty_response(self):
        self.server.response['data'] = b''
        grab = build_grab()
        grab.go(self.server.get_url())
        # pylint: disable=pointless-statement
        grab.doc.tree # should not raise exception
        # pylint: enable=pointless-statement

    #def test_emoji_processing(self):
    #    #html = u'''
    #    #
    #    #     👍🏻
    #    #
    #    #'''.encode('utf-8')
    #    grab = build_grab()
    #    #print('>>',grab.doc('//span').text(),'<<')
    #    #import grab
    #    #grab = grab.Grab()
    #    #grab.go('https://github.com/lorien/grab/issues/199'
    #            '#issuecomment-269854859')
    #    grab.go('https://en.wikipedia.org/wiki/Emoji')
    #    grab.doc.select("//*")

    def test_doc_tree_notags_document(self):
        data = b'test'
        grab = build_grab(data)
        self.assertEqual(grab.doc('//html').text(), 'test')
<commit_msg>Add lxml test for issue #199<commit_after># coding: utf-8
import os

from tests.util import temp_dir, build_grab, TEST_DIR
from tests.util import BaseGrabTestCase

from grab import GrabMisuseError

class GrabSimpleTestCase(BaseGrabTestCase):
    def setUp(self):
        self.server.reset()

    def test_body_inmemory_false(self):
        with temp_dir() as tmp_dir:
            grab = build_grab()
            grab.setup(body_inmemory=False)
            with self.assertRaises(GrabMisuseError):
                grab.go(self.server.get_url())

            self.server.response['get.data'] = b'foo'
            grab = build_grab()
            grab.setup(body_inmemory=False)
            grab.setup(body_storage_dir=tmp_dir)
            grab.go(self.server.get_url())
            self.assertTrue(os.path.exists(grab.doc.body_path))
            self.assertTrue(tmp_dir in grab.doc.body_path)
            with open(grab.doc.body_path, 'rb') as inp:
                self.assertEqual(b'foo', inp.read())
            # pylint: disable=protected-access
            self.assertEqual(grab.doc._bytes_body, None)
            # pylint: enable=protected-access
            old_path = grab.doc.body_path

            grab.go(self.server.get_url())
            self.assertTrue(old_path != grab.doc.body_path)

        with temp_dir() as tmp_dir:
            self.server.response['get.data'] = 'foo'
            grab = build_grab()
            grab.setup(body_inmemory=False)
            grab.setup(body_storage_dir=tmp_dir)
            grab.setup(body_storage_filename='music.mp3')
            grab.go(self.server.get_url())
            self.assertTrue(os.path.exists(grab.doc.body_path))
            self.assertTrue(tmp_dir in grab.doc.body_path)
            with open(grab.doc.body_path, 'rb') as inp:
                self.assertEqual(b'foo', inp.read())
            self.assertEqual(os.path.join(tmp_dir, 'music.mp3'),
                             grab.doc.body_path)
            self.assertEqual(grab.doc.body, b'foo')
            # pylint: disable=protected-access
            self.assertEqual(grab.doc._bytes_body, None)
            # pylint: enable=protected-access

    def test_body_inmemory_true(self):
        grab = build_grab()
        self.server.response['data'] = b'bar'
        grab.go(self.server.get_url())
        # pylint: disable=protected-access
        self.assertEqual(grab.doc._bytes_body, b'bar')
        # pylint: enable=protected-access

    def test_assign_unicode_to_body(self):
        grab = build_grab()
        grab.doc.body = b'abc'
        grab.doc.body = b'def'

        with self.assertRaises(GrabMisuseError):
            grab.doc.body = u'Спутник'

    def test_empty_response(self):
        self.server.response['data'] = b''
        grab = build_grab()
        grab.go(self.server.get_url())
        # pylint: disable=pointless-statement
        grab.doc.tree # should not raise exception
        # pylint: enable=pointless-statement

    #def test_emoji_processing(self):
    #    #html = u'''
    #    #
    #    #     👍🏻
    #    #
    #    #'''.encode('utf-8')
    #    grab = build_grab()
    #    #print('>>',grab.doc('//span').text(),'<<')
    #    #import grab
    #    #grab = grab.Grab()
    #    #grab.go('https://github.com/lorien/grab/issues/199'
    #            '#issuecomment-269854859')
    #    grab.go('https://en.wikipedia.org/wiki/Emoji')
    #    grab.doc.select("//*")

    def test_doc_tree_notags_document(self):
        data = b'test'
        grab = build_grab(data)
        self.assertEqual(grab.doc('//html').text(), 'test')

    def test_github_html_processing(self):
        """This test is for osx and py3.5
        See: https://github.com/lorien/grab/issues/199

        It should works with recent lxml builds on any platform.
        In the past it failed on previous lxml releases on macos platform
        """
        path = os.path.join(TEST_DIR, 'files/github_showcases.html')
        with open(path) as inp:
            data = inp.read()
        self.server.response['data'] = data
        grab = build_grab()
        grab.go(self.server.get_url())
        items = []
        for elem in grab.doc('//a[contains(@class, "exploregrid-item")]'):
            items.append(grab.make_url_absolute(elem.attr('href')))
        self.assertTrue('tools-for-open-source' in items[2])
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
    :codeauthor: :email:`Pedro Algarvio (pedro@algarvio.me)`

    tests.integration.shell.master
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
'''

# Import python libs
from __future__ import absolute_import
import os
import signal
import shutil

# Import 3rd-party libs
import yaml

# Import salt libs
import salt.utils

# Import salt test libs
import tests.integration.utils
from tests.support.case import ShellCase
from tests.support.paths import TMP
from tests.support.mixins import ShellCaseCommonTestsMixin
from tests.integration.utils import testprogram

class MasterTest(ShellCase, testprogram.TestProgramCase, ShellCaseCommonTestsMixin):

    _call_binary_ = 'salt-master'

    def test_issue_7754(self):
        old_cwd = os.getcwd()
        config_dir = os.path.join(TMP, 'issue-7754')
        if not os.path.isdir(config_dir):
            os.makedirs(config_dir)

        os.chdir(config_dir)

        config_file_name = 'master'
        pid_path = os.path.join(config_dir, '{0}.pid'.format(config_file_name))
        with salt.utils.fopen(self.get_config_file_path(config_file_name), 'r') as fhr:
            config = yaml.load(fhr.read())
            config['root_dir'] = config_dir
            config['log_file'] = 'file:///tmp/log/LOG_LOCAL3'
            config['ret_port'] = config['ret_port'] + 10
            config['publish_port'] = config['publish_port'] + 10

            with salt.utils.fopen(os.path.join(config_dir, config_file_name), 'w') as fhw:
                fhw.write(
                    yaml.dump(config, default_flow_style=False)
                )

        ret = self.run_script(
            self._call_binary_,
            '--config-dir {0} --pid-file {1} -l debug'.format(
                config_dir,
                pid_path
            ),
            timeout=5,
            catch_stderr=True,
            with_retcode=True
        )

        # Now kill it if still running
        if os.path.exists(pid_path):
            with salt.utils.fopen(pid_path) as fhr:
                try:
                    os.kill(int(fhr.read()), signal.SIGKILL)
                except OSError:
                    pass
        try:
            self.assertFalse(os.path.isdir(os.path.join(config_dir, 'file:')))
        finally:
            self.chdir(old_cwd)
            if os.path.isdir(config_dir):
                shutil.rmtree(config_dir)

    def test_exit_status_unknown_user(self):
        '''
        Ensure correct exit status when the master is configured to run as an unknown user.
        '''

        master = testprogram.TestDaemonSaltMaster(
            name='unknown_user',
            configs={'master': {'map': {'user': 'some_unknown_user_xyz'}}},
            parent_dir=self._test_dir,
        )
        # Call setup here to ensure config and script exist
        master.setup()
        stdout, stderr, status = master.run(
            args=['-d'],
            catch_stderr=True,
            with_retcode=True,
        )
        self.assert_exit_status(
            status, 'EX_NOUSER',
            message='unknown user not on system',
            stdout=stdout,
            stderr=tests.integration.utils.decode_byte_list(stderr)
        )
        # Although the start-up should fail, call shutdown() to set the internal
        # _shutdown flag and avoid the registered atexit calls to cause timeout
        # exeptions and respective traceback
        master.shutdown()

    # pylint: disable=invalid-name
    def test_exit_status_unknown_argument(self):
        '''
        Ensure correct exit status when an unknown argument is passed to salt-master.
        '''

        master = testprogram.TestDaemonSaltMaster(
            name='unknown_argument',
            parent_dir=self._test_dir,
        )
        # Call setup here to ensure config and script exist
        master.setup()
        stdout, stderr, status = master.run(
            args=['-d', '--unknown-argument'],
            catch_stderr=True,
            with_retcode=True,
        )
        self.assert_exit_status(
            status, 'EX_USAGE',
            message='unknown argument',
            stdout=stdout,
            stderr=tests.integration.utils.decode_byte_list(stderr)
        )
        # Although the start-up should fail, call shutdown() to set the internal
        # _shutdown flag and avoid the registered atexit calls to cause timeout
        # exeptions and respective traceback
        master.shutdown()

    def test_exit_status_correct_usage(self):
        '''
        Ensure correct exit status when salt-master starts correctly.
        '''

        master = testprogram.TestDaemonSaltMaster(
            name='correct_usage',
            parent_dir=self._test_dir,
        )
        # Call setup here to ensure config and script exist
        master.setup()
        stdout, stderr, status = master.run(
            args=['-d'],
            catch_stderr=True,
            with_retcode=True,
        )
        self.assert_exit_status(
            status, 'EX_OK',
            message='correct usage',
            stdout=stdout,
            stderr=tests.integration.utils.decode_byte_list(stderr)
        )
        master.shutdown()
<commit_msg>Check master shut down correctly.<commit_after># -*- coding: utf-8 -*-
'''
    :codeauthor: :email:`Pedro Algarvio (pedro@algarvio.me)`

    tests.integration.shell.master
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
'''

# Import python libs
from __future__ import absolute_import
import os
import signal
import shutil

# Import 3rd-party libs
import yaml

# Import salt libs
import salt.utils

# Import salt test libs
import tests.integration.utils
from tests.support.case import ShellCase
from tests.support.paths import TMP
from tests.support.mixins import ShellCaseCommonTestsMixin
from tests.integration.utils import testprogram

class MasterTest(ShellCase, testprogram.TestProgramCase, ShellCaseCommonTestsMixin):

    _call_binary_ = 'salt-master'

    def test_issue_7754(self):
        old_cwd = os.getcwd()
        config_dir = os.path.join(TMP, 'issue-7754')
        if not os.path.isdir(config_dir):
            os.makedirs(config_dir)

        os.chdir(config_dir)

        config_file_name = 'master'
        pid_path = os.path.join(config_dir, '{0}.pid'.format(config_file_name))
        with salt.utils.fopen(self.get_config_file_path(config_file_name), 'r') as fhr:
            config = yaml.load(fhr.read())
            config['root_dir'] = config_dir
            config['log_file'] = 'file:///tmp/log/LOG_LOCAL3'
            config['ret_port'] = config['ret_port'] + 10
            config['publish_port'] = config['publish_port'] + 10

            with salt.utils.fopen(os.path.join(config_dir, config_file_name), 'w') as fhw:
                fhw.write(
                    yaml.dump(config, default_flow_style=False)
                )

        ret = self.run_script(
            self._call_binary_,
            '--config-dir {0} --pid-file {1} -l debug'.format(
                config_dir,
                pid_path
            ),
            timeout=5,
            catch_stderr=True,
            with_retcode=True
        )

        # Now kill it if still running
        if os.path.exists(pid_path):
            with salt.utils.fopen(pid_path) as fhr:
                try:
                    os.kill(int(fhr.read()), signal.SIGKILL)
                except OSError:
                    pass
        try:
            self.assertFalse(os.path.isdir(os.path.join(config_dir, 'file:')))
        finally:
            self.chdir(old_cwd)
            if os.path.isdir(config_dir):
                shutil.rmtree(config_dir)

    def test_exit_status_unknown_user(self):
        '''
        Ensure correct exit status when the master is configured to run as an unknown user.
        '''

        master = testprogram.TestDaemonSaltMaster(
            name='unknown_user',
            configs={'master': {'map': {'user': 'some_unknown_user_xyz'}}},
            parent_dir=self._test_dir,
        )
        # Call setup here to ensure config and script exist
        master.setup()
        stdout, stderr, status = master.run(
            args=['-d'],
            catch_stderr=True,
            with_retcode=True,
        )
        self.assert_exit_status(
            status, 'EX_NOUSER',
            message='unknown user not on system',
            stdout=stdout,
            stderr=tests.integration.utils.decode_byte_list(stderr)
        )
        # Although the start-up should fail, call shutdown() to set the internal
        # _shutdown flag and avoid the registered atexit calls to cause timeout
        # exeptions and respective traceback
        master.shutdown()

    # pylint: disable=invalid-name
    def test_exit_status_unknown_argument(self):
        '''
        Ensure correct exit status when an unknown argument is passed to salt-master.
        '''

        master = testprogram.TestDaemonSaltMaster(
            name='unknown_argument',
            parent_dir=self._test_dir,
        )
        # Call setup here to ensure config and script exist
        master.setup()
        stdout, stderr, status = master.run(
            args=['-d', '--unknown-argument'],
            catch_stderr=True,
            with_retcode=True,
        )
        self.assert_exit_status(
            status, 'EX_USAGE',
            message='unknown argument',
            stdout=stdout,
            stderr=tests.integration.utils.decode_byte_list(stderr)
        )
        # Although the start-up should fail, call shutdown() to set the internal
        # _shutdown flag and avoid the registered atexit calls to cause timeout
        # exeptions and respective traceback
        master.shutdown()

    def test_exit_status_correct_usage(self):
        '''
        Ensure correct exit status when salt-master starts correctly.
        '''

        master = testprogram.TestDaemonSaltMaster(
            name='correct_usage',
            parent_dir=self._test_dir,
        )
        # Call setup here to ensure config and script exist
        master.setup()
        stdout, stderr, status = master.run(
            args=['-d'],
            catch_stderr=True,
            with_retcode=True,
        )
        self.assert_exit_status(
            status, 'EX_OK',
            message='correct usage',
            stdout=stdout,
            stderr=tests.integration.utils.decode_byte_list(stderr)
        )
        master.shutdown()
        # Do the test again to check does master shut down correctly
        stdout, stderr, status = master.run(
            args=['-d'],
            catch_stderr=True,
            with_retcode=True,
        )
        self.assert_exit_status(
            status, 'EX_OK',
            message='correct usage',
            stdout=stdout,
            stderr=tests.integration.utils.decode_byte_list(stderr)
        )
        master.shutdown()
<|endoftext|>


<commit_before>""" TACA controllers
"""
import os
import re
import shutil
import time

from cement.core import controller

from taca.controllers import BaseController
from taca.utils import filesystem, misc

class StorageController(BaseController):
    """ Storage Controller

    Entry point for all functionalities related to storage
    """
    class Meta:
        label = 'storage'
        description = "Entry point for all functionalities related to storage"
        stacked_on = 'base'
        stacked_type = 'nested'
        arguments = [
            (['-r', '--run'], dict(type=str, help="Work with a specific run")),
            (['-d', '--days'], dict(type=int, default=10, help="Days to consider a run \"old\""))
        ]

    #######################
    # Storage subcommands #
    #######################

    @controller.expose(help="Move old runs to nosync directory so they're not synced to the processing server")
    def cleanup(self):
        for data_dir in self.app.config.get('storage', 'data_dirs'):
            with filesystem.chdir(data_dir):
                for run in [r for r in os.listdir(data_dir) if re.match(filesystem.RUN_RE, r)]:
                    rta_file = os.path.join(run, 'RTAComplete.txt')
                    if os.path.exists(rta_file):
                        # 1 day == 60*60*24 seconds --> 86400
                        if os.stat(rta_file).st_mtime < time.time() - 86400:
                            self.app.log.info('Moving run {} to nosync directory'.format(os.path.basename(run)))
                            shutil.move(run, 'nosync')
                        else:
                            self.app.log.info('RTAComplete.txt file exists but is not older than 1 day, skipping run {}'.format(run))

    @controller.expose(help="Archive old runs to SWESTORE")
    def archive_to_swestore(self):
        # If the run is specified in the command line, check that exists and archive
        if self.app.pargs.run:
            run = os.path.basename(self.app.pargs.run)
            base_dir = os.path.dirname(self.app.pargs.run)
            if re.match(filesystem.RUN_RE, run):
                # If the parameter is not an absolute path, find the run in the archive_dirs
                if not base_dir:
                    for archive_dir in self.app.config.get('storage', 'archive_dirs'):
                        if os.path.exists(os.path.join(archive_dir, run)):
                            base_dir = archive_dir
                if not os.path.exists(os.path.join(base_dir, run)):
                    self.app.log.error(("Run {} not found. Please make sure to specify "
                        "the absolute path or relative path being in the correct directory.".format(run)))
                else:
                    with filesystem.chdir(base_dir):
                        self._archive_run(run)
            else:
                self.app.log.error("The name {} doesn't look like an Illumina run".format(os.path.basename(run)))
        # Otherwise find all runs in every data dir on the nosync partition
        else:
            self.app.log.info("Archiving old runs to SWESTORE")
            for data_dir in self.app.config.get('storage', 'data_dirs'):
                to_send_dir = os.path.join(data_dir, 'nosync')
                self.app.log.info('Checking {} directory'.format(to_send_dir))
                with filesystem.chdir(to_send_dir):
                    for run in [r for r in os.listdir(to_send_dir) if re.match(filesystem.RUN_RE, r)]:
                        self._archive_run(run)

    #############################################################
    # Class helper methods, not exposed as commands/subcommands #
    #############################################################
    def _archive_run(self, run):
        """ Archive a specific run to swestore

        :param str run: Run directory
        """
        def _send_to_swestore(f, dest, remove=True):
            """ Send file to swestore checking adler32 on destination and eventually
            removing the file from disk

            :param str f: File to remove
            :param str dest: Destination directory in Swestore
            :param bool remove: If True, remove original file from source
            """
            if not filesystem.is_in_swestore(f):
                self.app.log.info("Sending {} to swestore".format(f))
                misc.call_external_command('iput -K -P {file} {dest}'.format(file=f, dest=dest),
                        with_log_files=True)
                self.app.log.info('Run {} sent correctly and checksum was okay.'.format(f))
            else:
                self.app.log.warn('Run {} is already in Swestore, not sending it again'.format(f))
            if remove:
                self.app.log.info('Removing run'.format(f))
                os.remove(f)

        if run.endswith('bz2'):
            _send_to_swestore(run, self.app.config.get('storage', 'irods').get('irodsHome'))
        else:
            self.app.log.info("Compressing run {}".format(run))
            # Compress with pbzip2
            misc.call_external_command('tar --use-compress-program=pbzip2 -cf {run}.tar.bz2 {run}'.format(run=run))
            self.app.log.info('Run {} successfully compressed! Removing from disk...'.format(run))
            shutil.rmtree(run)
            _send_to_swestore('{}.tar.bz2'.format(run), self.app.config.get('storage', 'irods').get('irodsHome'))
<commit_msg>Detach iput command, fixes #49<commit_after>""" TACA controllers
"""
import os
import re
import shutil
import time

from cement.core import controller

from taca.controllers import BaseController
from taca.utils import filesystem, misc

class StorageController(BaseController):
    """ Storage Controller

    Entry point for all functionalities related to storage
    """
    class Meta:
        label = 'storage'
        description = "Entry point for all functionalities related to storage"
        stacked_on = 'base'
        stacked_type = 'nested'
        arguments = [
            (['-r', '--run'], dict(type=str, help="Work with a specific run")),
            (['-d', '--days'], dict(type=int, default=10, help="Days to consider a run \"old\""))
        ]

    #######################
    # Storage subcommands #
    #######################

    @controller.expose(help="Move old runs to nosync directory so they're not synced to the processing server")
    def cleanup(self):
        for data_dir in self.app.config.get('storage', 'data_dirs'):
            with filesystem.chdir(data_dir):
                for run in [r for r in os.listdir(data_dir) if re.match(filesystem.RUN_RE, r)]:
                    rta_file = os.path.join(run, 'RTAComplete.txt')
                    if os.path.exists(rta_file):
                        # 1 day == 60*60*24 seconds --> 86400
                        if os.stat(rta_file).st_mtime < time.time() - 86400:
                            self.app.log.info('Moving run {} to nosync directory'.format(os.path.basename(run)))
                            shutil.move(run, 'nosync')
                        else:
                            self.app.log.info('RTAComplete.txt file exists but is not older than 1 day, skipping run {}'.format(run))

    @controller.expose(help="Archive old runs to SWESTORE")
    def archive_to_swestore(self):
        # If the run is specified in the command line, check that exists and archive
        if self.app.pargs.run:
            run = os.path.basename(self.app.pargs.run)
            base_dir = os.path.dirname(self.app.pargs.run)
            if re.match(filesystem.RUN_RE, run):
                # If the parameter is not an absolute path, find the run in the archive_dirs
                if not base_dir:
                    for archive_dir in self.app.config.get('storage', 'archive_dirs'):
                        if os.path.exists(os.path.join(archive_dir, run)):
                            base_dir = archive_dir
                if not os.path.exists(os.path.join(base_dir, run)):
                    self.app.log.error(("Run {} not found. Please make sure to specify "
                        "the absolute path or relative path being in the correct directory.".format(run)))
                else:
                    with filesystem.chdir(base_dir):
                        self._archive_run(run)
            else:
                self.app.log.error("The name {} doesn't look like an Illumina run".format(os.path.basename(run)))
        # Otherwise find all runs in every data dir on the nosync partition
        else:
            self.app.log.info("Archiving old runs to SWESTORE")
            for data_dir in self.app.config.get('storage', 'data_dirs'):
                to_send_dir = os.path.join(data_dir, 'nosync')
                self.app.log.info('Checking {} directory'.format(to_send_dir))
                with filesystem.chdir(to_send_dir):
                    for run in [r for r in os.listdir(to_send_dir) if re.match(filesystem.RUN_RE, r)]:
                        self._archive_run(run)

    #############################################################
    # Class helper methods, not exposed as commands/subcommands #
    #############################################################
    def _archive_run(self, run):
        """ Archive a specific run to swestore

        :param str run: Run directory
        """
        def _send_to_swestore(f, dest, remove=True):
            """ Send file to swestore checking adler32 on destination and eventually
            removing the file from disk

            :param str f: File to remove
            :param str dest: Destination directory in Swestore
            :param bool remove: If True, remove original file from source
            """
            if not filesystem.is_in_swestore(f):
                self.app.log.info("Sending {} to swestore".format(f))
                misc.call_external_command_detached('iput -K -P {file} {dest}'.format(file=f, dest=dest),
                        with_log_files=True)
                self.app.log.info('Run {} sent correctly and checksum was okay.'.format(f))
            else:
                self.app.log.warn('Run {} is already in Swestore, not sending it again'.format(f))
            if remove:
                self.app.log.info('Removing run'.format(f))
                os.remove(f)

        if run.endswith('bz2'):
            _send_to_swestore(run, self.app.config.get('storage', 'irods').get('irodsHome'))
        else:
            self.app.log.info("Compressing run {}".format(run))
            # Compress with pbzip2
            misc.call_external_command('tar --use-compress-program=pbzip2 -cf {run}.tar.bz2 {run}'.format(run=run))
            self.app.log.info('Run {} successfully compressed! Removing from disk...'.format(run))
            shutil.rmtree(run)
            _send_to_swestore('{}.tar.bz2'.format(run), self.app.config.get('storage', 'irods').get('irodsHome'))
<|endoftext|>