<commit_before>#!/usr/bin/env python

"""
Functions to do automatic visualization of activation-like maps.

For 2D-only visualization, only matplotlib is required.
For 3D visualization, Mayavi, version 3.0 or greater, is required.

For a demo, see the 'demo_plot_map' function.

"""

# Author: Gael Varoquaux <gael dot varoquaux at normalesup dot org>
# License: BSD

# Standard library imports
import warnings
import operator

# Standard scientific libraries imports (more specific imports are
# delayed, so that the part module can be used without them).
import numpy as np

# Import pylab
from nipy.utils.skip_test import skip_if_running_nose

try:
    import pylab as pl
except ImportError:
    skip_if_running_nose('Could not import matplotlib')

from .anat_cache import mni_sform, mni_sform_inv, _AnatCache
from .coord_tools import (coord_transform,
                          get_cut_coords
                          )

from .slicers import SLICERS, _xyz_order
from edge_detect import _fast_abs_percentile

################################################################################
# Helper functions for 2D plotting of activation maps
################################################################################

def plot_map(map, affine, cut_coords=None, anat=None, anat_affine=None,
                    slicer='ortho', figure=None, axes=None, title=None,
                    threshold=None, annotate=True, draw_cross=True,
                    do3d=False, threshold_3d=None,
                    view_3d=(38.5, 70.5, 300, (-2.7, -12, 9.1)),
                    black_bg=False, **kwargs):
    """ Plot three cuts of a given activation map (Frontal, Axial, and Lateral)

        Parameters
        ----------
        map : 3D ndarray
            The activation map, as a 3D image.
        affine : 4x4 ndarray
            The affine matrix going from image voxel space to MNI space.
        cut_coords: None, or a tuple of floats
            The MNI coordinates of the point where the cut is performed, in
            MNI coordinates and order.
            If slicer is 'ortho', this should be a 3-tuple: (x, y, z)
            For slicer == 'x', 'y', or 'z', then these are the
            coordinates of each cut in the corresponding direction.
            If None is given, the cuts is calculated automaticaly. Further, if
            slicer= == 'z' (in future, this should work
            for "x", and "y" too), then we'll use a heuristic to automatically
            compute a series of cut_coords.
        anat : 3D ndarray or False, optional
            The anatomical image to be used as a background. If None, the
            MNI152 T1 1mm template is used. If False, no anat is displayed.
        anat_affine : 4x4 ndarray, optional
            The affine matrix going from the anatomical image voxel space to
            MNI space. This parameter is not used when the default
            anatomical is used, but it is compulsory when using an
            explicite anatomical image.
        slicer: {'ortho', 'x', 'y', 'z'}
            Choose the direction of the cuts. With 'ortho' three cuts are
            performed in orthogonal directions
        figure : integer or matplotlib figure, optional
            Matplotlib figure used or its number. If None is given, a
            new figure is created.
        axes : matplotlib axes or 4 tuple of float: (xmin, ymin, width, height), optional
            The axes, or the coordinates, in matplotlib figure space,
            of the axes used to display the plot. If None, the complete
            figure is used.
        title : string, optional
            The title dispayed on the figure.
        threshold : a number, None, or 'auto'
            If None is given, the maps are not thresholded.
            If a number is given, it is used to threshold the maps:
            values below the threshold are plotted as transparent. If
            auto is given, the threshold is determined magically by
            analysis of the map.
        annotate: boolean, optional
            If annotate is True, positions and left/right annotation
            are added to the plot.
        draw_cross: boolean, optional
            If draw_cross is True, a cross is drawn on the plot to
            indicate the cut plosition.
        do3d: {True, False or 'interactive'}, optional
            If True, Mayavi is used to plot a 3D view of the
            map in addition to the slicing. If 'interactive', the
            3D visualization is displayed in an additional interactive
            window.
        threshold_3d:
            The threshold to use for the 3D view (if any). Defaults to
            the same threshold as that used for the 2D view.
        view_3d: tuple,
            The view used to take the screenshot: azimuth, elevation,
            distance and focalpoint, see the docstring of mlab.view.
        black_bg: boolean, optional
            If True, the background of the image is set to be black. If
            you whish to save figures with a black background, you
            will need to pass "facecolor='k', edgecolor='k'" to pylab's
            savefig.
        kwargs: extra keyword arguments, optional
            Extra keyword arguments passed to pylab.imshow

        Notes
        -----
        Arrays should be passed in numpy convention: (x, y, z)
        ordered.

        Use masked arrays to create transparency:

            import numpy as np
            map = np.ma.masked_less(map, 0.5)
            plot_map(map, affine)
    """

    map, affine = _xyz_order(map, affine)

    nan_mask = np.isnan(np.asarray(map))
    if np.any(nan_mask):
        map = map.copy()
        map[nan_mask] = 0
    del nan_mask

    # Deal with automatic settings of plot parameters
    if threshold == 'auto':
        # Threshold epsilon above a percentile value, to be sure that some
        # voxels are indeed threshold
        threshold = _fast_abs_percentile(map) + 1e-5

    if do3d:
        try:
            try:
                from mayavi import version
            except ImportError:
                from enthought.mayavi import version
            if not int(version.version[0]) > 2:
                raise ImportError
        except ImportError:
            warnings.warn('Mayavi > 3.x not installed, plotting only 2D')
            do3d = False

    if cut_coords is None and slicer in 'xyz':
        cut_coords = get_cut_coords(map)

    slicer = SLICERS[slicer].init_with_figure(data=map, affine=affine,
                                          threshold=threshold,
                                          cut_coords=cut_coords,
                                          figure=figure, axes=axes,
                                          black_bg=black_bg,
                                          leave_space=do3d)

    # Use Mayavi for the 3D plotting
    if do3d:
        from .maps_3d import plot_map_3d, m2screenshot
        try:
            from tvtk.api import tvtk
        except ImportError:
            from enthought.tvtk.api import tvtk
        version = tvtk.Version()
        offscreen = True
        if (version.vtk_major_version, version.vtk_minor_version) < (5, 2):
            offscreen = False
        if do3d == 'interactive':
            offscreen = False

        cmap = kwargs.get('cmap', pl.cm.cmap_d[pl.rcParams['image.cmap']])
        # Computing vmin and vmax is costly in time, and is needed
        # later, so we compute them now, and store them for future
        # use
        vmin = kwargs.get('vmin', map.min())
        kwargs['vmin'] = vmin
        vmax = kwargs.get('vmax', map.max())
        kwargs['vmax'] = vmax
        try:
            from mayavi import mlab
        except ImportError:
            from enthought.mayavi import mlab
        if threshold_3d is None:
            threshold_3d = threshold
        plot_map_3d(np.asarray(map), affine, cut_coords=cut_coords,
                    anat=anat, anat_affine=anat_affine,
                    offscreen=offscreen, cmap=cmap,
                    threshold=threshold_3d,
                    view=view_3d,
                    vmin=vmin, vmax=vmax)

        ax = slicer.axes.values()[0].ax.figure.add_axes((0.001, 0, 0.29, 1))
        ax.axis('off')
        m2screenshot(mpl_axes=ax)
        if offscreen:
            # Clean up, so that the offscreen engine doesn't become the
            # default
            mlab.clf()
            engine = mlab.get_engine()
            try:
                from mayavi.core.registry import registry
            except:
                from enthought.mayavi.core.registry import registry
            for key, value in registry.engines.iteritems():
                if value is engine:
                    registry.engines.pop(key)
                    break

    if threshold:
        map = np.ma.masked_inside(map, -threshold, threshold, copy=False)

    _plot_anat(slicer, anat, anat_affine, title=title,
               annotate=annotate, draw_cross=draw_cross)

    slicer.plot_map(map, affine, **kwargs)
    return slicer

def _plot_anat(slicer, anat, anat_affine, title=None,
               annotate=True, draw_cross=True, dim=False, cmap=pl.cm.gray):
    """ Internal function used to plot anatomy
    """
    canonical_anat = False
    if anat is None:
        try:
            anat, anat_affine, vmax_anat = _AnatCache.get_anat()
            canonical_anat = True
        except OSError, e:
            anat = False
            warnings.warn(repr(e))

    black_bg = slicer._black_bg
    # XXX: Check that we should indeed plot an anat: we have one, and the
    # cut_coords are in its range

    if anat is not False:
        if canonical_anat:
            # We special-case the 'canonical anat', as we don't need
            # to do a few transforms to it.
            vmin = 0
            vmax = vmax_anat
        elif dim:
            vmin = anat.min()
            vmax = anat.max()
        else:
            vmin = None
            vmax = None
            anat, anat_affine = _xyz_order(anat, anat_affine)
        if dim:
            vmean = .5*(vmin + vmax)
            ptp = .5*(vmax - vmin)
            if not operator.isNumberType(dim):
                dim = .6
            if black_bg:
                vmax = vmean + (1+dim)*ptp
            else:
                vmin = vmean - (1+dim)*ptp
        slicer.plot_map(anat, anat_affine, cmap=cmap,
                              vmin=vmin, vmax=vmax)

        if annotate:
            slicer.annotate()
        if draw_cross:
            slicer.draw_cross()

    if black_bg:
        # To have a black background in PDF, we need to create a
        # patch in black for the background
        for ax in slicer.axes.values():
            ax.ax.imshow(np.zeros((2, 2, 3)),
                         extent=[-5000, 5000, -5000, 5000],
                         zorder=-500)

    if title is not None and not title == '':
        slicer.title(title)
    return slicer

def plot_anat(anat=None, anat_affine=None, cut_coords=None, slicer='ortho',
              figure=None, axes=None, title=None, annotate=True,
              draw_cross=True, black_bg=False, dim=False, cmap=pl.cm.gray):
    """ Plot three cuts of an anatomical image (Frontal, Axial, and Lateral)

        Parameters
        ----------
        anat : 3D ndarray, optional
            The anatomical image to be used as a background. If None is
            given, nipy tries to find a T1 template.
        anat_affine : 4x4 ndarray, optional
            The affine matrix going from the anatomical image voxel space to
            MNI space. This parameter is not used when the default
            anatomical is used, but it is compulsory when using an
            explicite anatomical image.
        figure : integer or matplotlib figure, optional
            Matplotlib figure used or its number. If None is given, a
            new figure is created.
        cut_coords: None, or a tuple of floats
            The MNI coordinates of the point where the cut is performed, in
            MNI coordinates and order.
            If slicer is 'ortho', this should be a 3-tuple: (x, y, z)
            For slicer == 'x', 'y', or 'z', then these are the
            coordinates of each cut in the corresponding direction.
            If None is given, the cuts is calculated automaticaly.
        slicer: {'ortho', 'x', 'y', 'z'}
            Choose the direction of the cuts. With 'ortho' three cuts are
            performed in orthogonal directions
        figure : integer or matplotlib figure, optional
            Matplotlib figure used or its number. If None is given, a
            new figure is created.
        axes : matplotlib axes or 4 tuple of float: (xmin, ymin, width, height), optional
            The axes, or the coordinates, in matplotlib figure space,
            of the axes used to display the plot. If None, the complete
            figure is used.
        title : string, optional
            The title dispayed on the figure.
        annotate: boolean, optional
            If annotate is True, positions and left/right annotation
            are added to the plot.
        draw_cross: boolean, optional
            If draw_cross is True, a cross is drawn on the plot to
            indicate the cut plosition.
        black_bg: boolean, optional
            If True, the background of the image is set to be black. If
            you whish to save figures with a black background, you
            will need to pass "facecolor='k', edgecolor='k'" to pylab's
            savefig.
        cmap: matplotlib colormap, optional
            The colormap for the anat

        Notes
        -----
        Arrays should be passed in numpy convention: (x, y, z)
        ordered.
    """
    slicer = SLICERS[slicer].init_with_figure(data=anat, affine=anat_affine,
                                          threshold=0, cut_coords=cut_coords,
                                          figure=figure, axes=axes,
                                          black_bg=black_bg)

    _plot_anat(slicer, anat, anat_affine, title=title,
               annotate=annotate, draw_cross=draw_cross, dim=dim, cmap=cmap)
    return slicer

def demo_plot_map(do3d=False, **kwargs):
    """ Demo activation map plotting.
    """
    map = np.zeros((182, 218, 182))
    # Color a asymetric rectangle around Broca area:
    x, y, z = -52, 10, 22
    x_map, y_map, z_map = coord_transform(x, y, z, mni_sform_inv)
    # Compare to values obtained using fslview. We need to add one as
    # voxels do not start at 0 in fslview.
    assert x_map == 142
    assert y_map +1 == 137
    assert z_map +1 == 95
    map[x_map-5:x_map+5, y_map-3:y_map+3, z_map-10:z_map+10] = 1
    return plot_map(map, mni_sform, threshold='auto',
                        title="Broca's area", do3d=do3d,
                        **kwargs)

<commit_msg>FIX: fixed misleading docstring (see bertrand's comments on PR #280)<commit_after>#!/usr/bin/env python

"""
Functions to do automatic visualization of activation-like maps.

For 2D-only visualization, only matplotlib is required.
For 3D visualization, Mayavi, version 3.0 or greater, is required.

For a demo, see the 'demo_plot_map' function.

"""

# Author: Gael Varoquaux <gael dot varoquaux at normalesup dot org>
# License: BSD

# Standard library imports
import warnings
import operator

# Standard scientific libraries imports (more specific imports are
# delayed, so that the part module can be used without them).
import numpy as np

# Import pylab
from nipy.utils.skip_test import skip_if_running_nose

try:
    import pylab as pl
except ImportError:
    skip_if_running_nose('Could not import matplotlib')

from .anat_cache import mni_sform, mni_sform_inv, _AnatCache
from .coord_tools import (coord_transform,
                          get_cut_coords
                          )

from .slicers import SLICERS, _xyz_order
from edge_detect import _fast_abs_percentile

################################################################################
# Helper functions for 2D plotting of activation maps
################################################################################

def plot_map(map, affine, cut_coords=None, anat=None, anat_affine=None,
                    slicer='ortho', figure=None, axes=None, title=None,
                    threshold=None, annotate=True, draw_cross=True,
                    do3d=False, threshold_3d=None,
                    view_3d=(38.5, 70.5, 300, (-2.7, -12, 9.1)),
                    black_bg=False, **kwargs):
    """ Plot three cuts of a given activation map (Frontal, Axial, and Lateral)

        Parameters
        ----------
        map : 3D ndarray
            The activation map, as a 3D image.
        affine : 4x4 ndarray
            The affine matrix going from image voxel space to MNI space.
        cut_coords: None, or a tuple of floats
            The MNI coordinates of the point where the cut is performed, in
            MNI coordinates and order.
            If slicer is 'ortho', this should be a 3-tuple: (x, y, z)
            For slicer == 'x', 'y', or 'z', then these are the
            coordinates of each cut in the corresponding direction.
            If None is given, the cuts is calculated automaticaly.
        anat : 3D ndarray or False, optional
            The anatomical image to be used as a background. If None, the
            MNI152 T1 1mm template is used. If False, no anat is displayed.
        anat_affine : 4x4 ndarray, optional
            The affine matrix going from the anatomical image voxel space to
            MNI space. This parameter is not used when the default
            anatomical is used, but it is compulsory when using an
            explicite anatomical image.
        slicer: {'ortho', 'x', 'y', 'z'}
            Choose the direction of the cuts. With 'ortho' three cuts are
            performed in orthogonal directions
        figure : integer or matplotlib figure, optional
            Matplotlib figure used or its number. If None is given, a
            new figure is created.
        axes : matplotlib axes or 4 tuple of float: (xmin, ymin, width, height), optional
            The axes, or the coordinates, in matplotlib figure space,
            of the axes used to display the plot. If None, the complete
            figure is used.
        title : string, optional
            The title dispayed on the figure.
        threshold : a number, None, or 'auto'
            If None is given, the maps are not thresholded.
            If a number is given, it is used to threshold the maps:
            values below the threshold are plotted as transparent. If
            auto is given, the threshold is determined magically by
            analysis of the map.
        annotate: boolean, optional
            If annotate is True, positions and left/right annotation
            are added to the plot.
        draw_cross: boolean, optional
            If draw_cross is True, a cross is drawn on the plot to
            indicate the cut plosition.
        do3d: {True, False or 'interactive'}, optional
            If True, Mayavi is used to plot a 3D view of the
            map in addition to the slicing. If 'interactive', the
            3D visualization is displayed in an additional interactive
            window.
        threshold_3d:
            The threshold to use for the 3D view (if any). Defaults to
            the same threshold as that used for the 2D view.
        view_3d: tuple,
            The view used to take the screenshot: azimuth, elevation,
            distance and focalpoint, see the docstring of mlab.view.
        black_bg: boolean, optional
            If True, the background of the image is set to be black. If
            you whish to save figures with a black background, you
            will need to pass "facecolor='k', edgecolor='k'" to pylab's
            savefig.
        kwargs: extra keyword arguments, optional
            Extra keyword arguments passed to pylab.imshow

        Notes
        -----
        Arrays should be passed in numpy convention: (x, y, z)
        ordered.

        Use masked arrays to create transparency:

            import numpy as np
            map = np.ma.masked_less(map, 0.5)
            plot_map(map, affine)
    """

    map, affine = _xyz_order(map, affine)

    nan_mask = np.isnan(np.asarray(map))
    if np.any(nan_mask):
        map = map.copy()
        map[nan_mask] = 0
    del nan_mask

    # Deal with automatic settings of plot parameters
    if threshold == 'auto':
        # Threshold epsilon above a percentile value, to be sure that some
        # voxels are indeed threshold
        threshold = _fast_abs_percentile(map) + 1e-5

    if do3d:
        try:
            try:
                from mayavi import version
            except ImportError:
                from enthought.mayavi import version
            if not int(version.version[0]) > 2:
                raise ImportError
        except ImportError:
            warnings.warn('Mayavi > 3.x not installed, plotting only 2D')
            do3d = False

    if cut_coords is None and slicer in 'xyz':
        cut_coords = get_cut_coords(map)

    slicer = SLICERS[slicer].init_with_figure(data=map, affine=affine,
                                          threshold=threshold,
                                          cut_coords=cut_coords,
                                          figure=figure, axes=axes,
                                          black_bg=black_bg,
                                          leave_space=do3d)

    # Use Mayavi for the 3D plotting
    if do3d:
        from .maps_3d import plot_map_3d, m2screenshot
        try:
            from tvtk.api import tvtk
        except ImportError:
            from enthought.tvtk.api import tvtk
        version = tvtk.Version()
        offscreen = True
        if (version.vtk_major_version, version.vtk_minor_version) < (5, 2):
            offscreen = False
        if do3d == 'interactive':
            offscreen = False

        cmap = kwargs.get('cmap', pl.cm.cmap_d[pl.rcParams['image.cmap']])
        # Computing vmin and vmax is costly in time, and is needed
        # later, so we compute them now, and store them for future
        # use
        vmin = kwargs.get('vmin', map.min())
        kwargs['vmin'] = vmin
        vmax = kwargs.get('vmax', map.max())
        kwargs['vmax'] = vmax
        try:
            from mayavi import mlab
        except ImportError:
            from enthought.mayavi import mlab
        if threshold_3d is None:
            threshold_3d = threshold
        plot_map_3d(np.asarray(map), affine, cut_coords=cut_coords,
                    anat=anat, anat_affine=anat_affine,
                    offscreen=offscreen, cmap=cmap,
                    threshold=threshold_3d,
                    view=view_3d,
                    vmin=vmin, vmax=vmax)

        ax = slicer.axes.values()[0].ax.figure.add_axes((0.001, 0, 0.29, 1))
        ax.axis('off')
        m2screenshot(mpl_axes=ax)
        if offscreen:
            # Clean up, so that the offscreen engine doesn't become the
            # default
            mlab.clf()
            engine = mlab.get_engine()
            try:
                from mayavi.core.registry import registry
            except:
                from enthought.mayavi.core.registry import registry
            for key, value in registry.engines.iteritems():
                if value is engine:
                    registry.engines.pop(key)
                    break

    if threshold:
        map = np.ma.masked_inside(map, -threshold, threshold, copy=False)

    _plot_anat(slicer, anat, anat_affine, title=title,
               annotate=annotate, draw_cross=draw_cross)

    slicer.plot_map(map, affine, **kwargs)
    return slicer

def _plot_anat(slicer, anat, anat_affine, title=None,
               annotate=True, draw_cross=True, dim=False, cmap=pl.cm.gray):
    """ Internal function used to plot anatomy
    """
    canonical_anat = False
    if anat is None:
        try:
            anat, anat_affine, vmax_anat = _AnatCache.get_anat()
            canonical_anat = True
        except OSError, e:
            anat = False
            warnings.warn(repr(e))

    black_bg = slicer._black_bg
    # XXX: Check that we should indeed plot an anat: we have one, and the
    # cut_coords are in its range

    if anat is not False:
        if canonical_anat:
            # We special-case the 'canonical anat', as we don't need
            # to do a few transforms to it.
            vmin = 0
            vmax = vmax_anat
        elif dim:
            vmin = anat.min()
            vmax = anat.max()
        else:
            vmin = None
            vmax = None
            anat, anat_affine = _xyz_order(anat, anat_affine)
        if dim:
            vmean = .5*(vmin + vmax)
            ptp = .5*(vmax - vmin)
            if not operator.isNumberType(dim):
                dim = .6
            if black_bg:
                vmax = vmean + (1+dim)*ptp
            else:
                vmin = vmean - (1+dim)*ptp
        slicer.plot_map(anat, anat_affine, cmap=cmap,
                              vmin=vmin, vmax=vmax)

        if annotate:
            slicer.annotate()
        if draw_cross:
            slicer.draw_cross()

    if black_bg:
        # To have a black background in PDF, we need to create a
        # patch in black for the background
        for ax in slicer.axes.values():
            ax.ax.imshow(np.zeros((2, 2, 3)),
                         extent=[-5000, 5000, -5000, 5000],
                         zorder=-500)

    if title is not None and not title == '':
        slicer.title(title)
    return slicer

def plot_anat(anat=None, anat_affine=None, cut_coords=None, slicer='ortho',
              figure=None, axes=None, title=None, annotate=True,
              draw_cross=True, black_bg=False, dim=False, cmap=pl.cm.gray):
    """ Plot three cuts of an anatomical image (Frontal, Axial, and Lateral)

        Parameters
        ----------
        anat : 3D ndarray, optional
            The anatomical image to be used as a background. If None is
            given, nipy tries to find a T1 template.
        anat_affine : 4x4 ndarray, optional
            The affine matrix going from the anatomical image voxel space to
            MNI space. This parameter is not used when the default
            anatomical is used, but it is compulsory when using an
            explicite anatomical image.
        figure : integer or matplotlib figure, optional
            Matplotlib figure used or its number. If None is given, a
            new figure is created.
        cut_coords: None, or a tuple of floats
            The MNI coordinates of the point where the cut is performed, in
            MNI coordinates and order.
            If slicer is 'ortho', this should be a 3-tuple: (x, y, z)
            For slicer == 'x', 'y', or 'z', then these are the
            coordinates of each cut in the corresponding direction.
            If None is given, the cuts is calculated automaticaly.
        slicer: {'ortho', 'x', 'y', 'z'}
            Choose the direction of the cuts. With 'ortho' three cuts are
            performed in orthogonal directions
        figure : integer or matplotlib figure, optional
            Matplotlib figure used or its number. If None is given, a
            new figure is created.
        axes : matplotlib axes or 4 tuple of float: (xmin, ymin, width, height), optional
            The axes, or the coordinates, in matplotlib figure space,
            of the axes used to display the plot. If None, the complete
            figure is used.
        title : string, optional
            The title dispayed on the figure.
        annotate: boolean, optional
            If annotate is True, positions and left/right annotation
            are added to the plot.
        draw_cross: boolean, optional
            If draw_cross is True, a cross is drawn on the plot to
            indicate the cut plosition.
        black_bg: boolean, optional
            If True, the background of the image is set to be black. If
            you whish to save figures with a black background, you
            will need to pass "facecolor='k', edgecolor='k'" to pylab's
            savefig.
        cmap: matplotlib colormap, optional
            The colormap for the anat

        Notes
        -----
        Arrays should be passed in numpy convention: (x, y, z)
        ordered.
    """
    slicer = SLICERS[slicer].init_with_figure(data=anat, affine=anat_affine,
                                          threshold=0, cut_coords=cut_coords,
                                          figure=figure, axes=axes,
                                          black_bg=black_bg)

    _plot_anat(slicer, anat, anat_affine, title=title,
               annotate=annotate, draw_cross=draw_cross, dim=dim, cmap=cmap)
    return slicer

def demo_plot_map(do3d=False, **kwargs):
    """ Demo activation map plotting.
    """
    map = np.zeros((182, 218, 182))
    # Color a asymetric rectangle around Broca area:
    x, y, z = -52, 10, 22
    x_map, y_map, z_map = coord_transform(x, y, z, mni_sform_inv)
    # Compare to values obtained using fslview. We need to add one as
    # voxels do not start at 0 in fslview.
    assert x_map == 142
    assert y_map +1 == 137
    assert z_map +1 == 95
    map[x_map-5:x_map+5, y_map-3:y_map+3, z_map-10:z_map+10] = 1
    return plot_map(map, mni_sform, threshold='auto',
                        title="Broca's area", do3d=do3d,
                        **kwargs)

<|endoftext|>


<commit_before># emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
import os
from tempfile import mkdtemp
from shutil import rmtree

from nipype.testing import (assert_equal, assert_true, assert_false,
                            assert_raises, skipif, parametric)
import nipype.interfaces.matlab as mlab
from nipype.interfaces.base import CommandLine, Bunch

try:
    matlab_cmd = os.environ['MATLABCMD']
except:
    matlab_cmd = 'matlab'

res = CommandLine(command='which', args=matlab_cmd).run()
matlab_path = res.runtime.stdout.strip()

matlab_command = ''
no_matlab = True
if matlab_path != '':
    matlab_command = '%s' % \
        matlab_path.split(os.path.sep)[-1]
    no_matlab = False

# If a test requires matlab, prefix it with the skipif decorator like
# below.  Must import skipif from nipype.testing
#
#@skipif(no_matlab)
# def test_func():
#     pass

def test_cmdline():
    basedir = mkdtemp()

    mi = mlab.MatlabCommand(script='whos',
                            script_file='testscript')

    yield assert_equal, mi.cmdline, \
        'matlab -nodesktop -nosplash -singleCompThread -r "fprintf(1,\'Executing code at %s:\\n\',datestr(now));ver,try,whos,catch ME,ME,ME.stack,fprintf(\'%s\\n\',ME.message);fprintf(2,\'<MatlabScriptException>\');fprintf(2,\'%s\\n\',ME.message);fprintf(2,\'File:%s\\nName:%s\\nLine:%d\\n\',ME.stack.file,ME.stack.name,ME.stack.line);fprintf(2,\'</MatlabScriptException>\');end;;exit"'

    yield assert_equal, mi.inputs.script, 'whos'
    yield assert_equal, mi.inputs.script_file, 'testscript'
    path_exists = os.path.exists(os.path.join(basedir,'testscript.m'))
    yield assert_false, path_exists
    rmtree(basedir)

def test_mlab_inputspec():
    spec = mlab.MatlabInputSpec()
    for k in ['paths', 'script', 'nosplash', 'mfile', 'logfile', 'script_file',
              'nodesktop']:
        yield assert_true, k in spec.copyable_trait_names()
    yield assert_true, spec.nodesktop
    yield assert_true, spec.nosplash
    yield assert_false, spec.mfile
    yield assert_equal, spec.script_file, 'pyscript.m'

@parametric
def test_mlab_init():
    yield assert_equal(mlab.MatlabCommand._cmd, 'matlab')
    yield assert_equal(mlab.MatlabCommand.input_spec, mlab.MatlabInputSpec)

    yield assert_equal(mlab.MatlabCommand().cmd, mlab.MatlabCommand._cmd)
    mc = mlab.MatlabCommand(matlab_cmd='foo_m')
    yield assert_equal(mc.cmd, 'foo_m')

@skipif(no_matlab)
def test_run_interface():
    mc = mlab.MatlabCommand(matlab_cmd='foo_m')
    yield assert_raises, ValueError, mc.run # script is mandatory
    mc.inputs.script = 'a=1;'
    res = mc.run
    yield assert_true, res.runtime.returncode != 0 # foo_m is not an executable
    cwd = os.getcwd()
    basedir = mkdtemp()
    os.chdir(basedir)
    res = mlab.MatlabCommand(script='foo', paths=[basedir], mfile=True).run() # bypasses ubuntu dash issue
    yield assert_equal, res.runtime.returncode, 1
    res = mlab.MatlabCommand(script='a=1;', paths=[basedir], mfile=True).run() # bypasses ubuntu dash issue
    yield assert_equal, res.runtime.returncode, 0
    os.chdir(cwd)
    rmtree(basedir)

def test_set_matlabcmd():
    mi = mlab.MatlabCommand()
    mi.set_default_matlab_cmd('foo')
    yield assert_equal, mi._default_matlab_cmd, 'foo'
    mi.set_default_matlab_cmd(matlab_command)

<commit_msg>fixed matlab test<commit_after># emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
import os
from tempfile import mkdtemp
from shutil import rmtree

from nipype.testing import (assert_equal, assert_true, assert_false,
                            assert_raises, skipif, parametric)
import nipype.interfaces.matlab as mlab
from nipype.interfaces.base import CommandLine, Bunch

try:
    matlab_cmd = os.environ['MATLABCMD']
except:
    matlab_cmd = 'matlab'

res = CommandLine(command='which', args=matlab_cmd).run()
matlab_path = res.runtime.stdout.strip()

matlab_command = ''
no_matlab = True
if matlab_path != '':
    matlab_command = '%s' % \
        matlab_path.split(os.path.sep)[-1]
    no_matlab = False

# If a test requires matlab, prefix it with the skipif decorator like
# below.  Must import skipif from nipype.testing
#
#@skipif(no_matlab)
# def test_func():
#     pass

def test_cmdline():
    basedir = mkdtemp()

    mi = mlab.MatlabCommand(script='whos',
                            script_file='testscript')

    yield assert_equal, mi.cmdline, \
        'matlab -nodesktop -nosplash -singleCompThread -r "fprintf(1,\'Executing code at %s:\\n\',datestr(now));ver,try,whos,catch ME,ME,ME.stack,fprintf(\'%s\\n\',ME.message);fprintf(2,\'<MatlabScriptException>\');fprintf(2,\'%s\\n\',ME.message);fprintf(2,\'File:%s\\nName:%s\\nLine:%d\\n\',ME.stack.file,ME.stack.name,ME.stack.line);fprintf(2,\'</MatlabScriptException>\');end;;exit"'

    yield assert_equal, mi.inputs.script, 'whos'
    yield assert_equal, mi.inputs.script_file, 'testscript'
    path_exists = os.path.exists(os.path.join(basedir,'testscript.m'))
    yield assert_false, path_exists
    rmtree(basedir)

def test_mlab_inputspec():
    spec = mlab.MatlabInputSpec()
    for k in ['paths', 'script', 'nosplash', 'mfile', 'logfile', 'script_file',
              'nodesktop']:
        yield assert_true, k in spec.copyable_trait_names()
    yield assert_true, spec.nodesktop
    yield assert_true, spec.nosplash
    yield assert_false, spec.mfile
    yield assert_equal, spec.script_file, 'pyscript.m'

@parametric
def test_mlab_init():
    yield assert_equal(mlab.MatlabCommand._cmd, 'matlab')
    yield assert_equal(mlab.MatlabCommand.input_spec, mlab.MatlabInputSpec)

    yield assert_equal(mlab.MatlabCommand().cmd, mlab.MatlabCommand._cmd)
    mc = mlab.MatlabCommand(matlab_cmd='foo_m')
    yield assert_equal(mc.cmd, 'foo_m')

@parametric
@skipif(no_matlab)
def test_run_interface():
    mc = mlab.MatlabCommand(matlab_cmd='foo_m')
    yield assert_raises(ValueError, mc.run) # script is mandatory
    mc.inputs.script = 'a=1;'
    yield assert_raises(IOError, mc.run) # foo_m is not an executable
    cwd = os.getcwd()
    basedir = mkdtemp()
    os.chdir(basedir)
    res = mlab.MatlabCommand(script='foo', paths=[basedir], mfile=True).run() # bypasses ubuntu dash issue
    yield assert_equal(res.runtime.returncode, 1)
    res = mlab.MatlabCommand(script='a=1;', paths=[basedir], mfile=True).run() # bypasses ubuntu dash issue
    yield assert_equal(res.runtime.returncode, 0)
    os.chdir(cwd)
    rmtree(basedir)

def test_set_matlabcmd():
    mi = mlab.MatlabCommand()
    mi.set_default_matlab_cmd('foo')
    yield assert_equal, mi._default_matlab_cmd, 'foo'
    mi.set_default_matlab_cmd(matlab_command)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#

import os
import re
from openpyxl import load_workbook

def iterate_all_xlsx(inpath):
    if os.path.isfile(inpath):
        yield inpath
    else:
        for f in os.listdir(inpath):
            if os.path.splitext(f)[1] == '.xlsx':
                yield os.path.join(inpath, f)

def retrieve_contents(inpath, outfile):
    regex = re.compile('\r\n|\n')
    with open(outfile, 'w') as of:
        for infile in iterate_all_xlsx(inpath):
            #print(infile)
            wb = load_workbook(infile)
            ws = wb.active
            cells = ws['C']
            #print(len(cells) - 1)
            for cell in cells:
                if cell.row < 2:
                    continue
                of.write(regex.sub(' ', str(cell.value)) + '\n')

def machine_filter(infile, outfile, kwfile):
    from functools import reduce
    kwdict = {}
    with open(kwfile, 'r') as kf:
        for w in kf:
            print(w[2:], w[:1])
            kwdict[w[2:].strip()] = w[:1]
    print(kwdict)
    regstr = reduce(lambda r, w : r + '|' + w, kwdict.keys())
    print(regstr)
    regex = re.compile(regstr)

    GOOD_CONTENT, BAD_CONTENT, SKIP_CONTENT = '1', '2', '3'
    count = 0
    with open(outfile, 'w') as rf:
        with open(infile, 'r') as df:
            for i, line in enumerate(df):
                #if regex.match(line):
                m = regex.search(line)
                if m:
                    rf.write(kwdict[m.group(0)] + '\n')
                    print(kwdict[m.group(0)], m.group(0))
                    count += 1
                else:
                    #pass
                    rf.write(SKIP_CONTENT + '\n')
                    #print('3')
    print('Matched %d lines.' % count)

<commit_msg>Modify mechine filter<commit_after># -*- coding: utf-8 -*-
#

import os
import re
from openpyxl import load_workbook

def iterate_all_xlsx(inpath):
    if os.path.isfile(inpath):
        yield inpath
    else:
        for f in os.listdir(inpath):
            if os.path.splitext(f)[1] == '.xlsx':
                yield os.path.join(inpath, f)

def retrieve_contents(inpath, outfile):
    regex = re.compile('\r\n|\n')
    with open(outfile, 'w') as of:
        for infile in iterate_all_xlsx(inpath):
            #print(infile)
            wb = load_workbook(infile)
            ws = wb.active
            cells = ws['C']
            #print(len(cells) - 1)
            for cell in cells:
                if cell.row < 2:
                    continue
                of.write(regex.sub(' ', str(cell.value)) + '\n')

def machine_filter(infile, outfile, kwfile):
    from functools import reduce
    kwdict = {}
    with open(kwfile, 'r') as kf:
        for w in kf:
            #print(w[2:].strip(), w[:1])
            kwdict[w[2:].strip()] = w[:1]
    #print(kwdict)
    regstr = reduce(lambda r, w : r + '|' + w, kwdict.keys())
    #print(regstr)
    regex = re.compile(regstr)

    GOOD_CONTENT, BAD_CONTENT, SKIP_CONTENT = '1', '2', '3'
    count = 0
    with open(outfile, 'w') as rf:
        with open(infile, 'r') as df:
            for i, line in enumerate(df):
                m = regex.search(line)
                if m:
                    rf.write(kwdict[m.group(0)] + ',' + line)
                    #rf.write(kwdict[m.group(0)] + '\n')
                    #print(kwdict[m.group(0)], m.group(0))
                    count += 1
                else:
                    pass
                    #rf.write(SKIP_CONTENT + '\n')
                    #print('3')
    #print('Matched %d lines.' % count)

<|endoftext|>


<commit_before># coding: utf-8

# Copyright (c) 2015-2016, thumbor-community
# Use of this source code is governed by the MIT license that can be
# found in the LICENSE file.

from json import loads, dumps
from os.path import join, splitext
from datetime import datetime
from dateutil.tz import tzutc

from tornado.concurrent import return_future
from thumbor.utils import logger

from .bucket import Bucket

class AwsStorage():
    """
    Base storage class
    """
    @property
    def is_auto_webp(self):
        """
        Determines based on context whether we automatically use webp or not
        :return: Use WebP?
        :rtype: bool
        """
        return self.context.config.AUTO_WEBP and hasattr(self.context, 'request') and self.context.request.accepts_webp

    @property
    def storage(self):
        """
        Instantiates bucket based on configuration
        :return: The bucket
        :rtype: Bucket
        """
        return Bucket(self._get_config('BUCKET'), self.context.config.get('TC_AWS_REGION'))

    def __init__(self, context, config_prefix):
        """
        Constructor
        :param Context context: An instance of thumbor's context
        :param string config_prefix: Prefix used to load configuration values
        """
        self.config_prefix = config_prefix
        self.context = context

    @return_future
    def get(self, path, callback):
        """
        Gets data at path
        :param string path: Path for data
        :param callable callback: Callback function for once the retrieval is done
        """
        file_abspath = self._normalize_path(path)

        self.storage.get(file_abspath, callback=callback)

    @return_future
    def set(self, bytes, abspath, callback=None):
        """
        Stores data at given path
        :param bytes bytes: Data to store
        :param string abspath: Path to store the data at
        :return: Path where the data is stored
        :rtype: string
        """
        metadata = {}

        if self.config_prefix is 'TC_AWS_RESULT_STORAGE' and self.context.config.get('TC_AWS_STORE_METADATA'):
            metadata = dict(self.context.headers)

        self.storage.put(
            abspath,
            bytes,
            metadata=metadata,
            reduced_redundancy=self.context.config.get('TC_AWS_STORAGE_RRS', False),
            encrypt_key=self.context.config.get('TC_AWS_STORAGE_SSE', False),
            callback=callback,
        )

    def remove(self, path):
        """
        Deletes data at path
        :param string path: Path to delete
        :return: Whether deletion is successful or not
        :rtype: bool
        """
        yield self.storage.delete(path)
        return

    @return_future
    def exists(self, path, callback):
        """
        Tells if data exists at given path
        :param string path: Path to check
        :param callable callback: Callback function for once the check is done
        """
        file_abspath = self._normalize_path(path)

        def return_data(file_key):
            if not file_key or self._get_error(file_key):
                callback(False)
            else:
                callback(True)

        self.storage.get(file_abspath, callback=return_data)

    def is_expired(self, key):
        """
        Tells whether key has expired
        :param string key: Path to check
        :return: Whether it is expired or not
        :rtype: bool
        """
        if key and self._get_error(key) is None and 'LastModified' in key:
            expire_in_seconds = self.context.config.get('STORAGE_EXPIRATION_SECONDS', 3600)

            # Never expire
            if expire_in_seconds is None or expire_in_seconds == 0:
                return False

            timediff = datetime.now(tzutc()) - key['LastModified']

            return timediff.seconds > self.context.config.get('STORAGE_EXPIRATION_SECONDS', 3600)
        else:
            #If our key is bad just say we're expired
            return True

    @return_future
    def last_updated(self, callback):
        """
        Tells when the image has last been updated
        :param callable callback: Callback function for once the retrieval is done
        """
        path = self.context.request.url
        file_abspath = self._normalize_path(path)

        def on_file_fetched(file):
            if not file or self._get_error(file) or self.is_expired(file) or 'LastModified' not in file:
                logger.warn("[AwsStorage] s3 key not found at %s" % file_abspath)
                callback(None)
            else:
                callback(file['LastModified'])

        self.storage.get(file_abspath, callback=on_file_fetched)

    @return_future
    def get_crypto(self, path, callback):
        """
        Retrieves crypto data at path
        :param string path: Path to search for crypto data
        :param callable callback: Callback function for once the retrieval is done
        """
        file_abspath = self._normalize_path(path)
        crypto_path = "%s.txt" % (splitext(file_abspath)[0])

        def return_data(file_key):
            if not file_key or self._get_error(file_key) or self.is_expired(file_key) or 'Body' not in file_key:
                logger.warn("[STORAGE] s3 key not found at %s" % crypto_path)
                callback(None)
            else:
                callback(file_key['Body'])

        self.storage.get(crypto_path, callback=return_data)

    def put_crypto(self, path):
        """
        Stores crypto data at given path
        :param string path: Path to store the data at
        :return: Path where the crypto data is stored
        """
        if not self.context.config.STORES_CRYPTO_KEY_FOR_EACH_IMAGE:
            return

        if not self.context.server.security_key:
            raise RuntimeError("STORES_CRYPTO_KEY_FOR_EACH_IMAGE can't be True if no SECURITY_KEY specified")

        file_abspath = self._normalize_path(path)
        crypto_path = '%s.txt' % splitext(file_abspath)[0]

        self.set(self.context.server.security_key, crypto_path)

        return crypto_path

    @return_future
    def get_detector_data(self, path, callback):
        """
        Retrieves detector data from storage
        :param string path: Path where the data is stored
        :param callable callback: Callback function for once the retrieval is done
        """
        file_abspath = self._normalize_path(path)
        path = '%s.detectors.txt' % splitext(file_abspath)[0]

        def return_data(file_key):
            if not file_key or self._get_error(file_key) or self.is_expired(file_key) or 'Body' not in file_key:
                logger.warn("[AwsStorage] s3 key not found at %s" % path)
                callback(None)
            else:
                callback(loads(file_key['Body'].read()))

        self.storage.get(path, callback=return_data)

    def put_detector_data(self, path, data):
        """
        Stores detector data at given path
        :param string path: Path to store the data at
        :param bytes data:  Data to store
        :return: Path where the data is stored
        :rtype: string
        """
        file_abspath = self._normalize_path(path)

        path = '%s.detectors.txt' % splitext(file_abspath)[0]

        self.set(dumps(data), path)

        return path

    def _get_error(self, response):
        """
        Returns error in response if it exists
        :param dict response: AWS Response
        :return: Error message if present, None otherwise
        :rtype: string
        """
        if 'Error' in response:
            return response['Error']['Message'] if 'Message' in response['Error'] else response['Error']

        return None

    def _handle_error(self, response):
        """
        Logs error if necessary
        :param dict response: AWS Response
        """
        if self._get_error(response):
            logger.warn("[STORAGE] error occured while storing data: %s" % self._get_error(response))

    def _get_config(self, config_key, default=None):
        """
        Retrieve specific config based on prefix
        :param string config_key: Requested config
        :param default: Default value if not found
        :return: Resolved config value
        """
        return getattr(self.context.config, '%s_%s' % (self.config_prefix, config_key))

    def _normalize_path(self, path):
        """
        Adapts path based on configuration (root_path for instance)
        :param string path: Path to adapt
        :return: Adapted path
        :rtype: string
        """
        path = path.lstrip('/')  # Remove leading '/'
        path_segments = [path]

        root_path = self._get_config('ROOT_PATH')
        if root_path and root_path is not '':
            path_segments.insert(0, root_path)

        if self.is_auto_webp:
            path_segments.append("webp")

        return join(path_segments[0], *path_segments[1:]).lstrip('/') if len(path_segments) > 1 else path_segments[0]
<commit_msg>Add missing argument to Bucket construction when instantiating storage<commit_after># coding: utf-8

# Copyright (c) 2015-2016, thumbor-community
# Use of this source code is governed by the MIT license that can be
# found in the LICENSE file.

from json import loads, dumps
from os.path import join, splitext
from datetime import datetime
from dateutil.tz import tzutc

from tornado.concurrent import return_future
from thumbor.utils import logger

from .bucket import Bucket

class AwsStorage():
    """
    Base storage class
    """
    @property
    def is_auto_webp(self):
        """
        Determines based on context whether we automatically use webp or not
        :return: Use WebP?
        :rtype: bool
        """
        return self.context.config.AUTO_WEBP and hasattr(self.context, 'request') and self.context.request.accepts_webp

    @property
    def storage(self):
        """
        Instantiates bucket based on configuration
        :return: The bucket
        :rtype: Bucket
        """
        return Bucket(self._get_config('BUCKET'), self.context.config.get('TC_AWS_REGION'),
                      self.context.config.get('TC_AWS_ENDPOINT'))

    def __init__(self, context, config_prefix):
        """
        Constructor
        :param Context context: An instance of thumbor's context
        :param string config_prefix: Prefix used to load configuration values
        """
        self.config_prefix = config_prefix
        self.context = context

    @return_future
    def get(self, path, callback):
        """
        Gets data at path
        :param string path: Path for data
        :param callable callback: Callback function for once the retrieval is done
        """
        file_abspath = self._normalize_path(path)

        self.storage.get(file_abspath, callback=callback)

    @return_future
    def set(self, bytes, abspath, callback=None):
        """
        Stores data at given path
        :param bytes bytes: Data to store
        :param string abspath: Path to store the data at
        :return: Path where the data is stored
        :rtype: string
        """
        metadata = {}

        if self.config_prefix is 'TC_AWS_RESULT_STORAGE' and self.context.config.get('TC_AWS_STORE_METADATA'):
            metadata = dict(self.context.headers)

        self.storage.put(
            abspath,
            bytes,
            metadata=metadata,
            reduced_redundancy=self.context.config.get('TC_AWS_STORAGE_RRS', False),
            encrypt_key=self.context.config.get('TC_AWS_STORAGE_SSE', False),
            callback=callback,
        )

    def remove(self, path):
        """
        Deletes data at path
        :param string path: Path to delete
        :return: Whether deletion is successful or not
        :rtype: bool
        """
        yield self.storage.delete(path)
        return

    @return_future
    def exists(self, path, callback):
        """
        Tells if data exists at given path
        :param string path: Path to check
        :param callable callback: Callback function for once the check is done
        """
        file_abspath = self._normalize_path(path)

        def return_data(file_key):
            if not file_key or self._get_error(file_key):
                callback(False)
            else:
                callback(True)

        self.storage.get(file_abspath, callback=return_data)

    def is_expired(self, key):
        """
        Tells whether key has expired
        :param string key: Path to check
        :return: Whether it is expired or not
        :rtype: bool
        """
        if key and self._get_error(key) is None and 'LastModified' in key:
            expire_in_seconds = self.context.config.get('STORAGE_EXPIRATION_SECONDS', 3600)

            # Never expire
            if expire_in_seconds is None or expire_in_seconds == 0:
                return False

            timediff = datetime.now(tzutc()) - key['LastModified']

            return timediff.seconds > self.context.config.get('STORAGE_EXPIRATION_SECONDS', 3600)
        else:
            #If our key is bad just say we're expired
            return True

    @return_future
    def last_updated(self, callback):
        """
        Tells when the image has last been updated
        :param callable callback: Callback function for once the retrieval is done
        """
        path = self.context.request.url
        file_abspath = self._normalize_path(path)

        def on_file_fetched(file):
            if not file or self._get_error(file) or self.is_expired(file) or 'LastModified' not in file:
                logger.warn("[AwsStorage] s3 key not found at %s" % file_abspath)
                callback(None)
            else:
                callback(file['LastModified'])

        self.storage.get(file_abspath, callback=on_file_fetched)

    @return_future
    def get_crypto(self, path, callback):
        """
        Retrieves crypto data at path
        :param string path: Path to search for crypto data
        :param callable callback: Callback function for once the retrieval is done
        """
        file_abspath = self._normalize_path(path)
        crypto_path = "%s.txt" % (splitext(file_abspath)[0])

        def return_data(file_key):
            if not file_key or self._get_error(file_key) or self.is_expired(file_key) or 'Body' not in file_key:
                logger.warn("[STORAGE] s3 key not found at %s" % crypto_path)
                callback(None)
            else:
                callback(file_key['Body'])

        self.storage.get(crypto_path, callback=return_data)

    def put_crypto(self, path):
        """
        Stores crypto data at given path
        :param string path: Path to store the data at
        :return: Path where the crypto data is stored
        """
        if not self.context.config.STORES_CRYPTO_KEY_FOR_EACH_IMAGE:
            return

        if not self.context.server.security_key:
            raise RuntimeError("STORES_CRYPTO_KEY_FOR_EACH_IMAGE can't be True if no SECURITY_KEY specified")

        file_abspath = self._normalize_path(path)
        crypto_path = '%s.txt' % splitext(file_abspath)[0]

        self.set(self.context.server.security_key, crypto_path)

        return crypto_path

    @return_future
    def get_detector_data(self, path, callback):
        """
        Retrieves detector data from storage
        :param string path: Path where the data is stored
        :param callable callback: Callback function for once the retrieval is done
        """
        file_abspath = self._normalize_path(path)
        path = '%s.detectors.txt' % splitext(file_abspath)[0]

        def return_data(file_key):
            if not file_key or self._get_error(file_key) or self.is_expired(file_key) or 'Body' not in file_key:
                logger.warn("[AwsStorage] s3 key not found at %s" % path)
                callback(None)
            else:
                callback(loads(file_key['Body'].read()))

        self.storage.get(path, callback=return_data)

    def put_detector_data(self, path, data):
        """
        Stores detector data at given path
        :param string path: Path to store the data at
        :param bytes data:  Data to store
        :return: Path where the data is stored
        :rtype: string
        """
        file_abspath = self._normalize_path(path)

        path = '%s.detectors.txt' % splitext(file_abspath)[0]

        self.set(dumps(data), path)

        return path

    def _get_error(self, response):
        """
        Returns error in response if it exists
        :param dict response: AWS Response
        :return: Error message if present, None otherwise
        :rtype: string
        """
        if 'Error' in response:
            return response['Error']['Message'] if 'Message' in response['Error'] else response['Error']

        return None

    def _handle_error(self, response):
        """
        Logs error if necessary
        :param dict response: AWS Response
        """
        if self._get_error(response):
            logger.warn("[STORAGE] error occured while storing data: %s" % self._get_error(response))

    def _get_config(self, config_key, default=None):
        """
        Retrieve specific config based on prefix
        :param string config_key: Requested config
        :param default: Default value if not found
        :return: Resolved config value
        """
        return getattr(self.context.config, '%s_%s' % (self.config_prefix, config_key))

    def _normalize_path(self, path):
        """
        Adapts path based on configuration (root_path for instance)
        :param string path: Path to adapt
        :return: Adapted path
        :rtype: string
        """
        path = path.lstrip('/')  # Remove leading '/'
        path_segments = [path]

        root_path = self._get_config('ROOT_PATH')
        if root_path and root_path is not '':
            path_segments.insert(0, root_path)

        if self.is_auto_webp:
            path_segments.append("webp")

        return join(path_segments[0], *path_segments[1:]).lstrip('/') if len(path_segments) > 1 else path_segments[0]
<|endoftext|>


<commit_before>__version__ = '0.2.5'
<commit_msg>Bumped version<commit_after>__version__ = '0.2.6'
<|endoftext|>


<commit_before># Copyright (c) 2012-2013 Rackspace Hosting
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.
"""
Unit Tests for cells scheduler filters.
"""

from nova.cells import filters
from nova import context
from nova.db.sqlalchemy import models
from nova import test
from nova.tests.cells import fakes

class FiltersTestCase(test.NoDBTestCase):
    """Makes sure the proper filters are in the directory."""

    def test_all_filters(self):
        filter_classes = filters.all_filters()
        class_names = [cls.__name__ for cls in filter_classes]
        self.assertIn("TargetCellFilter", class_names)

class _FilterTestClass(test.NoDBTestCase):
    """Base class for testing individual filter plugins."""
    filter_cls_name = None

    def setUp(self):
        super(_FilterTestClass, self).setUp()
        fakes.init(self)
        self.msg_runner = fakes.get_message_runner('api-cell')
        self.scheduler = self.msg_runner.scheduler
        self.my_cell_state = self.msg_runner.state_manager.get_my_state()
        self.filter_handler = filters.CellFilterHandler()
        self.filter_classes = self.filter_handler.get_matching_classes(
                [self.filter_cls_name])
        self.context = context.RequestContext('fake', 'fake',
                                              is_admin=True)

    def _filter_cells(self, cells, filter_properties):
        return self.filter_handler.get_filtered_objects(self.filter_classes,
                                                        cells,
                                                        filter_properties)

class ImagePropertiesFilter(_FilterTestClass):
    filter_cls_name = \
        'nova.cells.filters.image_properties.ImagePropertiesFilter'

    def setUp(self):
        super(ImagePropertiesFilter, self).setUp()
        self.cell1 = models.Cell()
        self.cell2 = models.Cell()
        self.cell3 = models.Cell()
        self.cells = [self.cell1, self.cell2, self.cell3]
        for cell in self.cells:
            cell.capabilities = {}
        self.filter_props = {'context': self.context, 'request_spec': {}}

    def test_missing_image_properties(self):
        self.assertEqual(self.cells,
                         self._filter_cells(self.cells, self.filter_props))

    def test_missing_hypervisor_version_requires(self):
        self.filter_props['request_spec'] = {'image': {'properties': {}}}
        for cell in self.cells:
            cell.capabilities = {"prominent_hypervisor_version": set([u"6.2"])}
        self.assertEqual(self.cells,
                         self._filter_cells(self.cells, self.filter_props))

    def test_missing_hypervisor_version_in_cells(self):
        image = {'properties': {'hypervisor_version_requires': '>6.2.1'}}
        self.filter_props['request_spec'] = {'image': image}
        self.cell1.capabilities = {"prominent_hypervisor_version": set([])}
        self.assertEqual(self.cells,
                         self._filter_cells(self.cells, self.filter_props))

    def test_cells_matching_hypervisor_version(self):
        image = {'properties': {'hypervisor_version_requires': '>6.0, <=6.3'}}
        self.filter_props['request_spec'] = {'image': image}

        self.cell1.capabilities = {"prominent_hypervisor_version":
                set([u"6.2"])}
        self.cell2.capabilities = {"prominent_hypervisor_version":
                set([u"6.3"])}
        self.cell3.capabilities = {"prominent_hypervisor_version":
                set([u"6.0"])}

        self.assertEqual([self.cell1, self.cell2],
                         self._filter_cells(self.cells, self.filter_props))

        # assert again to verify filter doesn't mutate state
        # LP bug #1325705
        self.assertEqual([self.cell1, self.cell2],
                         self._filter_cells(self.cells, self.filter_props))

class TestTargetCellFilter(_FilterTestClass):
    filter_cls_name = 'nova.cells.filters.target_cell.TargetCellFilter'

    def test_missing_scheduler_hints(self):
        cells = [1, 2, 3]
        # No filtering
        filter_props = {'context': self.context}
        self.assertEqual(cells, self._filter_cells(cells, filter_props))

    def test_no_target_cell_hint(self):
        cells = [1, 2, 3]
        filter_props = {'scheduler_hints': {},
                        'context': self.context}
        # No filtering
        self.assertEqual(cells, self._filter_cells(cells, filter_props))

    def test_target_cell_specified_me(self):
        cells = [1, 2, 3]
        target_cell = 'fake!cell!path'
        current_cell = 'fake!cell!path'
        filter_props = {'scheduler_hints': {'target_cell': target_cell},
                        'routing_path': current_cell,
                        'scheduler': self.scheduler,
                        'context': self.context}
        # Only myself in the list.
        self.assertEqual([self.my_cell_state],
                         self._filter_cells(cells, filter_props))

    def test_target_cell_specified_me_but_not_admin(self):
        ctxt = context.RequestContext('fake', 'fake')
        cells = [1, 2, 3]
        target_cell = 'fake!cell!path'
        current_cell = 'fake!cell!path'
        filter_props = {'scheduler_hints': {'target_cell': target_cell},
                        'routing_path': current_cell,
                        'scheduler': self.scheduler,
                        'context': ctxt}
        # No filtering, because not an admin.
        self.assertEqual(cells, self._filter_cells(cells, filter_props))

    def test_target_cell_specified_not_me(self):
        info = {}

        def _fake_build_instances(ctxt, cell, sched_kwargs):
            info['ctxt'] = ctxt
            info['cell'] = cell
            info['sched_kwargs'] = sched_kwargs

        self.stubs.Set(self.msg_runner, 'build_instances',
                       _fake_build_instances)
        cells = [1, 2, 3]
        target_cell = 'fake!cell!path'
        current_cell = 'not!the!same'
        filter_props = {'scheduler_hints': {'target_cell': target_cell},
                        'routing_path': current_cell,
                        'scheduler': self.scheduler,
                        'context': self.context,
                        'host_sched_kwargs': 'meow',
                        'cell_scheduler_method': 'build_instances'}
        # None is returned to bypass further scheduling.
        self.assertIsNone(self._filter_cells(cells, filter_props))
        # The filter should have re-scheduled to the child cell itself.
        expected_info = {'ctxt': self.context,
                         'cell': 'fake!cell!path',
                         'sched_kwargs': 'meow'}
        self.assertEqual(expected_info, info)
<commit_msg>Remove unused cell_scheduler_method<commit_after># Copyright (c) 2012-2013 Rackspace Hosting
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.
"""
Unit Tests for cells scheduler filters.
"""

from nova.cells import filters
from nova import context
from nova.db.sqlalchemy import models
from nova import test
from nova.tests.cells import fakes

class FiltersTestCase(test.NoDBTestCase):
    """Makes sure the proper filters are in the directory."""

    def test_all_filters(self):
        filter_classes = filters.all_filters()
        class_names = [cls.__name__ for cls in filter_classes]
        self.assertIn("TargetCellFilter", class_names)

class _FilterTestClass(test.NoDBTestCase):
    """Base class for testing individual filter plugins."""
    filter_cls_name = None

    def setUp(self):
        super(_FilterTestClass, self).setUp()
        fakes.init(self)
        self.msg_runner = fakes.get_message_runner('api-cell')
        self.scheduler = self.msg_runner.scheduler
        self.my_cell_state = self.msg_runner.state_manager.get_my_state()
        self.filter_handler = filters.CellFilterHandler()
        self.filter_classes = self.filter_handler.get_matching_classes(
                [self.filter_cls_name])
        self.context = context.RequestContext('fake', 'fake',
                                              is_admin=True)

    def _filter_cells(self, cells, filter_properties):
        return self.filter_handler.get_filtered_objects(self.filter_classes,
                                                        cells,
                                                        filter_properties)

class ImagePropertiesFilter(_FilterTestClass):
    filter_cls_name = \
        'nova.cells.filters.image_properties.ImagePropertiesFilter'

    def setUp(self):
        super(ImagePropertiesFilter, self).setUp()
        self.cell1 = models.Cell()
        self.cell2 = models.Cell()
        self.cell3 = models.Cell()
        self.cells = [self.cell1, self.cell2, self.cell3]
        for cell in self.cells:
            cell.capabilities = {}
        self.filter_props = {'context': self.context, 'request_spec': {}}

    def test_missing_image_properties(self):
        self.assertEqual(self.cells,
                         self._filter_cells(self.cells, self.filter_props))

    def test_missing_hypervisor_version_requires(self):
        self.filter_props['request_spec'] = {'image': {'properties': {}}}
        for cell in self.cells:
            cell.capabilities = {"prominent_hypervisor_version": set([u"6.2"])}
        self.assertEqual(self.cells,
                         self._filter_cells(self.cells, self.filter_props))

    def test_missing_hypervisor_version_in_cells(self):
        image = {'properties': {'hypervisor_version_requires': '>6.2.1'}}
        self.filter_props['request_spec'] = {'image': image}
        self.cell1.capabilities = {"prominent_hypervisor_version": set([])}
        self.assertEqual(self.cells,
                         self._filter_cells(self.cells, self.filter_props))

    def test_cells_matching_hypervisor_version(self):
        image = {'properties': {'hypervisor_version_requires': '>6.0, <=6.3'}}
        self.filter_props['request_spec'] = {'image': image}

        self.cell1.capabilities = {"prominent_hypervisor_version":
                set([u"6.2"])}
        self.cell2.capabilities = {"prominent_hypervisor_version":
                set([u"6.3"])}
        self.cell3.capabilities = {"prominent_hypervisor_version":
                set([u"6.0"])}

        self.assertEqual([self.cell1, self.cell2],
                         self._filter_cells(self.cells, self.filter_props))

        # assert again to verify filter doesn't mutate state
        # LP bug #1325705
        self.assertEqual([self.cell1, self.cell2],
                         self._filter_cells(self.cells, self.filter_props))

class TestTargetCellFilter(_FilterTestClass):
    filter_cls_name = 'nova.cells.filters.target_cell.TargetCellFilter'

    def test_missing_scheduler_hints(self):
        cells = [1, 2, 3]
        # No filtering
        filter_props = {'context': self.context}
        self.assertEqual(cells, self._filter_cells(cells, filter_props))

    def test_no_target_cell_hint(self):
        cells = [1, 2, 3]
        filter_props = {'scheduler_hints': {},
                        'context': self.context}
        # No filtering
        self.assertEqual(cells, self._filter_cells(cells, filter_props))

    def test_target_cell_specified_me(self):
        cells = [1, 2, 3]
        target_cell = 'fake!cell!path'
        current_cell = 'fake!cell!path'
        filter_props = {'scheduler_hints': {'target_cell': target_cell},
                        'routing_path': current_cell,
                        'scheduler': self.scheduler,
                        'context': self.context}
        # Only myself in the list.
        self.assertEqual([self.my_cell_state],
                         self._filter_cells(cells, filter_props))

    def test_target_cell_specified_me_but_not_admin(self):
        ctxt = context.RequestContext('fake', 'fake')
        cells = [1, 2, 3]
        target_cell = 'fake!cell!path'
        current_cell = 'fake!cell!path'
        filter_props = {'scheduler_hints': {'target_cell': target_cell},
                        'routing_path': current_cell,
                        'scheduler': self.scheduler,
                        'context': ctxt}
        # No filtering, because not an admin.
        self.assertEqual(cells, self._filter_cells(cells, filter_props))

    def test_target_cell_specified_not_me(self):
        info = {}

        def _fake_build_instances(ctxt, cell, sched_kwargs):
            info['ctxt'] = ctxt
            info['cell'] = cell
            info['sched_kwargs'] = sched_kwargs

        self.stubs.Set(self.msg_runner, 'build_instances',
                       _fake_build_instances)
        cells = [1, 2, 3]
        target_cell = 'fake!cell!path'
        current_cell = 'not!the!same'
        filter_props = {'scheduler_hints': {'target_cell': target_cell},
                        'routing_path': current_cell,
                        'scheduler': self.scheduler,
                        'context': self.context,
                        'host_sched_kwargs': 'meow'}
        # None is returned to bypass further scheduling.
        self.assertIsNone(self._filter_cells(cells, filter_props))
        # The filter should have re-scheduled to the child cell itself.
        expected_info = {'ctxt': self.context,
                         'cell': 'fake!cell!path',
                         'sched_kwargs': 'meow'}
        self.assertEqual(expected_info, info)
<|endoftext|>


<commit_before>from datetime import date, datetime, timedelta
from zlib import crc32
from django.core.cache import cache
from django.db.models import Sum
from accounts.models import UserProfile
from race.models import Map, MapType, Run, BestRun, Server
from lib.chunks import chunks
from celery.task import task
from tml.tml import Teemap

@task(rate_limit='60/m', ignore_result=True)
def redo_ranks(run_id):
	logger = redo_ranks.get_logger()
	try:
		user_run = Run.objects.get(pk=run_id)
	except Run.DoesNotExist:
		logger.error("How is that possible? (Run doesn't exist)")
		return False
	map_obj = user_run.map
	user_best = BestRun.objects.get(map=map_obj, user=user_run.user)
	if not user_best.run_id == user_run.id:
		return
	runs = BestRun.objects.filter(map=map_obj)
	# ranked = player that receives points for his place
	ranked_count = len(BestRun.SCORING)
	# exclude anonymous and banned users from scoring
	ranked = runs.exclude(user__is_active=False)
	ranked = ranked.order_by('run__time')[:ranked_count]
	try:
		if user_run.time >= ranked[ranked_count-1].run.time:
			return
	except IndexError:
		pass
	i = 0
	for run in ranked:
		run.points = BestRun.SCORING[i]
		run.save()
		# FIXME it's 3 AM, sorry for that
		run.user.profile.points = BestRun.objects.filter(user=run.user).aggregate(
			Sum('points')
		)['points__sum']
		run.user.profile.save()
		i += 1
	runs.exclude(id__in=ranked.values_list('id', flat=True)).update(
		points=0
	)
	logger.info("Processed rank for \"{0}\" map.".format(map_obj))

@task(rate_limit='10/m', ignore_result=True)
def rebuild_map_rank(map_id):
	map_obj = Map.objects.get(pk=map_id)
	runs = BestRun.objects.filter(map=map_obj)
	# ranked = player that receives points for his place
	ranked_count = len(BestRun.SCORING)
	# exclude anonymous and banned users from scoring
	ranked = runs.exclude(user__is_active=False) \
		.order_by('run__time')[:ranked_count]
	i = 0
	for run in ranked:
		run.points = BestRun.SCORING[i]
		run.save()
		i += 1
	runs.exclude(id__in=ranked.values_list('id', flat=True)).update(
		points=0
	)
	logger = rebuild_map_rank.get_logger()
	logger.info("Rebuilt rank for map \"{0}\".".format(map_obj.name))

@task(rate_limit='1/m', ignore_result=True)
def rebuild_global_rank():
	# FIXME oh man, fix me.
	runners = UserProfile.objects.exclude(user__pk=0)
	for runner in runners:
		runner.points = BestRun.objects.filter(user=runner.user).aggregate(
			Sum('points')
		)['points__sum'] or 0
		runner.save()
	logger = rebuild_global_rank.get_logger()
	logger.info("Rebuilt global rank.")

class index_factory(object):
	INDEXES = dict(
		DEATHTILE = 2,
		UNHOOKABLE = 3,
		RED_FLAG = 195,
		BLUE_FLAG = 196,
		SHIELD = 197,
		HEART = 198,
		GRENADE = 200,
	)

	def __getattr__(self, attr):
		try:
			return self.INDEXES.get(attr)
		except TypeError:
			raise AttributeError(attr)
indexes = index_factory()

@task(ignore_result=True)
def retrieve_map_details(map_id):
	"""
	WARNING!
	This task is CPU/(and highly) RAM expensive.

	Checks for presence of specified tiles, counts some of them
	and at the end takes a beautiful photo of the map.

	Thanks for your help, erdbeere!
	"""
	logger = retrieve_map_details.get_logger()
	try:
		map_obj = Map.objects.get(pk=map_id)
	except Map.DoesNotExist:
		logger.error("Are you kidding me? (map doesn't exist)")
		return False

	# I actually can use that! Thanks, erd!
	teemap = Teemap().load(map_obj.map_file.path)
	logger.info("Loaded \"{0}\" map.".format(map_obj.name))

	logger.info("Counting map items...")
	has_unhookables = has_deathtiles = is_fastcap = False
	shield_count = heart_count = grenade_count = 0
	for tile in teemap.gamelayer.tiles:
		if tile.index == indexes.DEATHTILE:
			has_deathtiles = True
		elif tile.index == indexes.UNHOOKABLE:
			has_unhookables = True
		elif tile.index in (indexes.RED_FLAG, indexes.BLUE_FLAG):
			is_fastcap = True
		elif tile.index == indexes.SHIELD:
			shield_count += 1
		elif tile.index == indexes.HEART:
			heart_count += 1
		elif tile.index == indexes.GRENADE:
			grenade_count += 1
	map_obj.has_unhookables = has_unhookables
	map_obj.has_deathtiles = has_deathtiles
	map_obj.shield_count = shield_count
	map_obj.heart_count = heart_count
	map_obj.grenade_count = grenade_count
	logger.info("Finished counting map items.")

	# DISABLED due to huge (counted in GiBs) memory usage
	# logger.info("Rendering map screenshot.")
	# map_image = teemap.render(gamelayer_on_top=True)
	# map_image.save('{0}images/maps/full/{1}.png'.format(settings.MEDIA_ROOT,
	# 	map_obj.name))
	# logger.info("Finished rendering map screenshot.")
	# map_obj.has_image = True

	logger.info("Generating map CRC...")
	map_obj.crc = '{0:x}'.format(crc32(map_obj.map_file.read()) & 0xffffffff)
	map_obj.map_file.close()

	if is_fastcap:
		logger.info("Turning map into fastcap type...")
		map_obj.map_type = MapType.objects.get(slug='fastcap')
		logger.info("Creating a non-weapon twin...")
		new_map = Map(
			name="{0}-no-weapons".format(map_obj.name),
			author=map_obj.author,
			added_by=map_obj.added_by,
			map_file=map_obj.map_file,
			crc=map_obj.crc,
			map_type=MapType.objects.get(slug='fastcap-no-weapons'),
			has_unhookables=has_unhookables,
			has_deathtiles=has_deathtiles,
		)
		new_map.save()
	map_obj.save()
	logger.info("Finished processing \"{0}\" map.".format(map_obj.name))

@task(ignore_result=True)
def set_server_map(server_id, map_id):
	try:
		map_obj = Map.objects.get(pk=map_id)
		server = Server.objects.get(pk=server_id)
	except (Map.DoesNotExist, Server.DoesNotExist):
		return False
	server.played_map = map_obj
	server.save()

@task(ignore_result=True)
def update_user_points_history():
	# everyday, around 4:30 AM
	users = UserProfile.objects.values_list('id', flat=True)
	for chunk in list(chunks(users, 200)):
		update_user_points_history_chunked.delay(chunk)

@task(ignore_result=True)
def update_user_points_history_chunked(chunk):
	today = date.today()
	for user_id in chunk:
		user = UserProfile.objects.get(pk=user_id)
		if user.points_history:
			yesterday_points = user.points_history[-1][1] # wee!
			user.points_progress = user.points - yesterday_points
		else:
			user.points_progress = user.points
			# let's make a list
			user.points_history = []
		user.points_history.append((today, user.points))
		user.save()

@task(ignore_result=True)
def update_yesterday_runs():
	# everyday, around 0:00 AM
	yesterday = datetime.today() - timedelta(days=1)
	runs_yesterday = Run.objects.filter(created_at__range=
		(datetime.combine(yesterday, time.min),
		datetime.combine(yesterday, time.max)))
	cache.set('runs_yesterday', runs_yesterday, 0)

@task(ignore_result=True)
def update_totals():
	# every 15 minutes
	total_runs = Run.objects.count()
	total_playtime = Run.objects.aggregate(Sum('time'))['time__sum']
	cache.set('total_runs', runs_total, 0)
	cache.set('total_playtime', runs_total, 0)
<commit_msg>fixed task updating totals on homepage<commit_after>from datetime import date, datetime, timedelta
from zlib import crc32
from django.core.cache import cache
from django.db.models import Sum
from accounts.models import UserProfile
from race.models import Map, MapType, Run, BestRun, Server
from lib.chunks import chunks
from celery.task import task
from tml.tml import Teemap

@task(rate_limit='60/m', ignore_result=True)
def redo_ranks(run_id):
	logger = redo_ranks.get_logger()
	try:
		user_run = Run.objects.get(pk=run_id)
	except Run.DoesNotExist:
		logger.error("How is that possible? (Run doesn't exist)")
		return False
	map_obj = user_run.map
	user_best = BestRun.objects.get(map=map_obj, user=user_run.user)
	if not user_best.run_id == user_run.id:
		return
	runs = BestRun.objects.filter(map=map_obj)
	# ranked = player that receives points for his place
	ranked_count = len(BestRun.SCORING)
	# exclude anonymous and banned users from scoring
	ranked = runs.exclude(user__is_active=False)
	ranked = ranked.order_by('run__time')[:ranked_count]
	try:
		if user_run.time >= ranked[ranked_count-1].run.time:
			return
	except IndexError:
		pass
	i = 0
	for run in ranked:
		run.points = BestRun.SCORING[i]
		run.save()
		# FIXME it's 3 AM, sorry for that
		run.user.profile.points = BestRun.objects.filter(user=run.user).aggregate(
			Sum('points')
		)['points__sum']
		run.user.profile.save()
		i += 1
	runs.exclude(id__in=ranked.values_list('id', flat=True)).update(
		points=0
	)
	logger.info("Processed rank for \"{0}\" map.".format(map_obj))

@task(rate_limit='10/m', ignore_result=True)
def rebuild_map_rank(map_id):
	map_obj = Map.objects.get(pk=map_id)
	runs = BestRun.objects.filter(map=map_obj)
	# ranked = player that receives points for his place
	ranked_count = len(BestRun.SCORING)
	# exclude anonymous and banned users from scoring
	ranked = runs.exclude(user__is_active=False) \
		.order_by('run__time')[:ranked_count]
	i = 0
	for run in ranked:
		run.points = BestRun.SCORING[i]
		run.save()
		i += 1
	runs.exclude(id__in=ranked.values_list('id', flat=True)).update(
		points=0
	)
	logger = rebuild_map_rank.get_logger()
	logger.info("Rebuilt rank for map \"{0}\".".format(map_obj.name))

@task(rate_limit='1/m', ignore_result=True)
def rebuild_global_rank():
	# FIXME oh man, fix me.
	runners = UserProfile.objects.exclude(user__pk=0)
	for runner in runners:
		runner.points = BestRun.objects.filter(user=runner.user).aggregate(
			Sum('points')
		)['points__sum'] or 0
		runner.save()
	logger = rebuild_global_rank.get_logger()
	logger.info("Rebuilt global rank.")

class index_factory(object):
	INDEXES = dict(
		DEATHTILE = 2,
		UNHOOKABLE = 3,
		RED_FLAG = 195,
		BLUE_FLAG = 196,
		SHIELD = 197,
		HEART = 198,
		GRENADE = 200,
	)

	def __getattr__(self, attr):
		try:
			return self.INDEXES.get(attr)
		except TypeError:
			raise AttributeError(attr)
indexes = index_factory()

@task(ignore_result=True)
def retrieve_map_details(map_id):
	"""
	WARNING!
	This task is CPU/(and highly) RAM expensive.

	Checks for presence of specified tiles, counts some of them
	and at the end takes a beautiful photo of the map.

	Thanks for your help, erdbeere!
	"""
	logger = retrieve_map_details.get_logger()
	try:
		map_obj = Map.objects.get(pk=map_id)
	except Map.DoesNotExist:
		logger.error("Are you kidding me? (map doesn't exist)")
		return False

	# I actually can use that! Thanks, erd!
	teemap = Teemap().load(map_obj.map_file.path)
	logger.info("Loaded \"{0}\" map.".format(map_obj.name))

	logger.info("Counting map items...")
	has_unhookables = has_deathtiles = is_fastcap = False
	shield_count = heart_count = grenade_count = 0
	for tile in teemap.gamelayer.tiles:
		if tile.index == indexes.DEATHTILE:
			has_deathtiles = True
		elif tile.index == indexes.UNHOOKABLE:
			has_unhookables = True
		elif tile.index in (indexes.RED_FLAG, indexes.BLUE_FLAG):
			is_fastcap = True
		elif tile.index == indexes.SHIELD:
			shield_count += 1
		elif tile.index == indexes.HEART:
			heart_count += 1
		elif tile.index == indexes.GRENADE:
			grenade_count += 1
	map_obj.has_unhookables = has_unhookables
	map_obj.has_deathtiles = has_deathtiles
	map_obj.shield_count = shield_count
	map_obj.heart_count = heart_count
	map_obj.grenade_count = grenade_count
	logger.info("Finished counting map items.")

	# DISABLED due to huge (counted in GiBs) memory usage
	# logger.info("Rendering map screenshot.")
	# map_image = teemap.render(gamelayer_on_top=True)
	# map_image.save('{0}images/maps/full/{1}.png'.format(settings.MEDIA_ROOT,
	# 	map_obj.name))
	# logger.info("Finished rendering map screenshot.")
	# map_obj.has_image = True

	logger.info("Generating map CRC...")
	map_obj.crc = '{0:x}'.format(crc32(map_obj.map_file.read()) & 0xffffffff)
	map_obj.map_file.close()

	if is_fastcap:
		logger.info("Turning map into fastcap type...")
		map_obj.map_type = MapType.objects.get(slug='fastcap')
		logger.info("Creating a non-weapon twin...")
		new_map = Map(
			name="{0}-no-weapons".format(map_obj.name),
			author=map_obj.author,
			added_by=map_obj.added_by,
			map_file=map_obj.map_file,
			crc=map_obj.crc,
			map_type=MapType.objects.get(slug='fastcap-no-weapons'),
			has_unhookables=has_unhookables,
			has_deathtiles=has_deathtiles,
		)
		new_map.save()
	map_obj.save()
	logger.info("Finished processing \"{0}\" map.".format(map_obj.name))

@task(ignore_result=True)
def set_server_map(server_id, map_id):
	try:
		map_obj = Map.objects.get(pk=map_id)
		server = Server.objects.get(pk=server_id)
	except (Map.DoesNotExist, Server.DoesNotExist):
		return False
	server.played_map = map_obj
	server.save()

@task(ignore_result=True)
def update_user_points_history():
	# everyday, around 4:30 AM
	users = UserProfile.objects.values_list('id', flat=True)
	for chunk in list(chunks(users, 200)):
		update_user_points_history_chunked.delay(chunk)

@task(ignore_result=True)
def update_user_points_history_chunked(chunk):
	today = date.today()
	for user_id in chunk:
		user = UserProfile.objects.get(pk=user_id)
		if user.points_history:
			yesterday_points = user.points_history[-1][1] # wee!
			user.points_progress = user.points - yesterday_points
		else:
			user.points_progress = user.points
			# let's make a list
			user.points_history = []
		user.points_history.append((today, user.points))
		user.save()

@task(ignore_result=True)
def update_yesterday_runs():
	# everyday, around 0:00 AM
	yesterday = datetime.today() - timedelta(days=1)
	runs_yesterday = Run.objects.filter(created_at__range=
		(datetime.combine(yesterday, time.min),
		datetime.combine(yesterday, time.max)))
	cache.set('runs_yesterday', runs_yesterday, 0)

@task(ignore_result=True)
def update_totals():
	# every 15 minutes
	total_runs = Run.objects.count()
	total_playtime = Run.objects.aggregate(Sum('time'))['time__sum']
	cache.set('total_runs', total_runs, 0)
	cache.set('total_playtime', total_playtime, 0)
<|endoftext|>


<commit_before>#!/usr/bin/env python
'''
This tool reads a firmwre image from nrf_spi.asm
'''
from __future__ import print_function

import argparse
import sys
import serial
from binascii import unhexlify

class NRFProgrammer(object):
    ''' Interface with the nrf_spi programmer hardware over serial '''

    VERSION = '00'
    ACCESS_MODE = '01'
    SPI = '02'
    ADDRESS = '03'
    PROGRAM = '04'
    READ = '05'
    RESET = '06'

    ACCESS_MODE_ENTER = '01'
    ACCESS_MODE_EXIT = '00'

    def __init__(self, args):
        self.args = args
        try:
            self.uart = serial.Serial(self.args.tty, self.args.baudrate)
        except serial.SerialException as error:
            print("Unable to open port %s: %s" % (self.args.tty, error))
            sys.exit(1)

    def command(self, command, arguments=""):
        ''' Send a command to the programmer and read the response '''
        message = "H" + command + arguments + "\n"
        self.uart.write(message)
        data = self.uart.readline()
        data = data.decode('ascii', errors='replace').strip()
        data = data.replace(" ", "")
        code = data[0]
        if code != "R":
            raise ValueError("Command {} failed".format(command))
        return data[1:]

    def version(self):
        ''' Return the version number of the nrf_spi programmer hardware '''
        response = self.command(self.VERSION)
        return response

    def read(self, start, length=0x10000):
        ''' Read *length* bytes, starting at *start* from the NRF flash '''
        self.command(self.ACCESS_MODE, self.ACCESS_MODE_ENTER)
        self.command(self.ADDRESS, "{:04X}".format(start))

        data = ""
        while length:
            if length >= 256:
                size = 0
            else:
                size = length
            data += self.command(self.READ, "{:02X}".format(size))
            if size == 0:
                size = 256
            length -= size

        self.command(self.ACCESS_MODE, self.ACCESS_MODE_EXIT)
        return data

def parse_commandline():
    ''' Command line argument parsing '''
    parser = argparse.ArgumentParser(
        description="Read a firmware image from the NRF24LE1.")

    parser.add_argument("-b", "--baudrate", type=int, default=115200,
        help='Baudrate to use. Default is 115200.')

    parser.add_argument("tty", nargs="?", default="/dev/ttyUSB0",
        help="serial port to use. ")

    parser.add_argument("binfile", nargs=1, type=argparse.FileType('wb'),
        help="the file name of the binary image file to write")

    return parser.parse_args()

def read_nrf_firmware():
    ''' Interface with nrf_spi.asm '''

    args = parse_commandline()
    programmer = NRFProgrammer(args)

    firmware = programmer.read(0, 0x4000)
    binary = unhexlify(firmware)
    args.binfile[0].write(binary)

def main():
    ''' Program start '''
    try:
        read_nrf_firmware()
    except KeyboardInterrupt:
        print("")
        sys.exit(0)

if __name__ == '__main__':
    main()
<commit_msg>Fix spelling<commit_after>#!/usr/bin/env python
'''
This tool reads a firmware image from nrf_spi.asm
'''
from __future__ import print_function

import argparse
import sys
import serial
from binascii import unhexlify

class NRFProgrammer(object):
    ''' Interface with the nrf_spi programmer hardware over serial '''

    VERSION = '00'
    ACCESS_MODE = '01'
    SPI = '02'
    ADDRESS = '03'
    PROGRAM = '04'
    READ = '05'
    RESET = '06'

    ACCESS_MODE_ENTER = '01'
    ACCESS_MODE_EXIT = '00'

    def __init__(self, args):
        self.args = args
        try:
            self.uart = serial.Serial(self.args.tty, self.args.baudrate)
        except serial.SerialException as error:
            print("Unable to open port %s: %s" % (self.args.tty, error))
            sys.exit(1)

    def command(self, command, arguments=""):
        ''' Send a command to the programmer and read the response '''
        message = "H" + command + arguments + "\n"
        self.uart.write(message)
        data = self.uart.readline()
        data = data.decode('ascii', errors='replace').strip()
        data = data.replace(" ", "")
        code = data[0]
        if code != "R":
            raise ValueError("Command {} failed".format(command))
        return data[1:]

    def version(self):
        ''' Return the version number of the nrf_spi programmer hardware '''
        response = self.command(self.VERSION)
        return response

    def read(self, start, length=0x10000):
        ''' Read *length* bytes, starting at *start* from the NRF flash '''
        self.command(self.ACCESS_MODE, self.ACCESS_MODE_ENTER)
        self.command(self.ADDRESS, "{:04X}".format(start))

        data = ""
        while length:
            if length >= 256:
                size = 0
            else:
                size = length
            data += self.command(self.READ, "{:02X}".format(size))
            if size == 0:
                size = 256
            length -= size

        self.command(self.ACCESS_MODE, self.ACCESS_MODE_EXIT)
        return data

def parse_commandline():
    ''' Command line argument parsing '''
    parser = argparse.ArgumentParser(
        description="Read a firmware image from the NRF24LE1.")

    parser.add_argument("-b", "--baudrate", type=int, default=115200,
        help='Baudrate to use. Default is 115200.')

    parser.add_argument("tty", nargs="?", default="/dev/ttyUSB0",
        help="serial port to use. ")

    parser.add_argument("binfile", nargs=1, type=argparse.FileType('wb'),
        help="the file name of the binary image file to write")

    return parser.parse_args()

def read_nrf_firmware():
    ''' Interface with nrf_spi.asm '''

    args = parse_commandline()
    programmer = NRFProgrammer(args)

    firmware = programmer.read(0, 0x4000)
    binary = unhexlify(firmware)
    args.binfile[0].write(binary)

def main():
    ''' Program start '''
    try:
        read_nrf_firmware()
    except KeyboardInterrupt:
        print("")
        sys.exit(0)

if __name__ == '__main__':
    main()
<|endoftext|>