<commit_before>from __future__ import absolute_import, division, print_function

import uuid

import numpy as np

from glue.utils import nonpartial
from glue.core.exceptions import IncompatibleAttribute

from .multi_scatter import MultiColorScatter
from .layer_state import ScatterLayerState
from ..common.layer_artist import VispyLayerArtist

class ScatterLayerArtist(VispyLayerArtist):
    """
    A layer artist to render 3d scatter plots.
    """

    def __init__(self, vispy_viewer, layer=None, layer_state=None):

        super(ScatterLayerArtist, self).__init__(layer)

        self._clip_limits = None

        self._marker_keep = Ellipsis

        # Set data caches
        self._marker_data = None
        self._color_data = None
        self._size_data = None

        self.layer = layer or layer_state.layer
        self.vispy_viewer = vispy_viewer
        self.vispy_widget = vispy_viewer._vispy_widget

        # TODO: need to remove layers when layer artist is removed
        self._viewer_state = vispy_viewer.state
        self.state = layer_state or ScatterLayerState(layer=self.layer)
        if self.state not in self._viewer_state.layers:
            self._viewer_state.layers.append(self.state)

        # We create a unique ID for this layer artist, that will be used to
        # refer to the layer artist in the MultiColorScatter. We have to do this
        # rather than use self.id because we can't guarantee the latter is
        # unique.
        self.id = str(uuid.uuid4())

        # We need to use MultiColorScatter instance to store scatter plots, but
        # we should only have one per canvas. Therefore, we store the
        # MultiColorScatter instance in the vispy viewer instance.
        if not hasattr(self.vispy_widget, '_multiscat'):
            multiscat = MultiColorScatter()
            multiscat.set_gl_state(depth_test=False,
                                   blend=True,
                                   blend_func=('src_alpha', 'one_minus_src_alpha'))

            self.vispy_widget.add_data_visual(multiscat)
            self.vispy_widget._multiscat = multiscat

        self._multiscat = self.vispy_widget._multiscat
        self._multiscat.allocate(self.id)
        self._multiscat.set_zorder(self.id, self.get_zorder)

        try:
            self.state.add_callback('*', self._update_from_state, as_kwargs=True)
        except TypeError:  # glue-core >= 0.11
            self.state.add_global_callback(self._update_from_state)

        self._update_from_state(**self.state.as_dict())

        self._viewer_state.add_callback('x_att', nonpartial(self._update_data))
        self._viewer_state.add_callback('y_att', nonpartial(self._update_data))
        self._viewer_state.add_callback('z_att', nonpartial(self._update_data))

        self._update_data()

        self.visible = True

    @property
    def visual(self):
        return self._multiscat

    def _update_visibility(self):
        self._multiscat.set_visible(self.id, self.visible)
        self.redraw()

    def get_zorder(self):
        return self.zorder

    def redraw(self):
        """
        Redraw the Vispy canvas
        """
        self._multiscat._update()
        self.vispy_widget.canvas.update()

    def clear(self):
        """
        Remove the layer artist from the visualization
        """
        self._multiscat.set_data_values(self.id, [], [], [])

    def update(self):
        """
        Update the visualization to reflect the underlying data
        """
        self.redraw()
        self._changed = False

    def _update_from_state(self, **props):
        if any('size' in prop for prop in props):
            self._update_sizes()
        if any('color' in prop or 'cmap' in prop for prop in props):
            self._update_colors()
        if 'alpha' in props:
            self._update_alpha()
        self.redraw()

    def _update_sizes(self):
        if self.state.size_mode is None:
            pass
        elif self.state.size_mode == 'Fixed':
            self._multiscat.set_size(self.id, self.state.size * self.state.size_scaling)
        else:
            data = self.layer[self.state.size_attribute].ravel()
            data = data[self._marker_keep]
            if self.state.size_vmax == self.state.size_vmin:
                size = np.ones(data.shape) * 10
            else:
                size = (20 * (data - self.state.size_vmin) /
                        (self.state.size_vmax - self.state.size_vmin))
            size_data = size * self.state.size_scaling
            size_data[np.isnan(data)] = 0.
            self._multiscat.set_size(self.id, size_data)

    def _update_colors(self):
        if self.state.color_mode is None:
            pass
        elif self.state.color_mode == 'Fixed':
            self._multiscat.set_color(self.id, self.state.color)
        else:
            data = self.layer[self.state.cmap_attribute].ravel()
            data = data[self._marker_keep]
            if self.state.cmap_vmax == self.state.cmap_vmin:
                cmap_data = np.ones(data.shape) * 0.5
            else:
                cmap_data = ((data - self.state.cmap_vmin) /
                             (self.state.cmap_vmax - self.state.cmap_vmin))
            cmap_data = self.state.cmap(cmap_data)
            cmap_data[:, 3][np.isnan(data)] = 0.
            self._multiscat.set_color(self.id, cmap_data)

    def _update_alpha(self):
        self._multiscat.set_alpha(self.id, self.state.alpha)

    def _update_data(self):

        try:
            x = self.layer[self._viewer_state.x_att].ravel()
            y = self.layer[self._viewer_state.y_att].ravel()
            z = self.layer[self._viewer_state.z_att].ravel()
        except AttributeError:
            return
        except (IncompatibleAttribute, IndexError):
            # The following includes a call to self.clear()
            self.disable_invalid_attributes(self._viewer_state.x_att,
                                            self._viewer_state.y_att,
                                            self._viewer_state.z_att)
            return
        else:
            self._enabled = True

        if self._clip_limits is None:
            keep = Ellipsis
        else:
            xmin, xmax, ymin, ymax, zmin, zmax = self._clip_limits
            keep = (x >= xmin) & (x <= xmax) & (y >= ymin) & (y <= ymax) & (z >= zmin) & (z <= zmax)
            x, y, z = x[keep], y[keep], z[keep]

        self._marker_data = np.array([x, y, z]).transpose()
        self._marker_keep = keep

        # We need to make sure we update the sizes and colors in case
        # these were set as arrays, since the size of the data might have
        # changed (in the case of subsets)

        with self._multiscat.delay_update():
            self._multiscat.set_data_values(self.id, x, y, z)
            self._update_sizes()
            self._update_colors()

        self.redraw()

    @property
    def default_limits(self):
        if self._marker_data is None:
            raise ValueError("Data not yet set")
        dmin = np.nanmin(self._marker_data, axis=0)
        dmax = np.nanmax(self._marker_data, axis=0)
        # TODO: the following can be optimized
        return tuple(np.array([dmin, dmax]).transpose().ravel())

    def set_clip(self, limits):
        self._clip_limits = limits
        self._update_data()
<commit_msg>Make sure the layer list shows the colormap icon<commit_after>from __future__ import absolute_import, division, print_function

import uuid

import numpy as np

from glue.utils import nonpartial
from glue.core.exceptions import IncompatibleAttribute

from .multi_scatter import MultiColorScatter
from .layer_state import ScatterLayerState
from ..common.layer_artist import VispyLayerArtist

class ScatterLayerArtist(VispyLayerArtist):
    """
    A layer artist to render 3d scatter plots.
    """

    def __init__(self, vispy_viewer, layer=None, layer_state=None):

        super(ScatterLayerArtist, self).__init__(layer)

        self._clip_limits = None

        self._marker_keep = Ellipsis

        # Set data caches
        self._marker_data = None
        self._color_data = None
        self._size_data = None

        self.layer = layer or layer_state.layer
        self.vispy_viewer = vispy_viewer
        self.vispy_widget = vispy_viewer._vispy_widget

        # TODO: need to remove layers when layer artist is removed
        self._viewer_state = vispy_viewer.state
        self.state = layer_state or ScatterLayerState(layer=self.layer)
        if self.state not in self._viewer_state.layers:
            self._viewer_state.layers.append(self.state)

        # We create a unique ID for this layer artist, that will be used to
        # refer to the layer artist in the MultiColorScatter. We have to do this
        # rather than use self.id because we can't guarantee the latter is
        # unique.
        self.id = str(uuid.uuid4())

        # We need to use MultiColorScatter instance to store scatter plots, but
        # we should only have one per canvas. Therefore, we store the
        # MultiColorScatter instance in the vispy viewer instance.
        if not hasattr(self.vispy_widget, '_multiscat'):
            multiscat = MultiColorScatter()
            multiscat.set_gl_state(depth_test=False,
                                   blend=True,
                                   blend_func=('src_alpha', 'one_minus_src_alpha'))

            self.vispy_widget.add_data_visual(multiscat)
            self.vispy_widget._multiscat = multiscat

        self._multiscat = self.vispy_widget._multiscat
        self._multiscat.allocate(self.id)
        self._multiscat.set_zorder(self.id, self.get_zorder)

        try:
            self.state.add_callback('*', self._update_from_state, as_kwargs=True)
        except TypeError:  # glue-core >= 0.11
            self.state.add_global_callback(self._update_from_state)

        self._update_from_state(**self.state.as_dict())

        self._viewer_state.add_callback('x_att', nonpartial(self._update_data))
        self._viewer_state.add_callback('y_att', nonpartial(self._update_data))
        self._viewer_state.add_callback('z_att', nonpartial(self._update_data))

        self._update_data()

        self.visible = True

    @property
    def visual(self):
        return self._multiscat

    def _update_visibility(self):
        self._multiscat.set_visible(self.id, self.visible)
        self.redraw()

    def get_zorder(self):
        return self.zorder

    def get_layer_color(self):
        if self.state.color_mode == 'Fixed':
            return self.state.color
        else:
            return self.state.cmap

    def redraw(self):
        """
        Redraw the Vispy canvas
        """
        self._multiscat._update()
        self.vispy_widget.canvas.update()

    def clear(self):
        """
        Remove the layer artist from the visualization
        """
        self._multiscat.set_data_values(self.id, [], [], [])

    def update(self):
        """
        Update the visualization to reflect the underlying data
        """
        self.redraw()
        self._changed = False

    def _update_from_state(self, **props):
        if any('size' in prop for prop in props):
            self._update_sizes()
        if any('color' in prop or 'cmap' in prop for prop in props):
            self._update_colors()
        if 'alpha' in props:
            self._update_alpha()
        self.redraw()

    def _update_sizes(self):
        if self.state.size_mode is None:
            pass
        elif self.state.size_mode == 'Fixed':
            self._multiscat.set_size(self.id, self.state.size * self.state.size_scaling)
        else:
            data = self.layer[self.state.size_attribute].ravel()
            data = data[self._marker_keep]
            if self.state.size_vmax == self.state.size_vmin:
                size = np.ones(data.shape) * 10
            else:
                size = (20 * (data - self.state.size_vmin) /
                        (self.state.size_vmax - self.state.size_vmin))
            size_data = size * self.state.size_scaling
            size_data[np.isnan(data)] = 0.
            self._multiscat.set_size(self.id, size_data)

    def _update_colors(self):
        if self.state.color_mode is None:
            pass
        elif self.state.color_mode == 'Fixed':
            self._multiscat.set_color(self.id, self.state.color)
        else:
            data = self.layer[self.state.cmap_attribute].ravel()
            data = data[self._marker_keep]
            if self.state.cmap_vmax == self.state.cmap_vmin:
                cmap_data = np.ones(data.shape) * 0.5
            else:
                cmap_data = ((data - self.state.cmap_vmin) /
                             (self.state.cmap_vmax - self.state.cmap_vmin))
            cmap_data = self.state.cmap(cmap_data)
            cmap_data[:, 3][np.isnan(data)] = 0.
            self._multiscat.set_color(self.id, cmap_data)

    def _update_alpha(self):
        self._multiscat.set_alpha(self.id, self.state.alpha)

    def _update_data(self):

        try:
            x = self.layer[self._viewer_state.x_att].ravel()
            y = self.layer[self._viewer_state.y_att].ravel()
            z = self.layer[self._viewer_state.z_att].ravel()
        except AttributeError:
            return
        except (IncompatibleAttribute, IndexError):
            # The following includes a call to self.clear()
            self.disable_invalid_attributes(self._viewer_state.x_att,
                                            self._viewer_state.y_att,
                                            self._viewer_state.z_att)
            return
        else:
            self._enabled = True

        if self._clip_limits is None:
            keep = Ellipsis
        else:
            xmin, xmax, ymin, ymax, zmin, zmax = self._clip_limits
            keep = (x >= xmin) & (x <= xmax) & (y >= ymin) & (y <= ymax) & (z >= zmin) & (z <= zmax)
            x, y, z = x[keep], y[keep], z[keep]

        self._marker_data = np.array([x, y, z]).transpose()
        self._marker_keep = keep

        # We need to make sure we update the sizes and colors in case
        # these were set as arrays, since the size of the data might have
        # changed (in the case of subsets)

        with self._multiscat.delay_update():
            self._multiscat.set_data_values(self.id, x, y, z)
            self._update_sizes()
            self._update_colors()

        self.redraw()

    @property
    def default_limits(self):
        if self._marker_data is None:
            raise ValueError("Data not yet set")
        dmin = np.nanmin(self._marker_data, axis=0)
        dmax = np.nanmax(self._marker_data, axis=0)
        # TODO: the following can be optimized
        return tuple(np.array([dmin, dmax]).transpose().ravel())

    def set_clip(self, limits):
        self._clip_limits = limits
        self._update_data()
<|endoftext|>


<commit_before># This file implements a MultiVolumeVisual class that can be used to show
# multiple volumes simultaneously. It is derived from the original VolumeVisual
# class in vispy.visuals.volume, which is releaed under a BSD license included
# here:
#
# ===========================================================================
# Vispy is licensed under the terms of the (new) BSD license:
#
# Copyright (c) 2015, authors of Vispy
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
# * Redistributions of source code must retain the above copyright
#   notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
#   notice, this list of conditions and the following disclaimer in the
#   documentation and/or other materials provided with the distribution.
# * Neither the name of Vispy Development Team nor the names of its
#   contributors may be used to endorse or promote products
#   derived from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
# OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# ===========================================================================
#
# This modified version is released under the BSD license given in the LICENSE
# file in this repository.

from __future__ import absolute_import, division, print_function

from distutils.version import LooseVersion
from collections import defaultdict

import numpy as np
from glue.external import six
from glue.utils import iterate_chunks

from ..extern.vispy.gloo import Texture3D, TextureEmulated3D, VertexBuffer, IndexBuffer
from ..extern.vispy.visuals import VolumeVisual, Visual
from ..extern.vispy.visuals.shaders import Function
from ..extern.vispy.color import get_colormap, Color
from ..extern.vispy.scene.visuals import create_visual_node

from .shaders import get_frag_shader, VERT_SHADER

NUMPY_LT_1_13 = LooseVersion(np.__version__) < LooseVersion('1.13')

class MultiVolumeVisual(VolumeVisual):
    """
    Displays multiple 3D volumes simultaneously.

    Parameters
    ----------
    volumes : list of tuples
        The volumes to show. Each tuple should contain three elements: the data
        array, the clim values, and the colormap to use. The clim values should
        be either a 2-element tuple, or None.
    relative_step_size : float
        The relative step size to step through the volume. Default 0.8.
        Increase to e.g. 1.5 to increase performance, at the cost of
        quality.
    emulate_texture : bool
        Use 2D textures to emulate a 3D texture. OpenGL ES 2.0 compatible,
        but has lower performance on desktop platforms.
    n_volume_max : int
        Absolute maximum number of volumes that can be shown.
    """

    def __init__(self, n_volume_max=10, relative_step_size=0.8,
                 emulate_texture=False, bgcolor='white', resolution=256):

        # Choose texture class
        tex_cls = TextureEmulated3D if emulate_texture else Texture3D

        self._n_volume_max = n_volume_max
        self._vol_shape = (resolution, resolution, resolution)
        self._need_vertex_update = True
        self._data_slice = Ellipsis

        self.resolution = resolution

        # We deliberately don't use super here because we don't want to call
        # VolumeVisual.__init__
        Visual.__init__(self, vcode=VERT_SHADER, fcode="")

        self.volumes = defaultdict(dict)

        self._clip_data = False
        self._update_shader()

        # Create gloo objects
        self._vertices = VertexBuffer()
        self._texcoord = VertexBuffer(
            np.array([[0, 0, 0],
                      [1, 0, 0],
                      [0, 1, 0],
                      [1, 1, 0],
                      [0, 0, 1],
                      [1, 0, 1],
                      [0, 1, 1],
                      [1, 1, 1]], dtype=np.float32))

        self.textures = []
        for i in range(n_volume_max):

            # Set up texture object
            self.textures.append(tex_cls(self._vol_shape, interpolation='linear',
                                         wrapping='clamp_to_edge'))

            # Pass texture object and default colormap to shader program
            self.shared_program['u_volumetex_{0}'.format(i)] = self.textures[i]

            # Make sure all textures are disbaled
            self.shared_program['u_enabled_{0}'.format(i)] = 0
            self.shared_program['u_weight_{0}'.format(i)] = 1

        self.shared_program['a_position'] = self._vertices
        self.shared_program['a_texcoord'] = self._texcoord
        self.shared_program['u_shape'] = self._vol_shape[::-1]

        self.shared_program['u_clip_min'] = [0, 0, 0]
        self.shared_program['u_clip_max'] = [1, 1, 1]

        self.shared_program['u_downsample'] = 1.

        self._draw_mode = 'triangle_strip'
        self._index_buffer = IndexBuffer()

        self.shared_program.frag['sampler_type'] = self.textures[0].glsl_sampler_type
        self.shared_program.frag['sample'] = self.textures[0].glsl_sample

        self.set_background(bgcolor)

        # Only show back faces of cuboid. This is required because if we are
        # inside the volume, then the front faces are outside of the clipping
        # box and will not be drawn.
        self.set_gl_state('translucent', cull_face=False)

        self.relative_step_size = relative_step_size
        self.relative_step_size_orig = self.relative_step_size

        self.volumes = defaultdict(dict)

        try:
            self.freeze()
        except AttributeError:  # Older versions of VisPy
            pass

    def _update_shader(self):
        shader = get_frag_shader(self.volumes, clipped=self._clip_data,
                                 n_volume_max=self._n_volume_max)
        if getattr(self, '_shader_cache', None) == shader:
            return
        self.shared_program.frag = shader
        self._shader_cache = shader
        for label in self.volumes:
            index = self.volumes[label]['index']
            cmap = self.volumes[label]['cmap']
            self.shared_program.frag['cmap{0:d}'.format(index)] = Function(cmap.glsl_map)

    def set_clip(self, clip_data, clip_limits):
        self._clip_data = int(clip_data)
        if clip_data:
            self.shared_program['u_clip_min'] = clip_limits[:3]
            self.shared_program['u_clip_max'] = clip_limits[3:]
        self._update_shader()

    def downsample(self):
        if self._vol_shape is None:
            return
        min_dimension = min(self._vol_shape)
        self.shared_program['u_downsample'] = min_dimension / 20

    def upsample(self):
        self.shared_program['u_downsample'] = 1.

    def set_background(self, color):
        self.shared_program['u_bgcolor'] = Color(color).rgba

    @property
    def _free_slot_index(self):
        for i in range(self._n_volume_max):
            if self.shared_program['u_enabled_{0}'.format(i)] == 0:
                return i
        raise ValueError("No free slots")

    def allocate(self, label):
        if label in self.volumes:
            raise ValueError("Label {0} already exists".format(label))
        index = self._free_slot_index
        self.volumes[label] = {}
        self.volumes[label]['index'] = index

    def enable(self, label):
        index = self.volumes[label]['index']
        self.volumes[label]['enabled'] = True
        self.shared_program['u_enabled_{0}'.format(index)] = 1
        self._update_shader()

    def disable(self, label):
        index = self.volumes[label]['index']
        self.volumes[label]['enabled'] = False
        self.shared_program['u_enabled_{0}'.format(index)] = 0
        self._update_shader()

    def deallocate(self, label):
        self.disable(label)
        self.volumes.pop(label)

    def set_cmap(self, label, cmap):
        if isinstance(cmap, six.string_types):
            cmap = get_colormap(cmap)
        self.volumes[label]['cmap'] = cmap
        self._update_shader()

    def set_clim(self, label, clim):
        # Avoid setting the same limits again
        if 'clim' in self.volumes[label] and self.volumes[label]['clim'] == clim:
            return
        self.volumes[label]['clim'] = clim
        if 'data' in self.volumes[label]:
            self._update_scaled_data(label)

    def set_weight(self, label, weight):
        index = self.volumes[label]['index']
        self.shared_program['u_weight_{0:d}'.format(index)] = weight

    def set_multiply(self, label, label_other):
        self.volumes[label]['multiply'] = label_other
        self._update_shader()

    def set_data(self, label, data, layer=None):

        if 'clim' not in self.volumes[label]:
            raise ValueError("set_clim should be called before set_data")

        # Avoid adding the same data again
        if 'data' in self.volumes[label] and self.volumes[label]['data'] is data:
            return

        self.volumes[label]['data'] = data
        self.volumes[label]['layer'] = layer
        self._update_scaled_data(label, initial_shape=True)

    def label_for_layer(self, layer):
        for label in self.volumes:
            if 'layer' in self.volumes[label]:
                if self.volumes[label]['layer'] is layer:
                    return label

    def _update_scaled_data(self, label, initial_shape=False):

        index = self.volumes[label]['index']
        clim = self.volumes[label]['clim']
        data = self.volumes[label]['data']

        # With certain graphics cards, sending the data in one chunk to OpenGL
        # causes artifacts in the rendering - see e.g.
        # https://github.com/vispy/vispy/issues/1412
        # To avoid this, we process the data in chunks. Since we need to do
        # this, we can also do the copy and renormalization on the chunk to
        # avoid excessive memory usage.

        # To start off we need to tell the texture about the new shape
        self.shared_program['u_volumetex_{0:d}'.format(index)].resize(data.shape)

        # Determine the chunk shape - the value of 100 as the minimum value
        # is arbitrary but appears to work nicely. We can reduce that in future
        # if needed.

        sliced_data = data[self._data_slice]

        chunk_shape = [min(x, 100) for x in sliced_data.shape]

        # FIXME: shouldn't be needed!
        zeros = np.zeros(self._vol_shape, dtype=np.float32)
        self.shared_program['u_volumetex_{0:d}'.format(index)].set_data(zeros)

        # Now loop over chunks
        for view in iterate_chunks(self._vol_shape, chunk_shape=chunk_shape):

            chunk = sliced_data[view]
            chunk = chunk.astype(np.float32)
            chunk -= clim[0]
            chunk *= 1 / (clim[1] - clim[0])

            # PERF: nan_to_num doesn't actually help memory usage as it runs
            # isnan internally, and it's slower, so we just use the following
            # methind. In future we could do this directly with a C extension.
            chunk[np.isnan(chunk)] = 0.

            offset = tuple([s.start for s in view])

            if chunk.size == 0:
                continue

            self.shared_program['u_volumetex_{0:d}'.format(index)].set_data(chunk, offset=offset)

    def _get_step_start(self, vmin, vmax):

        size = vmax - vmin
        if size < self.resolution:
            step = 1
            start = int(vmin)
        else:
            step = int(np.ceil(size / self.resolution))
            start = int(vmin)

        if start < 0:
            start = 0

        return step, start

    def _update_slice_transform(self, x_min, x_max, y_min, y_max, z_min, z_max):

        x_step, x_start = self._get_step_start(x_min, x_max)
        y_step, y_start = self._get_step_start(y_min, y_max)
        z_step, z_start = self._get_step_start(z_min, z_max)

        self._data_slice = [slice(z_start, z_start + self.resolution * z_step, z_step),
                            slice(y_start, y_start + self.resolution * y_step, y_step),
                            slice(x_start, x_start + self.resolution * x_step, x_step)]

        self.transform.inner.scale = [x_step, y_step, z_step]
        self.transform.inner.translate = [x_start, y_start, z_start]

        for label in self.volumes:
            self._update_scaled_data(label)

        self.transform._update_shaders()
        self.transform.update()

    @property
    def enabled(self):
        return [self.shared_program['u_enabled_{0}'.format(i)] == 1
                for i in range(self._n_volume_max)]

    def draw(self):
        if not any(self.enabled):
            return
        else:
            try:
                super(MultiVolumeVisual, self).draw()
            except Exception:
                pass

MultiVolume = create_visual_node(MultiVolumeVisual)
<commit_msg>Use 128 instead of 100 for the chunk size when uploading data to OpenGL<commit_after># This file implements a MultiVolumeVisual class that can be used to show
# multiple volumes simultaneously. It is derived from the original VolumeVisual
# class in vispy.visuals.volume, which is releaed under a BSD license included
# here:
#
# ===========================================================================
# Vispy is licensed under the terms of the (new) BSD license:
#
# Copyright (c) 2015, authors of Vispy
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
# * Redistributions of source code must retain the above copyright
#   notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
#   notice, this list of conditions and the following disclaimer in the
#   documentation and/or other materials provided with the distribution.
# * Neither the name of Vispy Development Team nor the names of its
#   contributors may be used to endorse or promote products
#   derived from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
# OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# ===========================================================================
#
# This modified version is released under the BSD license given in the LICENSE
# file in this repository.

from __future__ import absolute_import, division, print_function

from distutils.version import LooseVersion
from collections import defaultdict

import numpy as np
from glue.external import six
from glue.utils import iterate_chunks

from ..extern.vispy.gloo import Texture3D, TextureEmulated3D, VertexBuffer, IndexBuffer
from ..extern.vispy.visuals import VolumeVisual, Visual
from ..extern.vispy.visuals.shaders import Function
from ..extern.vispy.color import get_colormap, Color
from ..extern.vispy.scene.visuals import create_visual_node

from .shaders import get_frag_shader, VERT_SHADER

NUMPY_LT_1_13 = LooseVersion(np.__version__) < LooseVersion('1.13')

class MultiVolumeVisual(VolumeVisual):
    """
    Displays multiple 3D volumes simultaneously.

    Parameters
    ----------
    volumes : list of tuples
        The volumes to show. Each tuple should contain three elements: the data
        array, the clim values, and the colormap to use. The clim values should
        be either a 2-element tuple, or None.
    relative_step_size : float
        The relative step size to step through the volume. Default 0.8.
        Increase to e.g. 1.5 to increase performance, at the cost of
        quality.
    emulate_texture : bool
        Use 2D textures to emulate a 3D texture. OpenGL ES 2.0 compatible,
        but has lower performance on desktop platforms.
    n_volume_max : int
        Absolute maximum number of volumes that can be shown.
    """

    def __init__(self, n_volume_max=10, relative_step_size=0.8,
                 emulate_texture=False, bgcolor='white', resolution=256):

        # Choose texture class
        tex_cls = TextureEmulated3D if emulate_texture else Texture3D

        self._n_volume_max = n_volume_max
        self._vol_shape = (resolution, resolution, resolution)
        self._need_vertex_update = True
        self._data_slice = Ellipsis

        self.resolution = resolution

        # We deliberately don't use super here because we don't want to call
        # VolumeVisual.__init__
        Visual.__init__(self, vcode=VERT_SHADER, fcode="")

        self.volumes = defaultdict(dict)

        self._clip_data = False
        self._update_shader()

        # Create gloo objects
        self._vertices = VertexBuffer()
        self._texcoord = VertexBuffer(
            np.array([[0, 0, 0],
                      [1, 0, 0],
                      [0, 1, 0],
                      [1, 1, 0],
                      [0, 0, 1],
                      [1, 0, 1],
                      [0, 1, 1],
                      [1, 1, 1]], dtype=np.float32))

        self.textures = []
        for i in range(n_volume_max):

            # Set up texture object
            self.textures.append(tex_cls(self._vol_shape, interpolation='linear',
                                         wrapping='clamp_to_edge'))

            # Pass texture object and default colormap to shader program
            self.shared_program['u_volumetex_{0}'.format(i)] = self.textures[i]

            # Make sure all textures are disbaled
            self.shared_program['u_enabled_{0}'.format(i)] = 0
            self.shared_program['u_weight_{0}'.format(i)] = 1

        self.shared_program['a_position'] = self._vertices
        self.shared_program['a_texcoord'] = self._texcoord
        self.shared_program['u_shape'] = self._vol_shape[::-1]

        self.shared_program['u_clip_min'] = [0, 0, 0]
        self.shared_program['u_clip_max'] = [1, 1, 1]

        self.shared_program['u_downsample'] = 1.

        self._draw_mode = 'triangle_strip'
        self._index_buffer = IndexBuffer()

        self.shared_program.frag['sampler_type'] = self.textures[0].glsl_sampler_type
        self.shared_program.frag['sample'] = self.textures[0].glsl_sample

        self.set_background(bgcolor)

        # Only show back faces of cuboid. This is required because if we are
        # inside the volume, then the front faces are outside of the clipping
        # box and will not be drawn.
        self.set_gl_state('translucent', cull_face=False)

        self.relative_step_size = relative_step_size
        self.relative_step_size_orig = self.relative_step_size

        self.volumes = defaultdict(dict)

        try:
            self.freeze()
        except AttributeError:  # Older versions of VisPy
            pass

    def _update_shader(self):
        shader = get_frag_shader(self.volumes, clipped=self._clip_data,
                                 n_volume_max=self._n_volume_max)
        if getattr(self, '_shader_cache', None) == shader:
            return
        self.shared_program.frag = shader
        self._shader_cache = shader
        for label in self.volumes:
            index = self.volumes[label]['index']
            cmap = self.volumes[label]['cmap']
            self.shared_program.frag['cmap{0:d}'.format(index)] = Function(cmap.glsl_map)

    def set_clip(self, clip_data, clip_limits):
        self._clip_data = int(clip_data)
        if clip_data:
            self.shared_program['u_clip_min'] = clip_limits[:3]
            self.shared_program['u_clip_max'] = clip_limits[3:]
        self._update_shader()

    def downsample(self):
        if self._vol_shape is None:
            return
        min_dimension = min(self._vol_shape)
        self.shared_program['u_downsample'] = min_dimension / 20

    def upsample(self):
        self.shared_program['u_downsample'] = 1.

    def set_background(self, color):
        self.shared_program['u_bgcolor'] = Color(color).rgba

    @property
    def _free_slot_index(self):
        for i in range(self._n_volume_max):
            if self.shared_program['u_enabled_{0}'.format(i)] == 0:
                return i
        raise ValueError("No free slots")

    def allocate(self, label):
        if label in self.volumes:
            raise ValueError("Label {0} already exists".format(label))
        index = self._free_slot_index
        self.volumes[label] = {}
        self.volumes[label]['index'] = index

    def enable(self, label):
        index = self.volumes[label]['index']
        self.volumes[label]['enabled'] = True
        self.shared_program['u_enabled_{0}'.format(index)] = 1
        self._update_shader()

    def disable(self, label):
        index = self.volumes[label]['index']
        self.volumes[label]['enabled'] = False
        self.shared_program['u_enabled_{0}'.format(index)] = 0
        self._update_shader()

    def deallocate(self, label):
        self.disable(label)
        self.volumes.pop(label)

    def set_cmap(self, label, cmap):
        if isinstance(cmap, six.string_types):
            cmap = get_colormap(cmap)
        self.volumes[label]['cmap'] = cmap
        self._update_shader()

    def set_clim(self, label, clim):
        # Avoid setting the same limits again
        if 'clim' in self.volumes[label] and self.volumes[label]['clim'] == clim:
            return
        self.volumes[label]['clim'] = clim
        if 'data' in self.volumes[label]:
            self._update_scaled_data(label)

    def set_weight(self, label, weight):
        index = self.volumes[label]['index']
        self.shared_program['u_weight_{0:d}'.format(index)] = weight

    def set_multiply(self, label, label_other):
        self.volumes[label]['multiply'] = label_other
        self._update_shader()

    def set_data(self, label, data, layer=None):

        if 'clim' not in self.volumes[label]:
            raise ValueError("set_clim should be called before set_data")

        # Avoid adding the same data again
        if 'data' in self.volumes[label] and self.volumes[label]['data'] is data:
            return

        self.volumes[label]['data'] = data
        self.volumes[label]['layer'] = layer
        self._update_scaled_data(label, initial_shape=True)

    def label_for_layer(self, layer):
        for label in self.volumes:
            if 'layer' in self.volumes[label]:
                if self.volumes[label]['layer'] is layer:
                    return label

    def _update_scaled_data(self, label, initial_shape=False):

        index = self.volumes[label]['index']
        clim = self.volumes[label]['clim']
        data = self.volumes[label]['data']

        # With certain graphics cards, sending the data in one chunk to OpenGL
        # causes artifacts in the rendering - see e.g.
        # https://github.com/vispy/vispy/issues/1412
        # To avoid this, we process the data in chunks. Since we need to do
        # this, we can also do the copy and renormalization on the chunk to
        # avoid excessive memory usage.

        # To start off we need to tell the texture about the new shape
        self.shared_program['u_volumetex_{0:d}'.format(index)].resize(data.shape)

        # Determine the chunk shape - the value of 128 as the minimum value
        # is arbitrary but appears to work nicely. We can reduce that in future
        # if needed.

        sliced_data = data[self._data_slice]

        chunk_shape = [min(x, 128) for x in sliced_data.shape]

        # FIXME: shouldn't be needed!
        zeros = np.zeros(self._vol_shape, dtype=np.float32)
        self.shared_program['u_volumetex_{0:d}'.format(index)].set_data(zeros)

        # Now loop over chunks
        for view in iterate_chunks(self._vol_shape, chunk_shape=chunk_shape):

            chunk = sliced_data[view]
            chunk = chunk.astype(np.float32)
            chunk -= clim[0]
            chunk *= 1 / (clim[1] - clim[0])

            # PERF: nan_to_num doesn't actually help memory usage as it runs
            # isnan internally, and it's slower, so we just use the following
            # methind. In future we could do this directly with a C extension.
            chunk[np.isnan(chunk)] = 0.

            offset = tuple([s.start for s in view])

            if chunk.size == 0:
                continue

            self.shared_program['u_volumetex_{0:d}'.format(index)].set_data(chunk, offset=offset)

    def _get_step_start(self, vmin, vmax):

        size = vmax - vmin
        if size < self.resolution:
            step = 1
            start = int(vmin)
        else:
            step = int(np.ceil(size / self.resolution))
            start = int(vmin)

        if start < 0:
            start = 0

        return step, start

    def _update_slice_transform(self, x_min, x_max, y_min, y_max, z_min, z_max):

        x_step, x_start = self._get_step_start(x_min, x_max)
        y_step, y_start = self._get_step_start(y_min, y_max)
        z_step, z_start = self._get_step_start(z_min, z_max)

        self._data_slice = [slice(z_start, z_start + self.resolution * z_step, z_step),
                            slice(y_start, y_start + self.resolution * y_step, y_step),
                            slice(x_start, x_start + self.resolution * x_step, x_step)]

        self.transform.inner.scale = [x_step, y_step, z_step]
        self.transform.inner.translate = [x_start, y_start, z_start]

        for label in self.volumes:
            self._update_scaled_data(label)

        self.transform._update_shaders()
        self.transform.update()

    @property
    def enabled(self):
        return [self.shared_program['u_enabled_{0}'.format(i)] == 1
                for i in range(self._n_volume_max)]

    def draw(self):
        if not any(self.enabled):
            return
        else:
            try:
                super(MultiVolumeVisual, self).draw()
            except Exception:
                pass

MultiVolume = create_visual_node(MultiVolumeVisual)
<|endoftext|>


<commit_before>''' -- imports from python libraries -- '''
# import os -- Keep such imports here
import json
import os
from django.http import StreamingHttpResponse

''' -- imports from installed packages -- '''
from django.shortcuts import render_to_response  # , render
from django.template import RequestContext
from django.http import Http404

try:
    from bson import ObjectId
except ImportError:  # old pymongo
    from pymongo.objectid import ObjectId

from gnowsys_ndf.ndf.views.methods import get_execution_time
''' -- imports from application folders/files -- '''
from gnowsys_ndf.ndf.models import node_collection, triple_collection, gridfs_collection, NodeJSONEncoder,Author, Buddy

def tools_logging(request):

	#Method for tools logging
	userdata = json.loads(request.POST.get('user_data',' '))
	app_name = request.POST.get('app_name',' ')
	sessionid = ""
	sessionid = request.COOKIES.get("sessionid")
	old_data = []
	buddies_authid_list = request.session.get('buddies_authid_list', [])
	buddy_id_list = []
	buddy_id_list.append(userdata['user_id'])
	userdata['sessionid'] = sessionid
	if buddies_authid_list:
	    buddy_id_list = buddy_id_list +  Author.get_user_id_list_from_author_oid_list(buddies_authid_list)
	if userdata['user_id'] and userdata['user_id']!= "None":
	    if not os.path.exists('/data/gstudio_tools_logs/'+app_name+'/'+userdata['params']['language']):
	        os.makedirs('/data/gstudio_tools_logs/'+app_name+'/'+userdata['params']['language'])

	    for each_id in buddy_id_list:
	        old_data = []
	        if os.path.exists('/data/gstudio_tools_logs/'+app_name+'/'+userdata['params']['language']+'/'+str(each_id)+'.json'):
	            with open('/data/gstudio_tools_logs/'+app_name+'/'+userdata['params']['language']+'/'+str(each_id)+'.json') as rfile:
	                if os.stat('/data/gstudio_tools_logs/'+app_name+'/'+userdata['params']['language']+'/'+str(each_id)+'.json').st_size != 0:
	                    old_data = json.load(rfile)

	        with open('/data/gstudio_tools_logs/'+app_name+'/'+userdata['params']['language']+'/'+str(each_id)+'.json', 'w'): pass

	        with open('/data/gstudio_tools_logs/'+app_name+'/'+userdata['params']['language']+'/'+str(each_id)+'.json', 'w') as wrfile:
	            old_data.append(userdata)
	            json.dump(old_data, wrfile)
	return StreamingHttpResponse("Success")

@get_execution_time
def tools_temp(request):
    context_variables = {'title' : "tools"}
    return render_to_response(
        'ndf/tools_list.html',
                                  context_variables,
        context_instance=RequestContext(request)
    )

<commit_msg>logic of dumping .json file changed<commit_after>''' -- imports from python libraries -- '''
# import os -- Keep such imports here
import json
import os
from django.http import StreamingHttpResponse

''' -- imports from installed packages -- '''
from django.shortcuts import render_to_response  # , render
from django.template import RequestContext
from django.http import Http404

try:
    from bson import ObjectId
except ImportError:  # old pymongo
    from pymongo.objectid import ObjectId

from gnowsys_ndf.ndf.views.methods import get_execution_time
''' -- imports from application folders/files -- '''
from gnowsys_ndf.ndf.models import node_collection, triple_collection, gridfs_collection, NodeJSONEncoder,Author, Buddy

def tools_logging(request):

	#Method for tools logging
	userdata = json.loads(request.POST.get('payload',' '))
	# app_name = request.POST.get('app_name',' ')
	app_name = userdata['appName']
	sessionid = ""
	sessionid = request.COOKIES.get("sessionid")
	old_data = []
	buddies_authid_list = request.session.get('buddies_authid_list', [])
	buddy_id_list = []
	buddy_id_list.append(userdata['userId'])
	userdata['sessionId'] = sessionid
	if buddies_authid_list:
	    buddy_id_list = buddy_id_list +  Author.get_user_id_list_from_author_oid_list(buddies_authid_list)
	if userdata['userId'] and userdata['userId']!= "None":
	    if not os.path.exists('/data/gstudio_tools_logs/'+app_name+'/'):
	        os.makedirs('/data/gstudio_tools_logs/'+app_name+'/')

	    for each_id in buddy_id_list:
	        old_data = []
	        if os.path.exists('/data/gstudio_tools_logs/'+app_name+'/'+str(each_id)+"-"+app_name+'.json'):
	            with open('/data/gstudio_tools_logs/'+app_name+'/'+str(each_id)+"-"+app_name+'.json' ) as rfile:
	                if os.stat('/data/gstudio_tools_logs/'+app_name+'/'+str(each_id)+"-"+app_name+'.json').st_size != 0:
	                    old_data = json.load(rfile)

	        with open('/data/gstudio_tools_logs/'+app_name+'/'+str(each_id)+"-"+app_name+'.json', 'w'): pass

	        with open('/data/gstudio_tools_logs/'+app_name+'/'+str(each_id)+"-"+app_name+'.json', 'w') as wrfile:
	            old_data.append(userdata)
	            json.dump(old_data, wrfile)

	return StreamingHttpResponse("Success")

@get_execution_time
def tools_temp(request):
    context_variables = {'title' : "tools"}
    return render_to_response(
        'ndf/tools_list.html',
                                  context_variables,
        context_instance=RequestContext(request)
    )

<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
'''
@author: Pieter Moris
'''

import argparse
import pandas as pd

import enrichment_stats
import gaf_parser
import genelist_importer
import obo_tools

# Run as stand alone script
# https://stackoverflow.com/questions/419163/what-does-if-name-main-do
if __name__ == '__main__':

    # Check provided arguments
    parser = argparse.ArgumentParser(
        description='Script to perform GO enrichment analysis', formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    # https://stackoverflow.com/questions/18862836/how-to-open-file-using-argparse
    parser.add_argument('-b', '--background', type=str, default='full annotation set', dest='background',
                        help='File containing a list of uniprot accession numbers for the background set of genes')
    parser.add_argument('-s', '--subset', type=str, required=True, dest='subset',
                        help='File containing a list of uniprot accession numbers for the subset of genes of interest')
    parser.add_argument('-o', '--obo', type=str, required=True, dest='obo',
                        help='.obo file containing gene ontology')
    parser.add_argument('-g', '--gaf', type=str, required=True, dest='gaf',
                        help='.gaf file containing GO annotations')
    # parser.add_argument('-O', '--output', type=argparse.FileType('w'),
    # default='enrichment-results.csv', dest='output',
    # help='Specifies the name or path of the output csv file')
    parser.add_argument('-O', '--output', type=str, default='enrichment_results.csv', dest='outputFile',
                        help='Output file or path')
    parser.add_argument('-n', '--namespace', type=str, default='all',
                        help='Select the GO namespace to limit the enrichment test to')
    parser.add_argument('-m', '--min', type=int, default='3', dest='minGenes',
                        help='Minimum number of genes before considering a GO category')
    parser.add_argument('-T', '--testing-thresh', type=float, default='0.05', dest='testing_threshold',
                        help='P-value cut-off to use to stop GO tree propagation during testing')
    parser.add_argument('-t', '--thresh', type=float, default='0.1', dest='threshold',
                        help='P-value cut-off to use for significance testing')
    parser.add_argument('-c', '--condense', action="store_true",
                        help='Suppress verbose output.')
    args = parser.parse_args()

    # Import the uniprot set of interest
    interest = genelist_importer.importSubset(args.subset)

    # If no background is provided, import the gene association file and retrieve
    # the full background set from it
    if args.background == 'full annotation set':
        print('No background gene set provided, retrieving all genes from the gene annotation file...\n')
        gafDict = gaf_parser.importFullGAF(args.gaf)
        background = set(gafDict.keys())
    # otherwise import the background set and use this to limit gene association import
    else:
        background = genelist_importer.importBackground(args.background)
        gafDict = gaf_parser.importGAF(args.gaf, background)

    # Generate a gene association file for the subset of interest too
    gafSubset = gaf_parser.createSubsetGafDict(interest, gafDict)

    # Check if the background/interest gene sets contain genes not present in the gene association file
    background = gaf_parser.removeObsoleteGenes(background, gafDict, 'background')
    interest = gaf_parser.removeObsoleteGenes(interest, gafSubset, 'interest')

    # Check if the set of interest contains uniprot ac's not present in the background set
    interest = genelist_importer.isValidSubset(interest, background)

    # import gene ontology file
    GOterms = obo_tools.importOBO(args.obo)

    # Reduce gene ontology file to selected namespace
    if not args.namespace == 'all':
        GOterms = obo_tools.filterOnNamespace(GOterms, args.namespace)
        # and reduce the gene association files to these GO terms
        gaf_parser.removeObsoleteGOids(gafDict, GOterms)
        gaf_parser.removeObsoleteGOids(gafSubset, GOterms)

    # for i in ['GO:0060373', 'GO:0048245', 'GO:0044077', 'GO:0042925', 'GO:1902361', 'GO:1902361', 'GO:1902361', 'GO:0000001', 'GO:0000002']:
    #     print('id', i, 'parents', GOterms[i].parents)

    # Build full GO hierarchy
    print('Propagating through ontology to find all children and parents for each term...\n')
    obo_tools.buildGOtree(GOterms)

    # print('\n After propagating through parents')
    #
    # for i in ['GO:0060373', 'GO:0048245', 'GO:0044077', 'GO:0042925', 'GO:1902361', 'GO:1902361', 'GO:1902361', 'GO:0000001', 'GO:0000002']:
    #     print('id', i, 'parents', GOterms[i].parents)

    # Perform enrichment test
    pValues = enrichment_stats.enrichmentAnalysis(background, interest, GOterms, gafDict, gafSubset, minGenes=args.minGenes, threshold=args.testing_threshold)

    # print('\nTested GO-terms and their p-values:')
    # print(pValues)
    # print('\nOrdered p-values:')
    # print(sorted(pValues.values()))

    # Perform multiple testing correction
    correctedPvalues = enrichment_stats.multipleTestingCorrection(pValues, threshold = args.threshold)

    if not args.condense:
        print(correctedPvalues)

    # Create dataframe with tested GO terms and results
    output = enrichment_stats.annotateOutput(correctedPvalues, GOterms)

    if not args.condense:
        print(correctedPvalues)
        print('\nGO term, uncorrected and FDR-corrected p-values and description of GO term:\m')
        print(output)

    print('Saving output to', args.outputFile)
    output.to_csv(args.outputFile)<commit_msg>Add description of main script<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
'''
@author: Pieter Moris

Main script to perform a gene enrichment analysis.

# Method

Performs one-sided hypergeometric tests, starting from the most specific (child) GO terms associated with the
genes in the set of interest. If the p-value of the test does not fall below the specified significance level alpha,
the test wll be carried out for all of the term's parent terms, otherwise the process will terminate. This method
attempts to limit the total number of tests that need to be carried out, since a term that is enriched will likely
also have enriched parent terms. Furthermore, GO terms associated with a small number of genes are skipped.
Next, the Benjamini-Hochberg FDR or Bonferroni multiple testing correction are applied to the test results.
Finally, a csv file containing all the GO terms that were evaluated and their p-values are returned.

# Requires the following input files:
    - gene association file in .gaf format
    - gene ontology file in .obo format
    - text file with genes of interest separated by new lines
        expects uniprot AC's
    - optional text file with background set
        if not provided, the entire gene annotation found in the association file will be used

# The following options can be specified:
    - The output file in which results will be stored
    - The minimum number of genes required to be associated with a term before it will be tested.
    - The FDR cut-off to utilise.
    - The p-value threshold to limit the recursive testing of the GO tree.
    - The namespace to test: all, biological_process, molecular_function or cellular_component
        Uses the naming convention found in the .obo file.

'''

import argparse
import pandas

import enrichment_stats
import gaf_parser
import genelist_importer
import obo_tools

# Run as stand alone script
# https://stackoverflow.com/questions/419163/what-does-if-name-main-do
if __name__ == '__main__':

    # Check provided arguments
    parser = argparse.ArgumentParser(
        description='Script to perform GO enrichment analysis', formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    # https://stackoverflow.com/questions/18862836/how-to-open-file-using-argparse
    parser.add_argument('-b', '--background', type=str, default='full annotation set', dest='background',
                        help='File containing a list of uniprot accession numbers for the background set of genes')
    parser.add_argument('-s', '--subset', type=str, required=True, dest='subset',
                        help='File containing a list of uniprot accession numbers for the subset of genes of interest')
    parser.add_argument('-o', '--obo', type=str, required=True, dest='obo',
                        help='.obo file containing gene ontology')
    parser.add_argument('-g', '--gaf', type=str, required=True, dest='gaf',
                        help='.gaf file containing GO annotations')
    # parser.add_argument('-O', '--output', type=argparse.FileType('w'),
    # default='enrichment-results.csv', dest='output',
    # help='Specifies the name or path of the output csv file')
    parser.add_argument('-O', '--output', type=str, default='enrichment_results.csv', dest='outputFile',
                        help='Output file or path')
    parser.add_argument('-n', '--namespace', type=str, default='all',
                        help='Select the GO namespace to limit the enrichment test to')
    parser.add_argument('-m', '--min', type=int, default='3', dest='minGenes',
                        help='Minimum number of genes before considering a GO category')
    parser.add_argument('-T', '--testing-thresh', type=float, default='0.05', dest='testing_threshold',
                        help='P-value cut-off to use to stop GO tree propagation during testing')
    parser.add_argument('-t', '--thresh', type=float, default='0.1', dest='threshold',
                        help='P-value cut-off to use for significance testing')
    parser.add_argument('-c', '--condense', action="store_true",
                        help='Suppress verbose output.')
    args = parser.parse_args()

    # Import the uniprot set of interest
    interest = genelist_importer.importSubset(args.subset)

    # If no background is provided, import the gene association file and retrieve
    # the full background set from it
    if args.background == 'full annotation set':
        print('No background gene set provided, retrieving all genes from the gene annotation file...\n')
        gafDict = gaf_parser.importFullGAF(args.gaf)
        background = set(gafDict.keys())
    # otherwise import the background set and use this to limit gene association import
    else:
        background = genelist_importer.importBackground(args.background)
        gafDict = gaf_parser.importGAF(args.gaf, background)

    # Generate a gene association file for the subset of interest too
    gafSubset = gaf_parser.createSubsetGafDict(interest, gafDict)

    # Check if the background/interest gene sets contain genes not present in the gene association file
    background = gaf_parser.removeObsoleteGenes(background, gafDict, 'background')
    interest = gaf_parser.removeObsoleteGenes(interest, gafSubset, 'interest')

    # Check if the set of interest contains uniprot ac's not present in the background set
    interest = genelist_importer.isValidSubset(interest, background)

    # Import gene ontology file
    GOterms = obo_tools.importOBO(args.obo)

    # Reduce gene ontology file to selected namespace
    if not args.namespace == 'all':
        GOterms = obo_tools.filterOnNamespace(GOterms, args.namespace)
        # and reduce the gene association files to these GO terms
        gaf_parser.removeObsoleteGOids(gafDict, GOterms)
        gaf_parser.removeObsoleteGOids(gafSubset, GOterms)

    # for i in ['GO:0060373', 'GO:0048245', 'GO:0044077', 'GO:0042925', 'GO:1902361', 'GO:1902361', 'GO:1902361', 'GO:0000001', 'GO:0000002']:
    #     print('id', i, 'parents', GOterms[i].parents)

    # Build full GO hierarchy
    print('Propagating through ontology to find all children and parents for each term...\n')
    obo_tools.buildGOtree(GOterms)

    # print('\n After propagating through parents')
    #
    # for i in ['GO:0060373', 'GO:0048245', 'GO:0044077', 'GO:0042925', 'GO:1902361', 'GO:1902361', 'GO:1902361', 'GO:0000001', 'GO:0000002']:
    #     print('id', i, 'parents', GOterms[i].parents)

    # Perform enrichment test
    pValues = enrichment_stats.enrichmentAnalysis(background, interest, GOterms, gafDict, gafSubset, minGenes=args.minGenes, threshold=args.testing_threshold)

    # print('\nTested GO-terms and their p-values:')
    # print(pValues)
    # print('\nOrdered p-values:')
    # print(sorted(pValues.values()))

    # Perform multiple testing correction
    correctedPvalues = enrichment_stats.multipleTestingCorrection(pValues, threshold = args.threshold)

    # Print intermediate output
    if not args.condense:
        print(correctedPvalues)

    # Create dataframe with tested GO terms and results
    output = enrichment_stats.annotateOutput(correctedPvalues, GOterms)

    # Print intermediate output
    if not args.condense:
        print(correctedPvalues)
        print('\nGO term, uncorrected and FDR-corrected p-values and description of GO term:\m')
        print(output)

    # Save results
    print('Saving output to', args.outputFile)
    output.to_csv(args.outputFile)<|endoftext|>


<commit_before>import ImageGrab
import Image
import os
import time
from random import *
import win32api, win32con
import threading
from pytesser import *

# Globals

# Bounding box for QWOP
start_x, start_y = 520, 270
end_x, end_y = 1159, 668

frame = (start_x, start_y, end_x, end_y)

# Bounding box for the "metres" dialogue box
metres_start_x, metres_start_y = 170, 24
metres_end_x, metres_end_y = 413, 46

metres_box = (metres_start_x, metres_start_y, metres_end_x, metres_end_y)

# x, y coordinate of the ribbon that pops up when you die
ribbon_x, ribbon_y = 155, 125

ribbon_pixel = (ribbon_x, ribbon_y)

# Key codes
VK_CODE = {
    'SPACE':0x20,
    'O':0x4F,
    'P':0x50,
    'Q':0x51,
    'W':0x57
    }

def sendKey(key, duration=0.1):
    win32api.keybd_event(key, 0, 0, 0)
    time.sleep(duration)
    win32api.keybd_event(key, 0, win32con.KEYEVENTF_KEYUP, 0)

def leftClick(coords, duration=0.1):
    win32api.SetCursorPos((start_x + coords[0], start_y + coords[1]))

    win32api.mouse_event(win32con.MOUSEEVENTF_LEFTDOWN,0,0)

    time.sleep(duration)

    win32api.mouse_event(win32con.MOUSEEVENTF_LEFTUP,0,0)

class AutoQwopper:
    def __init__(self):
        self.update()

    def update(self):
        self.qwop_frame = ImageGrab.grab(frame)
        self.metres_frame = self.qwop_frame.crop(metres_box)

    def isDead(self):
        return (self.qwop_frame.getpixel(ribbon_pixel) == (255, 255, 0))

    def beginGame(self):
        leftClick((100, 100))

    def restartGame(self):
        sendKey(VK_CODE['SPACE'])

    def getMetres(self):
        metres = float(image_to_string(self.metres_frame)[:-9])
        self.metres = metres

    def randomKeyPress(self):
        keys = ['Q', 'W', 'O', 'P']

        # Send up to all 4 keys in parallel
        for i in xrange(4):
            if (random() < 0.5):
                key = keys[i]

                print ("Pressing key: " + key)

                duration = random() + 0.5

                t = threading.Thread(target=sendKey, args=(VK_CODE[key], duration))
                t.start()

    def run(self):
        self.beginGame()
        if (self.isDead()):
            # restart game if this isn't the first time playing
            self.restartGame()
            self.update()
            self.getMetres()

        while (not self.isDead()):
            self.getMetres()
            self.randomKeyPress()
            time.sleep(0.25)
            self.update()
        print ("Went a total of " + str(self.metres) + " metres before dying.")

def main():
    qwopper = AutoQwopper()
    qwopper.run()

if __name__ == '__main__':
    main()<commit_msg>Make window positions relative to top left corner<commit_after>import ImageGrab
import Image
import os
import time
from random import *
import win32api, win32con
import threading
from pytesser import *

# Globals

# Bounding box for QWOP
start_x, start_y = 9, 105
end_x, end_y = 640 + start_x, 400 + start_y

frame = (start_x, start_y, end_x, end_y)

# Bounding box for the "metres" dialogue box
metres_start_x, metres_start_y = 170, 24
metres_end_x, metres_end_y = 413, 46

metres_box = (metres_start_x, metres_start_y, metres_end_x, metres_end_y)

# x, y coordinate of the ribbon that pops up when you die
ribbon_x, ribbon_y = 155, 125

ribbon_pixel = (ribbon_x, ribbon_y)

# Key codes
VK_CODE = {
    'SPACE':0x20,
    'O':0x4F,
    'P':0x50,
    'Q':0x51,
    'W':0x57
    }

def sendKey(key, duration=0.1):
    win32api.keybd_event(key, 0, 0, 0)
    time.sleep(duration)
    win32api.keybd_event(key, 0, win32con.KEYEVENTF_KEYUP, 0)

def leftClick(coords, duration=0.1):
    win32api.SetCursorPos((start_x + coords[0], start_y + coords[1]))

    win32api.mouse_event(win32con.MOUSEEVENTF_LEFTDOWN,0,0)

    time.sleep(duration)

    win32api.mouse_event(win32con.MOUSEEVENTF_LEFTUP,0,0)

class AutoQwopper:
    def __init__(self):
        self.update()

    def update(self):
        self.qwop_frame = ImageGrab.grab(frame)
        self.metres_frame = self.qwop_frame.crop(metres_box)

    def isDead(self):
        return (self.qwop_frame.getpixel(ribbon_pixel) == (255, 255, 0))

    def beginGame(self):
        leftClick((100, 100))

    def restartGame(self):
        sendKey(VK_CODE['SPACE'])

    def getMetres(self):
        metres = float(image_to_string(self.metres_frame)[:-9])
        self.metres = metres

    def randomKeyPress(self):
        keys = ['Q', 'W', 'O', 'P']

        # Send up to all 4 keys in parallel
        for i in xrange(4):
            if (random() < 0.5):
                key = keys[i]

                print ("Pressing key: " + key)

                duration = random() + 0.5

                t = threading.Thread(target=sendKey, args=(VK_CODE[key], duration))
                t.start()

    def run(self):
        self.beginGame()
        if (self.isDead()):
            # restart game if this isn't the first time playing
            self.restartGame()
            self.update()
            self.getMetres()

        while (not self.isDead()):
            self.getMetres()
            self.randomKeyPress()
            time.sleep(0.25)
            self.update()
        print ("Went a total of " + str(self.metres) + " metres before dying.")

def main():
    qwopper = AutoQwopper()
    qwopper.run()

if __name__ == '__main__':
    main()<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf8 -*-

# ============================================================================
#  Copyright (c) 2013-2016 nexB Inc. http://www.nexb.com/ - All rights reserved.
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#      http://www.apache.org/licenses/LICENSE-2.0
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
# ============================================================================

from __future__ import print_function

import logging
import codecs

import click
import os
import unicodecsv

import about_code_tool.gen
import about_code_tool.model
import about_code_tool.attrib

from os.path import exists, join
from about_code_tool import NOTSET
from about_code_tool.util import to_posix

__version__ = '3.0.0dev'

__about_spec_version__ = '2.0.0dev'

__copyright__ = """
    Copyright (c) 2013-2016 nexB Inc. All rights reserved.
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at
        http://www.apache.org/licenses/LICENSE-2.0
    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License."""

prog_name = 'AboutCode'
no_stdout = 0

intro = '''%(prog_name)s, version %(__version__)s
ABOUT spec version: %(__about_spec_version__)s http://dejacode.org
%(__copyright__)s
''' % locals()

class AboutCommand(click.Command):
    def main(self, args=None, prog_name=None, complete_var=None,
             standalone_mode=True, **extra):
        """
        Workaround click 4.0 bug https://github.com/mitsuhiko/click/issues/365
        """
        return click.Command.main(self, args=args, prog_name=self.name,
                                  complete_var=complete_var,
                                  standalone_mode=standalone_mode, **extra)

@click.group(name='about')
@click.version_option(version=__version__, prog_name=prog_name, message=intro)
@click.option('-v', '--verbose', count=True,
              help='Increase verbosity. Repeat to print more output.')
@click.option('-q', '--quiet', count=True, help='Do not print any output.')
def cli(verbose, quiet):
    # Update the no_stdout value globally
    global no_stdout
    no_stdout = quiet

    pass
    # click.echo('Verbosity: %s' % verbose)

inventory_help = '''
LOCATION: Path to an ABOUT file or a directory containing ABOUT files
OUTPUT: Path to CSV file to write the inventory to
'''
@cli.command(help=inventory_help,
             short_help='LOCATION: directory, OUTPUT: csv file',
             cls=AboutCommand)
@click.argument('location', nargs=1, required=True,
                type=click.Path(exists=True, file_okay=True,
                                dir_okay=True, writable=False,
                                readable=True, resolve_path=True))
@click.argument('output', nargs=1, required=True,
                type=click.Path(exists=False, file_okay=True, writable=True,
                                dir_okay=False, resolve_path=True))
@click.option('--overwrite', is_flag=True, help='Overwrites the output file if it exists')
def inventory(overwrite, location, output):
    """
    Inventory components from an ABOUT file or a directory tree of ABOUT
    files.
    """
    click.echo('Running about-code-tool version ' + __version__)
    # Check is the <OUTPUT> valid.
    if os.path.exists(output) and not overwrite:
        click.echo('ERROR: <output> file already exists.')
        click.echo('Select a different file name or use the --overwrite option after the `inventory`.')
        click.echo()
        return
    if not output.endswith('.csv'):
        click.echo('ERROR: <output> must be a CSV file ending with ".csv".')
        click.echo()
        return
    if not os.path.exists(output):
        click.echo('ERROR: <output> does not exists. Please check and correct the <output>.')
        click.echo()
        return

    click.echo('Collecting the inventory from location: ''%(location)s '
               'and writing CSV output to: %(output)s' % locals())

    errors, abouts = about_code_tool.model.collect_inventory(location)
    log_errors(errors)
    about_code_tool.model.to_csv(abouts, output)

gen_help = '''
LOCATION: Path to a CSV inventory file
OUTPUT: Path to the directory to write ABOUT files to
'''
@cli.command(help=gen_help,
             short_help='LOCATION: csv file, OUTPUT: directory',
             cls=AboutCommand)
@click.argument('location', nargs=1, required=True,
                type=click.Path(exists=True, file_okay=True,
                                dir_okay=False, writable=False,
                                readable=True, resolve_path=True))
@click.argument('output', nargs=1, required=True,
                type=click.Path(exists=True, file_okay=False, writable=True,
                                dir_okay=True, resolve_path=True))
def gen(location, output):
    """
    Given a CVS inventory of ABOUT files at location, generate ABOUT files in
    base directory.
    """
    click.echo('Running about-code-tool version ' + __version__)
    click.echo('Generating ABOUT files...')
    errors, abouts = about_code_tool.gen.generate(location, output)

    lea = len(abouts)
    lee = len(errors)
    click.echo('Generated %(lea)d ABOUT files with %(lee)d errors or warning' % locals())
    log_errors(errors, output)

@cli.command(cls=AboutCommand)
def export():
    click.echo('Running about-code-tool version ' + __version__)
    click.echo('Exporting zip archive...')

@cli.command(cls=AboutCommand)
def fetch(location):
    """
    Given a directory of ABOUT files at location, calls the DejaCode API and
    update or create license data fields and license texts.
    """
    click.echo('Running about-code-tool version ' + __version__)
    click.echo('Updating ABOUT files...')

@cli.command(cls=AboutCommand)
@click.argument('location', nargs=1, required=True,
                type=click.Path(exists=True, file_okay=True,
                                dir_okay=True, writable=False,
                                readable=True, resolve_path=True))
@click.argument('output', nargs=1, required=True,
                type=click.Path(exists=False, file_okay=True, writable=True,
                                dir_okay=False, resolve_path=True))
@click.argument('template', nargs=1, required=False,
                type=click.Path(exists=False, file_okay=True, writable=True,
                                dir_okay=False, resolve_path=True))
@click.argument('inventory_location', nargs=1, required=False,
                type=click.Path(exists=False, file_okay=True, writable=True,
                                dir_okay=False, resolve_path=True))
def attrib(location, output, template=None, inventory_location=None,):
    """
    Generate attribution document at output using the directory of
    ABOUT files at location, the template file (or a default) and an
    inventory_location CSV file containing a list of ABOUT files path to
    generate attribution for.
    """
    click.echo('Running about-code-tool version ' + __version__)
    click.echo('Generating attribution...')
    errors, abouts = about_code_tool.model.collect_inventory(location)
    about_code_tool.attrib.generate_and_save(abouts, output,
                                             template_loc=template,
                                             inventory_location=inventory_location)
    log_errors(errors)

@cli.command(cls=AboutCommand)
def redist(input_dir, output, inventory_location=None,):
    """
    Collect redistributable code at output location using:
     - the input_dir of code and ABOUT files,
     - an inventory_location CSV file containing a list of ABOUT files to
     generate redistribution for.
     Only collect code when redistribute=yes
     Return a list of errors.
    """
    click.echo('Running about-code-tool version ' + __version__)
    click.echo('Collecting redistributable files...')

def log_errors(errors, base_dir=False, level=NOTSET):
    """
    Iterate of sequence of Error objects and print and log errors with a severity
    superior or equal to level.
    """
    logger = logging.getLogger(__name__)
    handler = logging.StreamHandler()
    handler.setLevel(logging.CRITICAL)
    handler.setFormatter(logging.Formatter('%(levelname)s: %(message)s'))
    logger.addHandler(handler)
    file_logger = logging.getLogger(__name__ + '_file')

    msg_format = '%(sever)s: %(message)s'
    # Create error.log
    if base_dir:
        bdir = to_posix(base_dir)
        LOG_FILENAME = 'error.log'
        log_path = join(bdir, LOG_FILENAME)
        if exists(log_path):
            os.remove(log_path)

        file_handler = logging.FileHandler(log_path)
        file_logger.addHandler(file_handler)
    for severity, message in errors:
        sever = about_code_tool.severities[severity]
        if no_stdout == 0:
            print(msg_format % locals())
        if base_dir:
            file_logger.log(30, msg_format % locals())

if __name__ == '__main__':
    cli()
<commit_msg>the global variable for 'no_stdout' should be boolean instead of int/count.<commit_after>#!/usr/bin/env python
# -*- coding: utf8 -*-

# ============================================================================
#  Copyright (c) 2013-2016 nexB Inc. http://www.nexb.com/ - All rights reserved.
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#      http://www.apache.org/licenses/LICENSE-2.0
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
# ============================================================================

from __future__ import print_function

import logging
import codecs

import click
import os
import unicodecsv

import about_code_tool.gen
import about_code_tool.model
import about_code_tool.attrib

from os.path import exists, join
from about_code_tool import NOTSET
from about_code_tool.util import to_posix

__version__ = '3.0.0dev'

__about_spec_version__ = '2.0.0dev'

__copyright__ = """
    Copyright (c) 2013-2016 nexB Inc. All rights reserved.
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at
        http://www.apache.org/licenses/LICENSE-2.0
    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License."""

prog_name = 'AboutCode'
no_stdout = False

intro = '''%(prog_name)s, version %(__version__)s
ABOUT spec version: %(__about_spec_version__)s http://dejacode.org
%(__copyright__)s
''' % locals()

class AboutCommand(click.Command):
    def main(self, args=None, prog_name=None, complete_var=None,
             standalone_mode=True, **extra):
        """
        Workaround click 4.0 bug https://github.com/mitsuhiko/click/issues/365
        """
        return click.Command.main(self, args=args, prog_name=self.name,
                                  complete_var=complete_var,
                                  standalone_mode=standalone_mode, **extra)

@click.group(name='about')
@click.version_option(version=__version__, prog_name=prog_name, message=intro)
@click.option('-v', '--verbose', count=True,
              help='Increase verbosity. Repeat to print more output.')
@click.option('-q', '--quiet', is_flag=True, help='Do not print any output.')
def cli(verbose, quiet):
    # Update the no_stdout value globally
    global no_stdout
    no_stdout = quiet

    pass
    # click.echo('Verbosity: %s' % verbose)

inventory_help = '''
LOCATION: Path to an ABOUT file or a directory containing ABOUT files
OUTPUT: Path to CSV file to write the inventory to
'''
@cli.command(help=inventory_help,
             short_help='LOCATION: directory, OUTPUT: csv file',
             cls=AboutCommand)
@click.argument('location', nargs=1, required=True,
                type=click.Path(exists=True, file_okay=True,
                                dir_okay=True, writable=False,
                                readable=True, resolve_path=True))
@click.argument('output', nargs=1, required=True,
                type=click.Path(exists=False, file_okay=True, writable=True,
                                dir_okay=False, resolve_path=True))
@click.option('--overwrite', is_flag=True, help='Overwrites the output file if it exists')
def inventory(overwrite, location, output):
    """
    Inventory components from an ABOUT file or a directory tree of ABOUT
    files.
    """
    click.echo('Running about-code-tool version ' + __version__)
    # Check is the <OUTPUT> valid.
    if os.path.exists(output) and not overwrite:
        click.echo('ERROR: <output> file already exists.')
        click.echo('Select a different file name or use the --overwrite option after the `inventory`.')
        click.echo()
        return
    if not output.endswith('.csv'):
        click.echo('ERROR: <output> must be a CSV file ending with ".csv".')
        click.echo()
        return
    if not os.path.exists(output):
        click.echo('ERROR: <output> does not exists. Please check and correct the <output>.')
        click.echo()
        return

    click.echo('Collecting the inventory from location: ''%(location)s '
               'and writing CSV output to: %(output)s' % locals())

    errors, abouts = about_code_tool.model.collect_inventory(location)
    log_errors(errors)
    about_code_tool.model.to_csv(abouts, output)

gen_help = '''
LOCATION: Path to a CSV inventory file
OUTPUT: Path to the directory to write ABOUT files to
'''
@cli.command(help=gen_help,
             short_help='LOCATION: csv file, OUTPUT: directory',
             cls=AboutCommand)
@click.argument('location', nargs=1, required=True,
                type=click.Path(exists=True, file_okay=True,
                                dir_okay=False, writable=False,
                                readable=True, resolve_path=True))
@click.argument('output', nargs=1, required=True,
                type=click.Path(exists=True, file_okay=False, writable=True,
                                dir_okay=True, resolve_path=True))
def gen(location, output):
    """
    Given a CVS inventory of ABOUT files at location, generate ABOUT files in
    base directory.
    """
    click.echo('Running about-code-tool version ' + __version__)
    click.echo('Generating ABOUT files...')
    errors, abouts = about_code_tool.gen.generate(location, output)

    lea = len(abouts)
    lee = len(errors)
    click.echo('Generated %(lea)d ABOUT files with %(lee)d errors or warning' % locals())
    log_errors(errors, output)

@cli.command(cls=AboutCommand)
def export():
    click.echo('Running about-code-tool version ' + __version__)
    click.echo('Exporting zip archive...')

@cli.command(cls=AboutCommand)
def fetch(location):
    """
    Given a directory of ABOUT files at location, calls the DejaCode API and
    update or create license data fields and license texts.
    """
    click.echo('Running about-code-tool version ' + __version__)
    click.echo('Updating ABOUT files...')

@cli.command(cls=AboutCommand)
@click.argument('location', nargs=1, required=True,
                type=click.Path(exists=True, file_okay=True,
                                dir_okay=True, writable=False,
                                readable=True, resolve_path=True))
@click.argument('output', nargs=1, required=True,
                type=click.Path(exists=False, file_okay=True, writable=True,
                                dir_okay=False, resolve_path=True))
@click.argument('template', nargs=1, required=False,
                type=click.Path(exists=False, file_okay=True, writable=True,
                                dir_okay=False, resolve_path=True))
@click.argument('inventory_location', nargs=1, required=False,
                type=click.Path(exists=False, file_okay=True, writable=True,
                                dir_okay=False, resolve_path=True))
def attrib(location, output, template=None, inventory_location=None,):
    """
    Generate attribution document at output using the directory of
    ABOUT files at location, the template file (or a default) and an
    inventory_location CSV file containing a list of ABOUT files path to
    generate attribution for.
    """
    click.echo('Running about-code-tool version ' + __version__)
    click.echo('Generating attribution...')
    errors, abouts = about_code_tool.model.collect_inventory(location)
    about_code_tool.attrib.generate_and_save(abouts, output,
                                             template_loc=template,
                                             inventory_location=inventory_location)
    log_errors(errors)

@cli.command(cls=AboutCommand)
def redist(input_dir, output, inventory_location=None,):
    """
    Collect redistributable code at output location using:
     - the input_dir of code and ABOUT files,
     - an inventory_location CSV file containing a list of ABOUT files to
     generate redistribution for.
     Only collect code when redistribute=yes
     Return a list of errors.
    """
    click.echo('Running about-code-tool version ' + __version__)
    click.echo('Collecting redistributable files...')

def log_errors(errors, base_dir=False, level=NOTSET):
    """
    Iterate of sequence of Error objects and print and log errors with a severity
    superior or equal to level.
    """
    logger = logging.getLogger(__name__)
    handler = logging.StreamHandler()
    handler.setLevel(logging.CRITICAL)
    handler.setFormatter(logging.Formatter('%(levelname)s: %(message)s'))
    logger.addHandler(handler)
    file_logger = logging.getLogger(__name__ + '_file')

    msg_format = '%(sever)s: %(message)s'
    # Create error.log
    if base_dir:
        bdir = to_posix(base_dir)
        LOG_FILENAME = 'error.log'
        log_path = join(bdir, LOG_FILENAME)
        if exists(log_path):
            os.remove(log_path)

        file_handler = logging.FileHandler(log_path)
        file_logger.addHandler(file_handler)
    for severity, message in errors:
        sever = about_code_tool.severities[severity]
        if not no_stdout:
            print(msg_format % locals())
        if base_dir:
            file_logger.log(30, msg_format % locals())

if __name__ == '__main__':
    cli()
<|endoftext|>