<commit_before>from __future__ import absolute_import, division, print_function

"""
This is for 3D selection in Glue 3d scatter plot viewer.
"""

import numpy as np

from glue.core import Data
from glue.config import viewer_tool
from glue.core.roi import RectangularROI, CircularROI
from glue.viewers.common.qt.tool import CheckableTool
from glue.core.subset import SubsetState, ElementSubsetState
from glue.core.exceptions import IncompatibleAttribute
from glue.core.edit_subset_mode import EditSubsetMode
from glue.utils.geometry import points_inside_poly

from ..utils import as_matrix_transform
from ..extern.vispy.scene import Rectangle, Line, Ellipse

class VispyMouseMode(CheckableTool):

    # this will create an abstract selection mode class to handle mouse events
    # instanced by lasso, rectangle, circular and point mode

    def __init__(self, viewer):
        super(VispyMouseMode, self).__init__(viewer)
        self._vispy_widget = viewer._vispy_widget
        self.current_visible_array = None

    def get_visible_data(self):
        visible = []
        # Loop over visible layer artists
        for layer_artist in self.viewer._layer_artist_container:
            # Only extract Data objects, not subsets
            if isinstance(layer_artist.layer, Data):
                visible.append(layer_artist.layer)
        visual = layer_artist.visual  # we only have one visual for each canvas
        return visible, visual

    def iter_data_layer_artists(self):
        for layer_artist in self.viewer._layer_artist_container:
            if isinstance(layer_artist.layer, Data):
                yield layer_artist

    def mark_selected(self, mask, data):
        # We now make a subset state. For scatter plots we'll want to use an
        # ElementSubsetState, while for cubes, we'll need to change to a
        # MaskSubsetState.
        subset_state = ElementSubsetState(indices=np.where(mask)[0], data=data)

        # We now check what the selection mode is, and update the selection as
        # needed (this is delegated to the correct subset mode).
        mode = EditSubsetMode()
        mode.update(self.viewer._data, subset_state, focus_data=data)

    def mark_selected_dict(self, mask_dict):
        subset_state = MultiMaskSubsetState(mask_dict=mask_dict)
        mode = EditSubsetMode()
        if len(mask_dict) > 0:
            mode.update(self.viewer._data, subset_state, focus_data=list(mask_dict)[0])

    def set_progress(self, value):
        if value < 0:
            self.viewer.show_status('')
        else:
            self.viewer.show_status('Calculating selection - {0}%'.format(int(value)))

class MultiMaskSubsetState(SubsetState):
    """
    A subset state that can include a different mask for different datasets.

    This is useful when doing 3D selections with multiple datasets. This used
    to be a class called MultiElementSubsetState but it is more efficient to
    store masks than element lists. However, for backward-compatibility,
    values of the mask_dict dictionary can be index lists (recognzied because
    they don't have a boolean type).
    """

    def __init__(self, mask_dict=None):
        super(MultiMaskSubsetState, self).__init__()
        mask_dict_uuid = {}
        for key in mask_dict:
            if isinstance(key, Data):
                mask_dict_uuid[key.uuid] = mask_dict[key]
            else:
                mask_dict_uuid[key] = mask_dict[key]
        self._mask_dict = mask_dict_uuid

    def to_mask(self, data, view=None):
        if data.uuid in self._mask_dict:
            mask = self._mask_dict[data.uuid]
            if mask.dtype.kind != 'b':  # backward-compatibility with indices_dict
                indices = mask
                mask = np.zeros(data.shape, dtype=bool)
                mask.flat[indices] = True
            if view is not None:
                mask = mask[view]
            return mask
        else:
            raise IncompatibleAttribute()

    def copy(self):
        state = MultiMaskSubsetState(mask_dict=self._mask_dict)
        return state

    def __gluestate__(self, context):
        serialized = {key: context.do(value) for key, value in self._mask_dict.items()}
        return {'mask_dict': serialized}

    @classmethod
    def __setgluestate__(cls, rec, context):
        # For backward-compatibility reasons we recognize indices_dict
        if 'indices_dict' in rec:
            mask_dict = {key: context.object(value) for key, value in rec['indices_dict'].items()}
        else:
            mask_dict = {key: context.object(value) for key, value in rec['mask_dict'].items()}
        state = cls(mask_dict=mask_dict)
        return state

# Backward-compatibility for reading files
MultiElementSubsetState = MultiMaskSubsetState

def get_mask_from_scatter(data, visual, vispy_widget, selection, progress=None):

    # Get the component IDs
    x_att = vispy_widget.viewer_state.x_att
    y_att = vispy_widget.viewer_state.y_att
    z_att = vispy_widget.viewer_state.z_att

    # Get the visible data
    layer_data = np.nan_to_num([data[x_att],
                                data[y_att],
                                data[z_att]]).transpose()

    tr = as_matrix_transform(visual.get_transform(map_from='visual', map_to='canvas'))
    data = tr.map(layer_data)
    data /= data[:, 3:]  # normalize with homogeneous coordinates

    return selection(data[:, 0], data[:, 1])

def get_mask_from_volume(data, visual, selection, progress=None):
    """
    Get the mapped buffer from self.visual to canvas.

    :return: Mapped data position on canvas.
    """

    # For large volumes, the code in this function would take up very large
    # amounts of memory, so we need to proceed in chunks. The chunk size is
    # chosen so that the chunking has negligeable performance implications.

    chunk_size = 1000000

    tr = as_matrix_transform(visual.get_transform(map_from='visual',
                                                  map_to='canvas'))

    # We chunk by selecting C-contiguous sets of data slices. We do this by
    # first finding the number of elements along each slice then finding the
    # numbe of slices we can fit in the chunk size

    values_per_slice = data.shape[1] * data.shape[2]
    slices_per_chunk = max(chunk_size // values_per_slice, 1)
    n_chunks = max(data.shape[0] // slices_per_chunk, 1)

    mask = np.zeros(data.shape, dtype=bool)

    for chunk in range(n_chunks):

        imin = chunk * slices_per_chunk
        imax = min((chunk + 1) * slices_per_chunk, data.shape[0])

        chunk_shape = (imax - imin,) + data.shape[1:]

        pos_data = np.indices(chunk_shape[::-1], dtype=float)
        pos_data[2] += imin
        pos_data = pos_data.reshape(3, -1).transpose()

        data_sub = tr.map(pos_data)

        data_sub /= data_sub[:, 3:]   # normalize with homogeneous coordinates

        mask_sub = selection(data_sub[:, 0], data_sub[:, 1])

        mask_sub = np.reshape(mask_sub, chunk_shape[::-1])
        mask_sub = np.transpose(mask_sub)

        mask[imin:imax] = mask_sub

        if progress is not None:
            progress(100. * (chunk + 1) / n_chunks)

    return mask

def get_mask_for_layer_artist(layer_artist, viewer, selection, progress=None):

    from ..scatter.layer_artist import ScatterLayerArtist
    from ..volume.layer_artist import VolumeLayerArtist

    if isinstance(layer_artist, ScatterLayerArtist):
        return get_mask_from_scatter(layer_artist.layer, layer_artist.visual,
                                     viewer._vispy_widget, selection, progress=progress)
    elif isinstance(layer_artist, VolumeLayerArtist):
        return get_mask_from_volume(layer_artist.layer, layer_artist.visual,
                                    selection, progress=progress)
    else:
        raise Exception("Unknown layer type: {0}".format(type(layer_artist)))

@viewer_tool
class LassoSelectionMode(VispyMouseMode):

    icon = 'glue_lasso'
    tool_id = 'vispy:lasso'
    action_text = 'Select data using a lasso selection'

    def __init__(self, viewer):
        super(LassoSelectionMode, self).__init__(viewer)
        self.line = Line(color='purple',
                         width=2, method='agg',
                         parent=self._vispy_widget.canvas.scene)

    def activate(self):
        self.reset()

    def reset(self):
        self.line_pos = []
        self.line.set_data(np.zeros((0, 2), dtype=float))
        self.line.parent = None

    def press(self, event):
        if event.button == 1:
            self.line_pos.append(event.pos)

    def move(self, event):
        if event.button == 1 and event.is_dragging:
            self.line_pos.append(event.pos)
            self.line.set_data(np.array(self.line_pos, dtype=float))
            self.line.parent = self._vispy_widget.canvas.scene

    def release(self, event):

        if event.button == 1:

            if len(self.line_pos) > 0:

                mask_dict = {}

                self.set_progress(0)

                for layer_artist in self.iter_data_layer_artists():

                    vx, vy = np.array(self.line_pos).transpose()

                    def selection(x, y):
                        return points_inside_poly(x, y, vx, vy)

                    mask = get_mask_for_layer_artist(layer_artist, self.viewer,
                                                     selection, progress=self.set_progress)

                    mask_dict[layer_artist.layer] = mask

                self.mark_selected_dict(mask_dict)

                self.set_progress(-1)

            self.reset()

@viewer_tool
class RectangleSelectionMode(VispyMouseMode):

    icon = 'glue_square'
    tool_id = 'vispy:rectangle'
    action_text = 'Select data using a rectangular selection'

    def __init__(self, viewer):
        super(RectangleSelectionMode, self).__init__(viewer)
        self.rectangle = Rectangle(center=(0, 0), width=1, height=1, border_width=2,
                                   color=(0, 0, 0, 0), border_color='purple')

    def activate(self):
        self.reset()

    def reset(self):
        self.corner1 = None
        self.corner2 = None
        self.rectangle.parent = None

    def press(self, event):
        if event.button == 1:
            self.corner1 = event.pos

    def move(self, event):
        if event.button == 1 and event.is_dragging:
            self.corner2 = event.pos
            x1, y1 = self.corner1
            x2, y2 = self.corner2
            if abs(x2 - x1) > 0 and abs(y2 - y1) > 0:
                self.rectangle.center = 0.5 * (x1 + x2), 0.5 * (y1 + y2)
                self.rectangle.width = abs(x2 - x1)
                self.rectangle.height = abs(y2 - y1)
                self.rectangle.parent = self._vispy_widget.canvas.scene

    @property
    def bounds(self):
        x1, y1 = self.corner1
        x2, y2 = self.corner2
        return (min(x1, x2), max(x1, x2), min(y1, y2), max(y1, y2))

    def release(self, event):

        if event.button == 1:

            if self.corner2 is not None:

                r = RectangularROI(*self.bounds)

                mask_dict = {}

                self.set_progress(0)

                for layer_artist in self.iter_data_layer_artists():

                    def selection(x, y):
                        return r.contains(x, y)

                    mask = get_mask_for_layer_artist(layer_artist, self.viewer,
                                                     selection, progress=self.set_progress)

                    mask_dict[layer_artist.layer] = mask

                self.mark_selected_dict(mask_dict)

                self.set_progress(-1)

            self.reset()

@viewer_tool
class CircleSelectionMode(VispyMouseMode):

    icon = 'glue_circle'
    tool_id = 'vispy:circle'
    action_text = 'Select data using a circular selection'

    def __init__(self, viewer):
        super(CircleSelectionMode, self).__init__(viewer)
        self.ellipse = Ellipse(center=(0, 0), radius=1, border_width=2,
                               color=(0, 0, 0, 0), border_color='purple')

    def activate(self):
        self.reset()

    def reset(self):
        self.center = None
        self.radius = 0
        self.ellipse.parent = None

    def press(self, event):
        if event.button == 1:
            self.center = event.pos

    def move(self, event):
        if event.button == 1 and event.is_dragging:
            self.radius = max(abs(event.pos[0] - self.center[0]),
                              abs(event.pos[1] - self.center[1]))
            if self.radius > 0:
                self.ellipse.center = self.center
                self.ellipse.radius = self.radius
                self.ellipse.parent = self._vispy_widget.canvas.scene

    def release(self, event):

        if event.button == 1:

            if self.radius > 0:
                c = CircularROI(self.center[0], self.center[1], self.radius)

                mask_dict = {}

                self.set_progress(0)

                for layer_artist in self.iter_data_layer_artists():

                    def selection(x, y):
                        return c.contains(x, y)

                    mask = get_mask_for_layer_artist(layer_artist, self.viewer,
                                                     selection, progress=self.set_progress)

                    mask_dict[layer_artist.layer] = mask

                self.mark_selected_dict(mask_dict)

                self.set_progress(-1)

            self.reset()
<commit_msg>Fix compatibility with latest version of glue in which EditSubsetMode is now instantiated at the Application-level instead of being a singleton.<commit_after>from __future__ import absolute_import, division, print_function

"""
This is for 3D selection in Glue 3d scatter plot viewer.
"""

import numpy as np

from glue.core import Data
from glue.config import viewer_tool
from glue.core.roi import RectangularROI, CircularROI
from glue.viewers.common.qt.tool import CheckableTool
from glue.core.subset import SubsetState, ElementSubsetState
from glue.core.exceptions import IncompatibleAttribute
from glue.core.edit_subset_mode import EditSubsetMode
from glue.utils.geometry import points_inside_poly

from ..utils import as_matrix_transform
from ..extern.vispy.scene import Rectangle, Line, Ellipse

class VispyMouseMode(CheckableTool):

    # this will create an abstract selection mode class to handle mouse events
    # instanced by lasso, rectangle, circular and point mode

    def __init__(self, viewer):
        super(VispyMouseMode, self).__init__(viewer)
        self._vispy_widget = viewer._vispy_widget
        self.current_visible_array = None

    def get_visible_data(self):
        visible = []
        # Loop over visible layer artists
        for layer_artist in self.viewer._layer_artist_container:
            # Only extract Data objects, not subsets
            if isinstance(layer_artist.layer, Data):
                visible.append(layer_artist.layer)
        visual = layer_artist.visual  # we only have one visual for each canvas
        return visible, visual

    def iter_data_layer_artists(self):
        for layer_artist in self.viewer._layer_artist_container:
            if isinstance(layer_artist.layer, Data):
                yield layer_artist

    def mark_selected(self, mask, data):
        # We now make a subset state. For scatter plots we'll want to use an
        # ElementSubsetState, while for cubes, we'll need to change to a
        # MaskSubsetState.
        subset_state = ElementSubsetState(indices=np.where(mask)[0], data=data)

        # We now check what the selection mode is, and update the selection as
        # needed (this is delegated to the correct subset mode).
        try:
            mode = self.viewer.session.edit_subset_mode
        except AttributeError:  # old versisons of glue
            mode = EditSubsetMode()
        mode.update(self.viewer._data, subset_state, focus_data=data)

    def mark_selected_dict(self, mask_dict):
        subset_state = MultiMaskSubsetState(mask_dict=mask_dict)
        if len(mask_dict) > 0:
            try:
                mode = self.viewer.session.edit_subset_mode
            except AttributeError:  # old versisons of glue
                mode = EditSubsetMode()
            mode.update(self.viewer._data, subset_state, focus_data=list(mask_dict)[0])

    def set_progress(self, value):
        if value < 0:
            self.viewer.show_status('')
        else:
            self.viewer.show_status('Calculating selection - {0}%'.format(int(value)))

class MultiMaskSubsetState(SubsetState):
    """
    A subset state that can include a different mask for different datasets.

    This is useful when doing 3D selections with multiple datasets. This used
    to be a class called MultiElementSubsetState but it is more efficient to
    store masks than element lists. However, for backward-compatibility,
    values of the mask_dict dictionary can be index lists (recognzied because
    they don't have a boolean type).
    """

    def __init__(self, mask_dict=None):
        super(MultiMaskSubsetState, self).__init__()
        mask_dict_uuid = {}
        for key in mask_dict:
            if isinstance(key, Data):
                mask_dict_uuid[key.uuid] = mask_dict[key]
            else:
                mask_dict_uuid[key] = mask_dict[key]
        self._mask_dict = mask_dict_uuid

    def to_mask(self, data, view=None):
        if data.uuid in self._mask_dict:
            mask = self._mask_dict[data.uuid]
            if mask.dtype.kind != 'b':  # backward-compatibility with indices_dict
                indices = mask
                mask = np.zeros(data.shape, dtype=bool)
                mask.flat[indices] = True
            if view is not None:
                mask = mask[view]
            return mask
        else:
            raise IncompatibleAttribute()

    def copy(self):
        state = MultiMaskSubsetState(mask_dict=self._mask_dict)
        return state

    def __gluestate__(self, context):
        serialized = {key: context.do(value) for key, value in self._mask_dict.items()}
        return {'mask_dict': serialized}

    @classmethod
    def __setgluestate__(cls, rec, context):
        # For backward-compatibility reasons we recognize indices_dict
        if 'indices_dict' in rec:
            mask_dict = {key: context.object(value) for key, value in rec['indices_dict'].items()}
        else:
            mask_dict = {key: context.object(value) for key, value in rec['mask_dict'].items()}
        state = cls(mask_dict=mask_dict)
        return state

# Backward-compatibility for reading files
MultiElementSubsetState = MultiMaskSubsetState

def get_mask_from_scatter(data, visual, vispy_widget, selection, progress=None):

    # Get the component IDs
    x_att = vispy_widget.viewer_state.x_att
    y_att = vispy_widget.viewer_state.y_att
    z_att = vispy_widget.viewer_state.z_att

    # Get the visible data
    layer_data = np.nan_to_num([data[x_att],
                                data[y_att],
                                data[z_att]]).transpose()

    tr = as_matrix_transform(visual.get_transform(map_from='visual', map_to='canvas'))
    data = tr.map(layer_data)
    data /= data[:, 3:]  # normalize with homogeneous coordinates

    return selection(data[:, 0], data[:, 1])

def get_mask_from_volume(data, visual, selection, progress=None):
    """
    Get the mapped buffer from self.visual to canvas.

    :return: Mapped data position on canvas.
    """

    # For large volumes, the code in this function would take up very large
    # amounts of memory, so we need to proceed in chunks. The chunk size is
    # chosen so that the chunking has negligeable performance implications.

    chunk_size = 1000000

    tr = as_matrix_transform(visual.get_transform(map_from='visual',
                                                  map_to='canvas'))

    # We chunk by selecting C-contiguous sets of data slices. We do this by
    # first finding the number of elements along each slice then finding the
    # numbe of slices we can fit in the chunk size

    values_per_slice = data.shape[1] * data.shape[2]
    slices_per_chunk = max(chunk_size // values_per_slice, 1)
    n_chunks = max(data.shape[0] // slices_per_chunk, 1)

    mask = np.zeros(data.shape, dtype=bool)

    for chunk in range(n_chunks):

        imin = chunk * slices_per_chunk
        imax = min((chunk + 1) * slices_per_chunk, data.shape[0])

        chunk_shape = (imax - imin,) + data.shape[1:]

        pos_data = np.indices(chunk_shape[::-1], dtype=float)
        pos_data[2] += imin
        pos_data = pos_data.reshape(3, -1).transpose()

        data_sub = tr.map(pos_data)

        data_sub /= data_sub[:, 3:]   # normalize with homogeneous coordinates

        mask_sub = selection(data_sub[:, 0], data_sub[:, 1])

        mask_sub = np.reshape(mask_sub, chunk_shape[::-1])
        mask_sub = np.transpose(mask_sub)

        mask[imin:imax] = mask_sub

        if progress is not None:
            progress(100. * (chunk + 1) / n_chunks)

    return mask

def get_mask_for_layer_artist(layer_artist, viewer, selection, progress=None):

    from ..scatter.layer_artist import ScatterLayerArtist
    from ..volume.layer_artist import VolumeLayerArtist

    if isinstance(layer_artist, ScatterLayerArtist):
        return get_mask_from_scatter(layer_artist.layer, layer_artist.visual,
                                     viewer._vispy_widget, selection, progress=progress)
    elif isinstance(layer_artist, VolumeLayerArtist):
        return get_mask_from_volume(layer_artist.layer, layer_artist.visual,
                                    selection, progress=progress)
    else:
        raise Exception("Unknown layer type: {0}".format(type(layer_artist)))

@viewer_tool
class LassoSelectionMode(VispyMouseMode):

    icon = 'glue_lasso'
    tool_id = 'vispy:lasso'
    action_text = 'Select data using a lasso selection'

    def __init__(self, viewer):
        super(LassoSelectionMode, self).__init__(viewer)
        self.line = Line(color='purple',
                         width=2, method='agg',
                         parent=self._vispy_widget.canvas.scene)

    def activate(self):
        self.reset()

    def reset(self):
        self.line_pos = []
        self.line.set_data(np.zeros((0, 2), dtype=float))
        self.line.parent = None

    def press(self, event):
        if event.button == 1:
            self.line_pos.append(event.pos)

    def move(self, event):
        if event.button == 1 and event.is_dragging:
            self.line_pos.append(event.pos)
            self.line.set_data(np.array(self.line_pos, dtype=float))
            self.line.parent = self._vispy_widget.canvas.scene

    def release(self, event):

        if event.button == 1:

            if len(self.line_pos) > 0:

                mask_dict = {}

                self.set_progress(0)

                for layer_artist in self.iter_data_layer_artists():

                    vx, vy = np.array(self.line_pos).transpose()

                    def selection(x, y):
                        return points_inside_poly(x, y, vx, vy)

                    mask = get_mask_for_layer_artist(layer_artist, self.viewer,
                                                     selection, progress=self.set_progress)

                    mask_dict[layer_artist.layer] = mask

                self.mark_selected_dict(mask_dict)

                self.set_progress(-1)

            self.reset()

@viewer_tool
class RectangleSelectionMode(VispyMouseMode):

    icon = 'glue_square'
    tool_id = 'vispy:rectangle'
    action_text = 'Select data using a rectangular selection'

    def __init__(self, viewer):
        super(RectangleSelectionMode, self).__init__(viewer)
        self.rectangle = Rectangle(center=(0, 0), width=1, height=1, border_width=2,
                                   color=(0, 0, 0, 0), border_color='purple')

    def activate(self):
        self.reset()

    def reset(self):
        self.corner1 = None
        self.corner2 = None
        self.rectangle.parent = None

    def press(self, event):
        if event.button == 1:
            self.corner1 = event.pos

    def move(self, event):
        if event.button == 1 and event.is_dragging:
            self.corner2 = event.pos
            x1, y1 = self.corner1
            x2, y2 = self.corner2
            if abs(x2 - x1) > 0 and abs(y2 - y1) > 0:
                self.rectangle.center = 0.5 * (x1 + x2), 0.5 * (y1 + y2)
                self.rectangle.width = abs(x2 - x1)
                self.rectangle.height = abs(y2 - y1)
                self.rectangle.parent = self._vispy_widget.canvas.scene

    @property
    def bounds(self):
        x1, y1 = self.corner1
        x2, y2 = self.corner2
        return (min(x1, x2), max(x1, x2), min(y1, y2), max(y1, y2))

    def release(self, event):

        if event.button == 1:

            if self.corner2 is not None:

                r = RectangularROI(*self.bounds)

                mask_dict = {}

                self.set_progress(0)

                for layer_artist in self.iter_data_layer_artists():

                    def selection(x, y):
                        return r.contains(x, y)

                    mask = get_mask_for_layer_artist(layer_artist, self.viewer,
                                                     selection, progress=self.set_progress)

                    mask_dict[layer_artist.layer] = mask

                self.mark_selected_dict(mask_dict)

                self.set_progress(-1)

            self.reset()

@viewer_tool
class CircleSelectionMode(VispyMouseMode):

    icon = 'glue_circle'
    tool_id = 'vispy:circle'
    action_text = 'Select data using a circular selection'

    def __init__(self, viewer):
        super(CircleSelectionMode, self).__init__(viewer)
        self.ellipse = Ellipse(center=(0, 0), radius=1, border_width=2,
                               color=(0, 0, 0, 0), border_color='purple')

    def activate(self):
        self.reset()

    def reset(self):
        self.center = None
        self.radius = 0
        self.ellipse.parent = None

    def press(self, event):
        if event.button == 1:
            self.center = event.pos

    def move(self, event):
        if event.button == 1 and event.is_dragging:
            self.radius = max(abs(event.pos[0] - self.center[0]),
                              abs(event.pos[1] - self.center[1]))
            if self.radius > 0:
                self.ellipse.center = self.center
                self.ellipse.radius = self.radius
                self.ellipse.parent = self._vispy_widget.canvas.scene

    def release(self, event):

        if event.button == 1:

            if self.radius > 0:
                c = CircularROI(self.center[0], self.center[1], self.radius)

                mask_dict = {}

                self.set_progress(0)

                for layer_artist in self.iter_data_layer_artists():

                    def selection(x, y):
                        return c.contains(x, y)

                    mask = get_mask_for_layer_artist(layer_artist, self.viewer,
                                                     selection, progress=self.set_progress)

                    mask_dict[layer_artist.layer] = mask

                self.mark_selected_dict(mask_dict)

                self.set_progress(-1)

            self.reset()
<|endoftext|>


<commit_before>import numpy as np
import pandas as pd
import json
from tl_algs import tl_alg

def sim_minmax(column):
    """Similarity score using the range between min and max
    for a value

    Args:
      column: a given feature column

    Returns:
      tuple: A tuple of the form (min, max)

    """
    return min(column), max(column)

def sim_std(column):
    """Similarity score using the standard error for a column

    Args:
      column: a given feature column

    Returns:
      tuple: tuple with the first element one std dev below the mean
      and the second element one std dev above the mean

    """
    return (np.mean(column) - np.std(column), np.mean(column) + np.std(column))

class TransferNaiveBayes(tl_alg.Base_Transfer):
    """
    Transfer Naive Bayes algorithm, as described by Ma [1]

    Args:
        similarity_func: the function used to determine if two features
                          are similar
        num_disc_bins: the number of bins to use for discretization
        discretize: whether to discretize (if passing in already categorical
                    data, set this to false.

    Returns:
        A classifier

    [1] Ma, Y., Luo, G., Zeng, X., & Chen, A. (2012). Transfer learning
    for cross-company software defect prediction. Information and
    Software Technology, 54(3), 248-256.
    https://doi.org/10.1016/j.infsof.2011.09.007
    """

    def __init__(
            self,
            test_set_X,
            test_set_domain,
            train_pool_X,
            train_pool_y,
            train_pool_domain,
            rand_seed=None,
            classifier_params={},
            similarity_func=sim_std,
            num_disc_bins=10,
            alpha=1,
            discretize=True):

        super(
            TransferNaiveBayes,
            self).__init__(
            test_set_X,
            test_set_domain,
            train_pool_X,
            train_pool_y,
            train_pool_domain,
            None,
            rand_seed=rand_seed,
            classifier_params=classifier_params)

        self.similarity_func = similarity_func
        self.num_disc_bins = 10
        self.discretize = discretize
        self.alpha = alpha

        self.cached_n_j = {}
        self.cached_n_c = None
        self.cached_cond_prob = {}

    def isinrange(self, row, ranges):
        """ returns a boolean vector where the ith entry corresponds
        to whether the ith feature is within the range specified in ranges

        Args:
            row: The row to check
            ranges: a list of the form [(low, high),...] for each feature

        Returns:
            A boolean vector

        """
        return sum([cell >= ranges[i][0] and cell <= ranges[i][1]
            for i, cell in enumerate(row)])

    def get_weights(self):
        """
        Gets the prior distirbutions for the features (aka the weights)
        """
        # ranges is analogous to min_ij and max_ij in the paper
        ranges = self.test_set_X.apply(self.similarity_func)
        # for each row, compute similarity, analogous to s_i in the paper
        similarity_count = self.train_pool_X.apply(lambda x: self.isinrange(x, ranges), axis=1)
        # for each row, calculate weight, analogous to w_i in the paper
        # note that in the paper the number of features (train_pool_X.shape[0]) is k
        weight_vec = map(lambda x: float(
            float(x) / (self.train_pool_X.shape[1] - x + 1)**2), similarity_count)
        return weight_vec

    def get_discretized_X(self):
        """
        Creates discretized versions of the training and test instances.  Test instances are
        used as the "baseline" which dictates the bins.  The training instances are then
        discretized using the same bin edges.  The number of bins is controlled by the
        self.num_disc_bins parameter.

        Returns:
            (X_train_disc, X_test_disc)
        """
        test_disc_arr, train_disc_arr = [], []
        for col_ind in range(self.test_set_X.shape[1]):
            # start with discretizing test set, save bins
            try:
                test_disc, bins = pd.cut(self.test_set_X.iloc[:, col_ind],
                            bins = self.num_disc_bins,
                            labels=map(str, range(self.num_disc_bins)),
                            retbins=True)
            except ValueError:
                test_disc, bins = pd.cut(self.test_set_X.iloc[:, col_ind],
                            bins = [float('inf') * -1, float('inf')],
                            labels = [str(0)],
                            retbins=True)

            # makde sure bins cover entire interval
            bins[0] = -1 * float('inf')
            bins[-1] = float('inf')
            # use (modified) test set bins for training set discretization
            train_disc = pd.cut(self.train_pool_X.iloc[:, col_ind],
                            bins=bins,
                            labels=map(str, range(len(bins)-1)))
            test_disc_arr.append(test_disc)
            train_disc_arr.append(train_disc)
        # combine discretized series to data frame
        return pd.concat(train_disc_arr, axis=1), pd.concat(test_disc_arr, axis=1)

    def get_cached_n_j(self, feature_index, X_weighted):
        if feature_index not in self.cached_n_j.keys():
            self.cached_n_j[feature_index] = len(X_weighted.iloc[:, feature_index].unique())
        return self.cached_n_j[feature_index]

    def get_cached_n_c(self):
        if not self.cached_n_c:
            self.cached_n_c = len(self.train_pool_y.unique())
        return self.cached_n_c

    def get_cached_conditional_prob(self, label, feature_index, feature_val,
            X_weighted, n_c, n_j, alpha):
        if (label, feature_index, feature_val) not in self.cached_cond_prob.keys():
            feature_mask = np.asarray(self.train_pool_y == label).reshape(-1) & \
                np.asarray(X_weighted.iloc[:, feature_index] == feature_val)\
                .reshape(-1)
            class_mask = np.asarray(self.train_pool_y == label).reshape(-1)
            self.cached_cond_prob[(label, feature_index, feature_val)] = \
                    (X_weighted.myweight.loc[feature_mask].sum() + alpha) / \
                    (X_weighted.myweight.loc[class_mask].sum() + n_c * alpha)
        return self.cached_cond_prob[(label, feature_index, feature_val)]

    def get_class_prob(self, X_weighted, label, alpha):
        """ Computes P(C) according to equation 7 in [1]

        Args:
            X_weighted: dataframe with columns of the form [feature1, feature2,
                        ..., weight]
            label: the label to calculate the probability of
            alpha: the laplace smoothing factor, default is 1
        """
        # number of classes
        n_c = len(self.train_pool_y.unique())
        mask = np.asarray(self.train_pool_y == label).reshape(-1)
        return (X_weighted[mask].myweight.sum() + alpha) / \
                    (X_weighted.myweight.sum() + n_c * alpha)

    def get_conditional_prob(self, X_weighted, label, feature_index,
            feature_val, alpha):
        """ Computes P(a_j|c), where a_j is value j for feature a, and
        c is the class

        Calculated according to equation 8 from [1]

        Args:
            X_weighted: dataframe with columns of the form [feature1, feature2,
                        ..., weight]
            label: the label to calculate the probability of
            feature_val: the value of the feature to calculate the prob. of
            alpha: the laplace smoothing factor, default is 1
        """
        n_j = self.get_cached_n_j(feature_index, X_weighted)
        n_c = self.get_cached_n_c()
        return self.get_cached_conditional_prob(label, feature_index, feature_val,
                X_weighted, n_c, n_j, alpha)

    def get_posterior_prob(self, X_weighted, label, instance, alpha):
        """ Compute P(c|u), where c is the class and u is the train instance

        Calculated according to equation 1 from [1]

        Args:
            label: the label to calculate the probability of
            instance: a row from the training set

        Returns:
            P(c|u)
        """

        # building numerator
        numerator = self.get_class_prob(X_weighted, label, alpha)
        # column index = j
        for j in range(len(instance)):
            numerator *= self.get_conditional_prob(X_weighted,
                    label = label, feature_index = j, feature_val=instance[j],
                    alpha=alpha)

        # building denominator
        denominator = 0
        for c in self.train_pool_y.unique():
            term = self.get_class_prob(X_weighted, c, alpha)
            for j in range(len(instance)):
                term *= self.get_conditional_prob(X_weighted,
                        label = c, feature_index = j,
                        feature_val=instance[j],
                        alpha=alpha)
            denominator += term

        return numerator / denominator if denominator != 0 else 0

    def train_filter_test(self):
        """Applies weight filter and returns predictions"""
        weights = self.get_weights()

        if self.discretize:
            X_train_disc, X_test_disc = self.get_discretized_X()
        else:
            X_train_disc, X_test_disc = self.train_pool_X, self.test_set_X

        X_train_disc['myweight'] = weights

        y_pred, y_conf = [], []
        for __, row in X_test_disc.iterrows():
            class_probs = [self.get_posterior_prob(X_train_disc,
                c, row, self.alpha) for c in [False, True]]
            i = np.argmax(class_probs)
            y_pred.append(i == 1)
            # always get probability of positive prediction
            y_conf.append(class_probs[1])
        return np.array(y_conf), np.array(y_pred)

    def json_encode(self):
        """Encodes this class as a json object"""
        base = tl_alg.Base_Transfer.json_encode(self)
        base.update({"similarity_func": self.similarity_func.__name__,
            "num_disc_bins" : self.num_disc_bins,
            "discretize" : self.discretize
            })
        return base
<commit_msg>fixes bug in tnb<commit_after>import numpy as np
import pandas as pd
import json
from tl_algs import tl_alg

def sim_minmax(column):
    """Similarity score using the range between min and max
    for a value

    Args:
      column: a given feature column

    Returns:
      tuple: A tuple of the form (min, max)

    """
    return min(column), max(column)

def sim_std(column):
    """Similarity score using the standard error for a column

    Args:
      column: a given feature column

    Returns:
      tuple: tuple with the first element one std dev below the mean
      and the second element one std dev above the mean

    """
    return (np.mean(column) - np.std(column), np.mean(column) + np.std(column))

class TransferNaiveBayes(tl_alg.Base_Transfer):
    """
    Transfer Naive Bayes algorithm, as described by Ma [1]

    Args:
        similarity_func: the function used to determine if two features
                          are similar
        num_disc_bins: the number of bins to use for discretization
        discretize: whether to discretize (if passing in already categorical
                    data, set this to false.

    Returns:
        A classifier

    [1] Ma, Y., Luo, G., Zeng, X., & Chen, A. (2012). Transfer learning
    for cross-company software defect prediction. Information and
    Software Technology, 54(3), 248-256.
    https://doi.org/10.1016/j.infsof.2011.09.007
    """

    def __init__(
            self,
            test_set_X,
            test_set_domain,
            train_pool_X,
            train_pool_y,
            train_pool_domain,
            rand_seed=None,
            classifier_params={},
            similarity_func=sim_std,
            num_disc_bins=10,
            alpha=1,
            discretize=True):

        super(
            TransferNaiveBayes,
            self).__init__(
            test_set_X,
            test_set_domain,
            train_pool_X,
            train_pool_y,
            train_pool_domain,
            None,
            rand_seed=rand_seed,
            classifier_params=classifier_params)

        self.similarity_func = similarity_func
        self.num_disc_bins = num_disc_bins
        self.discretize = discretize
        self.alpha = alpha

        self.cached_n_j = {}
        self.cached_n_c = None
        self.cached_cond_prob = {}

    def isinrange(self, row, ranges):
        """ returns a boolean vector where the ith entry corresponds
        to whether the ith feature is within the range specified in ranges

        Args:
            row: The row to check
            ranges: a list of the form [(low, high),...] for each feature

        Returns:
            A boolean vector

        """
        return sum([cell >= ranges[i][0] and cell <= ranges[i][1]
            for i, cell in enumerate(row)])

    def get_weights(self):
        """
        Gets the prior distirbutions for the features (aka the weights)
        """
        # ranges is analogous to min_ij and max_ij in the paper
        ranges = self.test_set_X.apply(self.similarity_func)
        # for each row, compute similarity, analogous to s_i in the paper
        similarity_count = self.train_pool_X.apply(lambda x: self.isinrange(x, ranges), axis=1)
        # for each row, calculate weight, analogous to w_i in the paper
        # note that in the paper the number of features (train_pool_X.shape[0]) is k
        weight_vec = map(lambda x: float(
            float(x) / (self.train_pool_X.shape[1] - x + 1)**2), similarity_count)
        return weight_vec

    def get_discretized_X(self):
        """
        Creates discretized versions of the training and test instances.  Test instances are
        used as the "baseline" which dictates the bins.  The training instances are then
        discretized using the same bin edges.  The number of bins is controlled by the
        self.num_disc_bins parameter.

        Returns:
            (X_train_disc, X_test_disc)
        """
        test_disc_arr, train_disc_arr = [], []
        for col_ind in range(self.test_set_X.shape[1]):
            # start with discretizing test set, save bins
            try:
                test_disc, bins = pd.cut(self.test_set_X.iloc[:, col_ind],
                            bins = self.num_disc_bins,
                            labels=map(str, range(self.num_disc_bins)),
                            retbins=True)
            except ValueError:
                test_disc, bins = pd.cut(self.test_set_X.iloc[:, col_ind],
                            bins = [float('inf') * -1, float('inf')],
                            labels = [str(0)],
                            retbins=True)

            # makde sure bins cover entire interval
            bins[0] = -1 * float('inf')
            bins[-1] = float('inf')
            # use (modified) test set bins for training set discretization
            train_disc = pd.cut(self.train_pool_X.iloc[:, col_ind],
                            bins=bins,
                            labels=map(str, range(len(bins)-1)))
            test_disc_arr.append(test_disc)
            train_disc_arr.append(train_disc)
        # combine discretized series to data frame
        return pd.concat(train_disc_arr, axis=1), pd.concat(test_disc_arr, axis=1)

    def get_cached_n_j(self, feature_index, X_weighted):
        if feature_index not in self.cached_n_j.keys():
            self.cached_n_j[feature_index] = len(X_weighted.iloc[:, feature_index].unique())
        return self.cached_n_j[feature_index]

    def get_cached_n_c(self):
        if not self.cached_n_c:
            self.cached_n_c = len(self.train_pool_y.unique())
        return self.cached_n_c

    def get_cached_conditional_prob(self, label, feature_index, feature_val,
            X_weighted, n_c, n_j, alpha):
        if (label, feature_index, feature_val) not in self.cached_cond_prob.keys():
            feature_mask = np.asarray(self.train_pool_y == label).reshape(-1) & \
                np.asarray(X_weighted.iloc[:, feature_index] == feature_val)\
                .reshape(-1)
            class_mask = np.asarray(self.train_pool_y == label).reshape(-1)
            self.cached_cond_prob[(label, feature_index, feature_val)] = \
                    (X_weighted.myweight.loc[feature_mask].sum() + alpha) / \
                    (X_weighted.myweight.loc[class_mask].sum() + n_c * alpha)
        return self.cached_cond_prob[(label, feature_index, feature_val)]

    def get_class_prob(self, X_weighted, label, alpha):
        """ Computes P(C) according to equation 7 in [1]

        Args:
            X_weighted: dataframe with columns of the form [feature1, feature2,
                        ..., weight]
            label: the label to calculate the probability of
            alpha: the laplace smoothing factor, default is 1
        """
        # number of classes
        n_c = len(self.train_pool_y.unique())
        mask = np.asarray(self.train_pool_y == label).reshape(-1)
        return (X_weighted[mask].myweight.sum() + alpha) / \
                    (X_weighted.myweight.sum() + n_c * alpha)

    def get_conditional_prob(self, X_weighted, label, feature_index,
            feature_val, alpha):
        """ Computes P(a_j|c), where a_j is value j for feature a, and
        c is the class

        Calculated according to equation 8 from [1]

        Args:
            X_weighted: dataframe with columns of the form [feature1, feature2,
                        ..., weight]
            label: the label to calculate the probability of
            feature_val: the value of the feature to calculate the prob. of
            alpha: the laplace smoothing factor, default is 1
        """
        n_j = self.get_cached_n_j(feature_index, X_weighted)
        n_c = self.get_cached_n_c()
        return self.get_cached_conditional_prob(label, feature_index, feature_val,
                X_weighted, n_c, n_j, alpha)

    def get_posterior_prob(self, X_weighted, label, instance, alpha):
        """ Compute P(c|u), where c is the class and u is the train instance

        Calculated according to equation 1 from [1]

        Args:
            label: the label to calculate the probability of
            instance: a row from the training set

        Returns:
            P(c|u)
        """

        # building numerator
        numerator = self.get_class_prob(X_weighted, label, alpha)
        # column index = j
        for j in range(len(instance)):
            numerator *= self.get_conditional_prob(X_weighted,
                    label = label, feature_index = j, feature_val=instance[j],
                    alpha=alpha)

        # building denominator
        denominator = 0
        for c in self.train_pool_y.unique():
            term = self.get_class_prob(X_weighted, c, alpha)
            for j in range(len(instance)):
                term *= self.get_conditional_prob(X_weighted,
                        label = c, feature_index = j,
                        feature_val=instance[j],
                        alpha=alpha)
            denominator += term

        return numerator / denominator if denominator != 0 else 0

    def train_filter_test(self):
        """Applies weight filter and returns predictions"""
        weights = self.get_weights()

        if self.discretize:
            X_train_disc, X_test_disc = self.get_discretized_X()
        else:
            X_train_disc, X_test_disc = self.train_pool_X, self.test_set_X

        X_train_disc['myweight'] = weights

        y_pred, y_conf = [], []
        for __, row in X_test_disc.iterrows():
            class_probs = [self.get_posterior_prob(X_train_disc,
                c, row, self.alpha) for c in [False, True]]
            i = np.argmax(class_probs)
            y_pred.append(i == 1)
            # always get probability of positive prediction
            y_conf.append(class_probs[1])
        return np.array(y_conf), np.array(y_pred)

    def json_encode(self):
        """Encodes this class as a json object"""
        base = tl_alg.Base_Transfer.json_encode(self)
        base.update({"similarity_func": self.similarity_func.__name__,
            "num_disc_bins" : self.num_disc_bins,
            "discretize" : self.discretize
            })
        return base
<|endoftext|>


<commit_before>import http.cookiejar, urllib.request
class CouchDbHttpApiBase():
    def __init__(self):
        self.cookieJar = http.cookiejar.LWPCookieJar()
        self.cookieProcessor = urllib.request.HTTPCookieProcessor(self.cookieJar)
        self.openerDirector = urllib.request.build_opener(self.cookieProcessor)
        print(self.openerDirector)
        self.cookieJar.load("cookie.txt")

    def saveCookie(self):
        self.cookieJar.save("cookie.txt")

    def getMotd(self):
        r = self.openerDirector.open("http://127.0.0.1:5984/")
        print (r.status)
        print (r.getheaders())
        print (self.cookieJar)

if __name__=="__main__":
    couch_db_http_api_base = CouchDbHttpApiBase()
    couch_db_http_api_base.getMotd()
<commit_msg>implemented getAllDbs<commit_after>import http.cookiejar, urllib.request
class CouchDbHttpApiBase():
    def __init__(self):
        self.cookieJar = http.cookiejar.LWPCookieJar()
        self.cookieProcessor = urllib.request.HTTPCookieProcessor(self.cookieJar)
        self.openerDirector = urllib.request.build_opener(self.cookieProcessor)
        print(self.openerDirector)
        self.cookieJar.load("cookie.txt")

    def saveCookie(self):
        self.cookieJar.save("cookie.txt")

    def getMotd(self):
        r = self.openerDirector.open("http://127.0.0.1:5984/")
        print (r.status)
        print (r.read())
        print (r.getheaders())
        print (self.cookieJar)

    def getAllDbs(self):
        r = self.openerDirector.open("http://127.0.0.1:5984/_all_dbs")
        print (r.status)
        print (r.read())
        print (r.getheaders())
        print (self.cookieJar)

if __name__=="__main__":
    couch_db_http_api_base = CouchDbHttpApiBase()
    couch_db_http_api_base.getMotd()
    couch_db_http_api_base.getAllDbs()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#-----------------------
# Name: cache.py
# Python Library
# Author: Raymond Wagner
# Purpose: Persistant file-backed cache using /tmp/ to share data and flock
#          to allow safe concurrent access.
#-----------------------

import struct
import fcntl
import errno
import json
import time
import os

from tmdb_exceptions import *

def _donothing(*args, **kwargs):
    pass

class CacheEngineType( type ):
    """
    Cache Engine Metaclass that registers new engines against the cache
    for named selection and use.
    """
    def __init__(mcs, name, bases, attrs):
        super(CacheEngineType, mcs).__init__(name, bases, attrs)
        if name != 'CacheEngine':
            # skip base class
            Cache.register(mcs)

class CacheEngine( object ):
    __metaclass__ = CacheEngineType

    def __init__(self, parent):
        self.parent = parent
    def configure(self):
        raise RuntimeError
    def get(self, key):
        raise RuntimeError
    def put(self, key, value, lifetime):
        raise RuntimeError
    def expire(self, key):
        raise RuntimeError

class Cache( object ):
    """
    This class implements a persistent cache, backed in a file specified in
    the object creation. The file is protected for safe, concurrent access
    by multiple instances using flock.
    This cache uses JSON for speed and storage efficiency, so only simple
    data types are supported.
    Data is stored in a simple format {key:(expiretimestamp, data)}
    """
    __engines = {}

    @classmethod
    def register(cls, engine):
        cls.__engines[engine.__name__] = engine
        cls.__engines[engine.name] = engine

    def __init__(self, engine=None, *args, **kwargs):
        self._engine = None
        self.configure(engine, *args, **kwargs)

    def configure(self, engine, *args, **kwargs):
        if engine is None:
            engine = 'file'
        elif engine not in self.__engines:
            raise TMDBCacheError("Invalid cache engine specified: "+engine)
        self._engine = self.__engines[engine](self)
        self._engine.configure(*args, **kwargs)

    def put(self, key, data, lifetime=3600):
        # pull existing data, so cache will be fresh when written back out
        if self._engine is None:
            raise TMDBCacheError("No cache engine configured")
        self._engine.put(key, data, lifetime)

    def get(self, key):
        if self._engine is None:
            raise TMDBCacheError("No cache engine configured")
        return self._engine.get(key)

        self._read()
        if (key in self._cache) and (time.time() < self._cache[key][0]):
            return self._cache[key][1]
        return None

    def cached(self, callback):
        """
        Returns a decorator that uses a callback to specify the key to use
        for caching the responses from the decorated function.
        """
        return self.Cached(self, callback)

    class Cached( object ):
        def __init__(self, cache, callback, func=None, inst=None):
            self.cache = cache
            self.callback = callback
            self.func = func
            self.inst = inst

            if func:
                self.__module__ = func.__module__
                self.__name__ = func.__name__
                self.__doc__ = func.__doc__

        def __call__(self, *args, **kwargs):
            if self.func is None: # decorator is waiting to be given a function
                if len(kwargs) or (len(args) != 1):
                    raise TMDBCacheError('Cache.Cached decorator must be called '+\
                                         'a single callable argument before it '+\
                                         'be used.')
                elif args[0] is None:
                    raise TMDBCacheError('Cache.Cached decorator called before '+\
                                         'being given a function to wrap.')
                elif not callable(args[0]):
                    raise TMDBCacheError('Cache.Cached must be provided a '+\
                                         'callable object.')
                return self.__class__(self.cache, self.callback, args[0])
            else:
                key = self.callback()
                data = self.cache.get(key)
                if data is None:
                    data = self.func(*args, **kwargs)
                    if hasattr(self.inst, 'lifetime'):
                        self.cache.put(key, data, self.inst.lifetime)
                    else:
                        self.cache.put(key, data)
                return data

        def __get__(self, inst, owner):
            if inst is None:
                return self
            func = self.func.__get__(inst, owner)
            callback = self.callback.__get__(inst, owner)
            return self.__class__(self.cache, callback, func, inst)

class NullEngine( CacheEngine ):
    """Non-caching engine for debugging."""
    name = 'null'
    def configure(self): pass
    def get(self, key): return None
    def put(self, key, value, lifetime): pass
    def expire(self, key): pass

class Flock( object ):
    """
    Context manager to flock file for the duration the object exists.
    Referenced file will be automatically unflocked as the interpreter
    exits the context.
    Supports an optional callback to process the error and optionally
    suppress it.
    """
    def __init__(self, fileobj, operation, callback=None):
        self.fileobj = fileobj
        self.operation = operation
        self.callback = callback
    def __enter__(self):
        fcntl.flock(self.fileobj, self.operation)
    def __exit__(self, exc_type, exc_value, exc_tb):
        suppress = False
        if callable(self.callback):
            suppress = self.callback(exc_type, exc_value, exc_tb)
        fcntl.flock(self.fileobj, fcntl.LOCK_UN)
        return suppress

class FileEngine( CacheEngine ):
    """Simple file-backed engine."""
    name = 'file'
    __time_struct = struct.Struct('d') # double placed at start of file to
                                       # check for updated data

    def __init__(self, parent):
        super(FileEngine, self).__init__(parent)
        self.configure(None)

    def configure(self, filename):
        self.cachefile = filename
        self.cacheage = 0
        self.cache = None

    def _init_cache(self):
        # only run this once
        self._init_cache = _donothing

        if self.cachefile is None:
            raise TMDBCacheError("No cache filename given.")
        if self.cachefile.startswith('~'):
            self.cachefile = os.path.expanduser(self.cachefile)
        elif not self.cachefile.startswith('/'):
            self.cachefile = '/tmp/' + self.cachefile

        try:
            # attempt to read existing cache at filename
            # handle any errors that occur
            self._read()
            # seems to have read fine, make sure we have write access
            if not os.access(self.cachefile, os.W_OK):
                raise TMDBCacheWriteError(self.cachefile)

        except IOError as e:
            if e.errno == errno.ENOENT:
                # file does not exist, create a new one
                self.cache = {}
                try:
                    self._write()
                except IOError as e:
                    if e.errno == errno.ENOENT:
                        # directory does not exist
                        raise TMDBCacheDirectoryError(self.cachefile)
                    elif e.errno == errno.EACCES:
                        # user does not have rights to create new file
                        raise TMDBCacheWriteError(self.cachefile)
                    else:
                        # let the unhandled error continue through
                        raise
            elif e.errno == errno.EACCESS:
                # file exists, but we do not have permission to access it
                raise TMDBCacheReadError(self.cachefile)
            else:
                # let the unhandled error continue through
                raise

    def _open(self, mode='r'):
        # enforce binary operation
        mode += 'b'
        try:
            if self.cachefd.mode == mode:
                # already opened in requested mode, nothing to do
                self.cachefd.seek(0)
                return
        except: pass # catch issue of no cachefile yet opened
        self.cachefd = open(self.cachefile, mode)

    def _read(self):
        self._init_cache()
        self._open('r')

        with Flock(self.cachefd, fcntl.LOCK_SH): # lock for shared access
            try:
                age = self.__time_struct.unpack(self.cachefd.read(8))
            except:
                # failed to read age, ignore and we'll clean up when
                # it gets rewritten
                if self.cache is None:
                    self.cache = {}
                return

            if self.cacheage >= age:
                # local copy is sufficiently new, no need to read
                return
            # read remaining data from file
            self.cacheage = age
            self.cache = json.load(self.cachefd)

    def _write(self):
        self._init_cache()
        # WARNING: this does no testing to ensure this instance has the newest
        # copy of the file cache
        self._open('w')
        # the slight delay between truncating the file with 'w' and flocking
        # could cause problems with another instance simultaneously trying to
        # read the timestamp
        with Flock(self.cachefd, fcntl.LOCK_EX): # lock for exclusive access
            # filter expired data from cache
            # running this while flocked means the file is locked for additional
            # time, however we do not want anyone else writing to the file
            # before we write our stuff
            self.expire()
            self.cacheage = time.time()
            self.cachefd.write(self.__time_struct.pack(self.cacheage))
            json.dump(self.cache, self.cachefd)

    def get(self, key):
        self._read()
        if (key in self.cache) and (time.time() < self.cache[key][0]):
            return self.cache[key][1]
        return None

    def put(self, key, value, lifetime):
        self._read()
        self.cache[key] = (time.time()+lifetime, value)
        self._write()

    def expire(self, key=None):
        t = time.time()
        for k,v in self.cache.items():
            if v[0] < t: # expiration has passed
                del self._cache[k]

<commit_msg>Correct missed variable name change.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#-----------------------
# Name: cache.py
# Python Library
# Author: Raymond Wagner
# Purpose: Persistant file-backed cache using /tmp/ to share data and flock
#          to allow safe concurrent access.
#-----------------------

import struct
import fcntl
import errno
import json
import time
import os

from tmdb_exceptions import *

def _donothing(*args, **kwargs):
    pass

class CacheEngineType( type ):
    """
    Cache Engine Metaclass that registers new engines against the cache
    for named selection and use.
    """
    def __init__(mcs, name, bases, attrs):
        super(CacheEngineType, mcs).__init__(name, bases, attrs)
        if name != 'CacheEngine':
            # skip base class
            Cache.register(mcs)

class CacheEngine( object ):
    __metaclass__ = CacheEngineType

    def __init__(self, parent):
        self.parent = parent
    def configure(self):
        raise RuntimeError
    def get(self, key):
        raise RuntimeError
    def put(self, key, value, lifetime):
        raise RuntimeError
    def expire(self, key):
        raise RuntimeError

class Cache( object ):
    """
    This class implements a persistent cache, backed in a file specified in
    the object creation. The file is protected for safe, concurrent access
    by multiple instances using flock.
    This cache uses JSON for speed and storage efficiency, so only simple
    data types are supported.
    Data is stored in a simple format {key:(expiretimestamp, data)}
    """
    __engines = {}

    @classmethod
    def register(cls, engine):
        cls.__engines[engine.__name__] = engine
        cls.__engines[engine.name] = engine

    def __init__(self, engine=None, *args, **kwargs):
        self._engine = None
        self.configure(engine, *args, **kwargs)

    def configure(self, engine, *args, **kwargs):
        if engine is None:
            engine = 'file'
        elif engine not in self.__engines:
            raise TMDBCacheError("Invalid cache engine specified: "+engine)
        self._engine = self.__engines[engine](self)
        self._engine.configure(*args, **kwargs)

    def put(self, key, data, lifetime=3600):
        # pull existing data, so cache will be fresh when written back out
        if self._engine is None:
            raise TMDBCacheError("No cache engine configured")
        self._engine.put(key, data, lifetime)

    def get(self, key):
        if self._engine is None:
            raise TMDBCacheError("No cache engine configured")
        return self._engine.get(key)

        self._read()
        if (key in self._cache) and (time.time() < self._cache[key][0]):
            return self._cache[key][1]
        return None

    def cached(self, callback):
        """
        Returns a decorator that uses a callback to specify the key to use
        for caching the responses from the decorated function.
        """
        return self.Cached(self, callback)

    class Cached( object ):
        def __init__(self, cache, callback, func=None, inst=None):
            self.cache = cache
            self.callback = callback
            self.func = func
            self.inst = inst

            if func:
                self.__module__ = func.__module__
                self.__name__ = func.__name__
                self.__doc__ = func.__doc__

        def __call__(self, *args, **kwargs):
            if self.func is None: # decorator is waiting to be given a function
                if len(kwargs) or (len(args) != 1):
                    raise TMDBCacheError('Cache.Cached decorator must be called '+\
                                         'a single callable argument before it '+\
                                         'be used.')
                elif args[0] is None:
                    raise TMDBCacheError('Cache.Cached decorator called before '+\
                                         'being given a function to wrap.')
                elif not callable(args[0]):
                    raise TMDBCacheError('Cache.Cached must be provided a '+\
                                         'callable object.')
                return self.__class__(self.cache, self.callback, args[0])
            else:
                key = self.callback()
                data = self.cache.get(key)
                if data is None:
                    data = self.func(*args, **kwargs)
                    if hasattr(self.inst, 'lifetime'):
                        self.cache.put(key, data, self.inst.lifetime)
                    else:
                        self.cache.put(key, data)
                return data

        def __get__(self, inst, owner):
            if inst is None:
                return self
            func = self.func.__get__(inst, owner)
            callback = self.callback.__get__(inst, owner)
            return self.__class__(self.cache, callback, func, inst)

class NullEngine( CacheEngine ):
    """Non-caching engine for debugging."""
    name = 'null'
    def configure(self): pass
    def get(self, key): return None
    def put(self, key, value, lifetime): pass
    def expire(self, key): pass

class Flock( object ):
    """
    Context manager to flock file for the duration the object exists.
    Referenced file will be automatically unflocked as the interpreter
    exits the context.
    Supports an optional callback to process the error and optionally
    suppress it.
    """
    def __init__(self, fileobj, operation, callback=None):
        self.fileobj = fileobj
        self.operation = operation
        self.callback = callback
    def __enter__(self):
        fcntl.flock(self.fileobj, self.operation)
    def __exit__(self, exc_type, exc_value, exc_tb):
        suppress = False
        if callable(self.callback):
            suppress = self.callback(exc_type, exc_value, exc_tb)
        fcntl.flock(self.fileobj, fcntl.LOCK_UN)
        return suppress

class FileEngine( CacheEngine ):
    """Simple file-backed engine."""
    name = 'file'
    __time_struct = struct.Struct('d') # double placed at start of file to
                                       # check for updated data

    def __init__(self, parent):
        super(FileEngine, self).__init__(parent)
        self.configure(None)

    def configure(self, filename):
        self.cachefile = filename
        self.cacheage = 0
        self.cache = None

    def _init_cache(self):
        # only run this once
        self._init_cache = _donothing

        if self.cachefile is None:
            raise TMDBCacheError("No cache filename given.")
        if self.cachefile.startswith('~'):
            self.cachefile = os.path.expanduser(self.cachefile)
        elif not self.cachefile.startswith('/'):
            self.cachefile = '/tmp/' + self.cachefile

        try:
            # attempt to read existing cache at filename
            # handle any errors that occur
            self._read()
            # seems to have read fine, make sure we have write access
            if not os.access(self.cachefile, os.W_OK):
                raise TMDBCacheWriteError(self.cachefile)

        except IOError as e:
            if e.errno == errno.ENOENT:
                # file does not exist, create a new one
                self.cache = {}
                try:
                    self._write()
                except IOError as e:
                    if e.errno == errno.ENOENT:
                        # directory does not exist
                        raise TMDBCacheDirectoryError(self.cachefile)
                    elif e.errno == errno.EACCES:
                        # user does not have rights to create new file
                        raise TMDBCacheWriteError(self.cachefile)
                    else:
                        # let the unhandled error continue through
                        raise
            elif e.errno == errno.EACCESS:
                # file exists, but we do not have permission to access it
                raise TMDBCacheReadError(self.cachefile)
            else:
                # let the unhandled error continue through
                raise

    def _open(self, mode='r'):
        # enforce binary operation
        mode += 'b'
        try:
            if self.cachefd.mode == mode:
                # already opened in requested mode, nothing to do
                self.cachefd.seek(0)
                return
        except: pass # catch issue of no cachefile yet opened
        self.cachefd = open(self.cachefile, mode)

    def _read(self):
        self._init_cache()
        self._open('r')

        with Flock(self.cachefd, fcntl.LOCK_SH): # lock for shared access
            try:
                age = self.__time_struct.unpack(self.cachefd.read(8))
            except:
                # failed to read age, ignore and we'll clean up when
                # it gets rewritten
                if self.cache is None:
                    self.cache = {}
                return

            if self.cacheage >= age:
                # local copy is sufficiently new, no need to read
                return
            # read remaining data from file
            self.cacheage = age
            self.cache = json.load(self.cachefd)

    def _write(self):
        self._init_cache()
        # WARNING: this does no testing to ensure this instance has the newest
        # copy of the file cache
        self._open('w')
        # the slight delay between truncating the file with 'w' and flocking
        # could cause problems with another instance simultaneously trying to
        # read the timestamp
        with Flock(self.cachefd, fcntl.LOCK_EX): # lock for exclusive access
            # filter expired data from cache
            # running this while flocked means the file is locked for additional
            # time, however we do not want anyone else writing to the file
            # before we write our stuff
            self.expire()
            self.cacheage = time.time()
            self.cachefd.write(self.__time_struct.pack(self.cacheage))
            json.dump(self.cache, self.cachefd)

    def get(self, key):
        self._read()
        if (key in self.cache) and (time.time() < self.cache[key][0]):
            return self.cache[key][1]
        return None

    def put(self, key, value, lifetime):
        self._read()
        self.cache[key] = (time.time()+lifetime, value)
        self._write()

    def expire(self, key=None):
        t = time.time()
        for k,v in self.cache.items():
            if v[0] < t: # expiration has passed
                del self.cache[k]

<|endoftext|>


<commit_before># Copyright 2017-2019 TensorHub, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from __future__ import division

import logging
import os
import re
import shlex
import subprocess
import sys

import click
import six

import guild.log
import guild.opref
import guild.run

from guild import exit_code
from guild import op_util
from guild import run_check
from guild import util

log = None # intialized in _init_logging

class Step(object):

    used_params = (
        "disable_plugins",
        "flags",
        "gpus",
        "label",
        "needed",
        "no_gpus",
        "opspec",
        "stop_after",
    )

    def __init__(self, data, parent_flags, parent_opref):
        if isinstance(data, six.string_types):
            data = {
                "run": data
            }
        if not isinstance(data, dict):
            _error("invalid step data: %r" % data)
        params = self._parse_run(data)
        assert params["opspec"], params
        opspec_param = params["opspec"]
        self.op_spec = self._apply_default_model(opspec_param, parent_opref)
        self.name = data.get("name") or opspec_param
        self.flags = self._init_flags(params, parent_flags)
        self.checks = self._init_checks(data)
        self.label = self._resolve_param(params, "label", parent_flags)
        self.disable_plugins = self._resolve_param(
            params, "disable_plugins", parent_flags)
        self.gpus = self._resolve_param(params, "gpus", parent_flags)
        self.no_gpus = params["no_gpus"]
        self.stop_after = params["stop_after"]
        self.needed = params["needed"]

    def _parse_run(self, data):
        from guild.commands.run import run as run_cmd
        run_spec = data.get("run", "").strip()
        if not run_spec:
            _error("invalid step %r: must define run" % data)
        args = shlex.split(run_spec)
        try:
            ctx = run_cmd.make_context("run", args)
        except click.exceptions.ClickException as e:
            _error("invalid run spec %r: %s" % (run_spec, e))
        else:
            self._warn_ignored_params(ctx, run_spec)
            return ctx.params

    def _warn_ignored_params(self, ctx, run_spec):
        "Warn if any params set that we ignore."""
        defaults = {p.name: p.default for p in ctx.command.params}
        for name, val in ctx.params.items():
            if name not in self.used_params and val != defaults[name]:
                log.warning(
                    "run parameter %s used in %r ignored",
                    name, run_spec)

    @staticmethod
    def _apply_default_model(step_opspec, parent_opref):
        step_opref = guild.opref.OpRef.from_string(step_opspec)
        if not step_opref.model_name:
            step_opref = guild.opref.OpRef(
                step_opref.pkg_type,
                step_opref.pkg_name,
                step_opref.pkg_version,
                parent_opref.model_name,
                step_opref.op_name)
        return step_opref.to_opspec()

    def _init_flags(self, params, parent_flags):
        try:
            parsed = op_util.parse_flags(params["flags"])
        except op_util.ArgValueError as e:
            _error("invalid argument '%s' - expected NAME=VAL" % e.arg)
        else:
            resolved = self._resolve_flag_vals(parsed, parent_flags)
            return self._remove_undefined_flags(resolved)

    @staticmethod
    def _resolve_flag_vals(flags, parent_flags):
        return {
            name: util.resolve_refs(val, parent_flags)
            for name, val in flags.items()
        }

    @staticmethod
    def _remove_undefined_flags(flag_vals):
        return {
            name: val for name, val in flag_vals.items()
            if val is not None
        }

    @staticmethod
    def _resolve_param(params, name, flags):
        resolved = util.resolve_refs(params[name], flags)
        if resolved is None:
            return resolved
        return str(resolved)

    @staticmethod
    def _init_checks(data):
        expect = data.get("expect") or []
        if not isinstance(expect, list):
            log.warning("invalid expect data: %r", data)
            return []
        checks = []
        for check_data in expect:
            try:
                check = run_check.init_check(check_data)
            except ValueError as e:
                log.warning("invalid check %r: %e", data, e)
            else:
                checks.append(check)
        return checks

    def __str__(self):
        return self.name or self.op_spec

def main():
    _init_logging()
    _run_steps()

def _init_logging():
    level = int(os.getenv("LOG_LEVEL", logging.WARN))
    format = os.getenv("LOG_FORMAT", "%(levelname)s: [%(name)s] %(message)s")
    guild.log.init_logging(level, {"_": format})
    globals()["log"] = logging.getLogger("guild")

def _run_steps():
    run = _init_run()
    steps = _init_steps(run)
    if not steps:
        log.warning("no steps defined for run %s", run.id)
        return
    for step in steps:
        step_run = _run_step(step, run)
        _maybe_check_step_run(step, step_run)

def _init_run():
    run_id, run_dir = _run_environ()
    return guild.run.Run(run_id, run_dir)

def _run_environ():
    try:
        return os.environ["RUN_ID"], os.environ["RUN_DIR"]
    except KeyError as e:
        _internal_error("missing required env %s" % e.args[0])

def _init_steps(run):
    data = run.get("steps")
    if not data:
        return []
    if not isinstance(data, list):
        _error("invalid steps data %r: expected list" % data)
    opref = guild.opref.OpRef.from_run(run)
    flags = run.get("flags")
    return [Step(step_data, flags, opref) for step_data in data]

def _run_step(step, parent_run):
    step_run = _init_step_run(parent_run)
    cmd = _init_step_cmd(step, step_run.path)
    _link_to_step_run(step, step_run.path, parent_run.path)
    env = dict(os.environ)
    env["NO_WARN_RUNDIR"] = "1"
    log.info("running %s: %s", step, _format_step_cmd(cmd))
    log.debug("cmd for %s: %s", step, cmd)
    returncode = subprocess.call(cmd, env=env, cwd=os.getenv("CMD_DIR"))
    if returncode != 0:
        sys.exit(returncode)
    return step_run

def _init_step_run(parent_run):
    """Returns the run dir for a step run.

    Directory is based on a new, unique run ID but is not created.
    """
    runs_dir = os.path.dirname(parent_run.path)
    step_run_id = guild.run.mkid()
    step_run_dir = os.path.join(runs_dir, step_run_id)
    return guild.run.Run(step_run_id, step_run_dir)

def _init_step_cmd(step, step_run_dir):
    base_args = [
        sys.executable, "-um", "guild.main_bootstrap",
        "run", "-y",
        "--run-dir", step_run_dir,
        step.op_spec]
    step_options = _step_options(step)
    flag_args = _step_flag_args(step)
    return base_args + step_options + flag_args

def _step_options(step):
    opts = []
    if step.label:
        opts.extend(["--label", step.label])
    if step.disable_plugins:
        opts.extend(["--disable-plugins", step.disable_plugins])
    if step.gpus is not None:
        opts.extend(["--gpus", step.gpus])
    elif step.no_gpus:
        opts.append("--no-gpus")
    if step.stop_after:
        opts.extend(["--stop-after", step.stop_after])
    if step.needed:
        opts.append("--needed")
    return opts

def _step_flag_args(step):
    if not step.flags:
        return []
    return [
        "%s=%s" % (name, val)
        for name, val in sorted(step.flags.items())
    ]

def _link_to_step_run(step, step_run_dir, parent_run_dir):
    link_name = _step_link_name(step)
    link_path_base = os.path.join(parent_run_dir, link_name)
    link_path = _ensure_unique_link(link_path_base)
    os.symlink(step_run_dir, link_path)

def _step_link_name(step):
    return re.sub(r"[ :/\\]", "_", str(step))

def _ensure_unique_link(path_base):
    v = 2
    path = path_base
    while True:
        assert v < 1e6
        if not os.path.exists(path):
            return path
        path = "%s_%i" % (path_base, v)
        v += 1

def _format_step_cmd(cmd):
    from six.moves import shlex_quote
    # Just show opspec on - assert front matter to catch changes to cmd.
    assert cmd[0:6] == [
        sys.executable, "-um", "guild.main_bootstrap",
        "run", "-y", "--run-dir"
    ], cmd
    return " ".join([shlex_quote(arg) for arg in cmd[7:]])

def _maybe_check_step_run(step, run):
    if not step.checks:
        return
    if _run_skipped(run):
        log.info("skipping checks for %s", step.name)
        return
    checks_passed = _check_step_run(step, run)
    if not checks_passed:
        _error(
            "stopping because a check failed",
            exit_code.TEST_FAILED)

def _run_skipped(run):
    """Returns True if run was skipped.

    We infer that a run was skipped if it's directory doesn't
    exist. The rationale relies on the assertion that the step run
    creates the specified run directory only when the run is not
    skipped.
    """
    return not os.path.exists(run.path)

def _check_step_run(step, run):
    if not step.checks:
        return True
    passed = 0
    failed = 0
    for check in step.checks:
        try:
            check.check_run(run)
        except run_check.Failed as e:
            log.error("check failed: %s", e)
            failed += 1
        else:
            passed += 1
    log.info("%i of %i checks passed", passed, passed + failed)
    if failed > 0:
        log.error("%i check(s) failed - see above for details", failed)
    return failed == 0

def _internal_error(msg):
    sys.stderr.write("guild.steps_main: %s\n" % msg)
    sys.exit(exit_code.INTERNAL_ERROR)

def _error(msg, exit_code=exit_code.DEFAULT):
    sys.stderr.write("guild: %s\n" % msg)
    sys.exit(exit_code)

if __name__ == "__main__":
    main()
<commit_msg>Deterministic ordering<commit_after># Copyright 2017-2019 TensorHub, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from __future__ import division

import logging
import os
import re
import shlex
import subprocess
import sys

import click
import six

import guild.log
import guild.opref
import guild.run

from guild import exit_code
from guild import op_util
from guild import run_check
from guild import util

log = None # intialized in _init_logging

class Step(object):

    used_params = (
        "disable_plugins",
        "flags",
        "gpus",
        "label",
        "needed",
        "no_gpus",
        "opspec",
        "stop_after",
    )

    def __init__(self, data, parent_flags, parent_opref):
        if isinstance(data, six.string_types):
            data = {
                "run": data
            }
        if not isinstance(data, dict):
            _error("invalid step data: %r" % data)
        params = self._parse_run(data)
        assert params["opspec"], params
        opspec_param = params["opspec"]
        self.op_spec = self._apply_default_model(opspec_param, parent_opref)
        self.name = data.get("name") or opspec_param
        self.flags = self._init_flags(params, parent_flags)
        self.checks = self._init_checks(data)
        self.label = self._resolve_param(params, "label", parent_flags)
        self.disable_plugins = self._resolve_param(
            params, "disable_plugins", parent_flags)
        self.gpus = self._resolve_param(params, "gpus", parent_flags)
        self.no_gpus = params["no_gpus"]
        self.stop_after = params["stop_after"]
        self.needed = params["needed"]

    def _parse_run(self, data):
        from guild.commands.run import run as run_cmd
        run_spec = data.get("run", "").strip()
        if not run_spec:
            _error("invalid step %r: must define run" % data)
        args = shlex.split(run_spec)
        try:
            ctx = run_cmd.make_context("run", args)
        except click.exceptions.ClickException as e:
            _error("invalid run spec %r: %s" % (run_spec, e))
        else:
            self._warn_ignored_params(ctx, run_spec)
            return ctx.params

    def _warn_ignored_params(self, ctx, run_spec):
        "Warn if any params set that we ignore."""
        defaults = {p.name: p.default for p in ctx.command.params}
        for name, val in sorted(ctx.params.items()):
            if name not in self.used_params and val != defaults[name]:
                log.warning(
                    "run parameter %s used in %r ignored",
                    name, run_spec)

    @staticmethod
    def _apply_default_model(step_opspec, parent_opref):
        step_opref = guild.opref.OpRef.from_string(step_opspec)
        if not step_opref.model_name:
            step_opref = guild.opref.OpRef(
                step_opref.pkg_type,
                step_opref.pkg_name,
                step_opref.pkg_version,
                parent_opref.model_name,
                step_opref.op_name)
        return step_opref.to_opspec()

    def _init_flags(self, params, parent_flags):
        try:
            parsed = op_util.parse_flags(params["flags"])
        except op_util.ArgValueError as e:
            _error("invalid argument '%s' - expected NAME=VAL" % e.arg)
        else:
            resolved = self._resolve_flag_vals(parsed, parent_flags)
            return self._remove_undefined_flags(resolved)

    @staticmethod
    def _resolve_flag_vals(flags, parent_flags):
        return {
            name: util.resolve_refs(val, parent_flags)
            for name, val in flags.items()
        }

    @staticmethod
    def _remove_undefined_flags(flag_vals):
        return {
            name: val for name, val in flag_vals.items()
            if val is not None
        }

    @staticmethod
    def _resolve_param(params, name, flags):
        resolved = util.resolve_refs(params[name], flags)
        if resolved is None:
            return resolved
        return str(resolved)

    @staticmethod
    def _init_checks(data):
        expect = data.get("expect") or []
        if not isinstance(expect, list):
            log.warning("invalid expect data: %r", data)
            return []
        checks = []
        for check_data in expect:
            try:
                check = run_check.init_check(check_data)
            except ValueError as e:
                log.warning("invalid check %r: %e", data, e)
            else:
                checks.append(check)
        return checks

    def __str__(self):
        return self.name or self.op_spec

def main():
    _init_logging()
    _run_steps()

def _init_logging():
    level = int(os.getenv("LOG_LEVEL", logging.WARN))
    format = os.getenv("LOG_FORMAT", "%(levelname)s: [%(name)s] %(message)s")
    guild.log.init_logging(level, {"_": format})
    globals()["log"] = logging.getLogger("guild")

def _run_steps():
    run = _init_run()
    steps = _init_steps(run)
    if not steps:
        log.warning("no steps defined for run %s", run.id)
        return
    for step in steps:
        step_run = _run_step(step, run)
        _maybe_check_step_run(step, step_run)

def _init_run():
    run_id, run_dir = _run_environ()
    return guild.run.Run(run_id, run_dir)

def _run_environ():
    try:
        return os.environ["RUN_ID"], os.environ["RUN_DIR"]
    except KeyError as e:
        _internal_error("missing required env %s" % e.args[0])

def _init_steps(run):
    data = run.get("steps")
    if not data:
        return []
    if not isinstance(data, list):
        _error("invalid steps data %r: expected list" % data)
    opref = guild.opref.OpRef.from_run(run)
    flags = run.get("flags")
    return [Step(step_data, flags, opref) for step_data in data]

def _run_step(step, parent_run):
    step_run = _init_step_run(parent_run)
    cmd = _init_step_cmd(step, step_run.path)
    _link_to_step_run(step, step_run.path, parent_run.path)
    env = dict(os.environ)
    env["NO_WARN_RUNDIR"] = "1"
    log.info("running %s: %s", step, _format_step_cmd(cmd))
    log.debug("cmd for %s: %s", step, cmd)
    returncode = subprocess.call(cmd, env=env, cwd=os.getenv("CMD_DIR"))
    if returncode != 0:
        sys.exit(returncode)
    return step_run

def _init_step_run(parent_run):
    """Returns the run dir for a step run.

    Directory is based on a new, unique run ID but is not created.
    """
    runs_dir = os.path.dirname(parent_run.path)
    step_run_id = guild.run.mkid()
    step_run_dir = os.path.join(runs_dir, step_run_id)
    return guild.run.Run(step_run_id, step_run_dir)

def _init_step_cmd(step, step_run_dir):
    base_args = [
        sys.executable, "-um", "guild.main_bootstrap",
        "run", "-y",
        "--run-dir", step_run_dir,
        step.op_spec]
    step_options = _step_options(step)
    flag_args = _step_flag_args(step)
    return base_args + step_options + flag_args

def _step_options(step):
    opts = []
    if step.label:
        opts.extend(["--label", step.label])
    if step.disable_plugins:
        opts.extend(["--disable-plugins", step.disable_plugins])
    if step.gpus is not None:
        opts.extend(["--gpus", step.gpus])
    elif step.no_gpus:
        opts.append("--no-gpus")
    if step.stop_after:
        opts.extend(["--stop-after", step.stop_after])
    if step.needed:
        opts.append("--needed")
    return opts

def _step_flag_args(step):
    if not step.flags:
        return []
    return [
        "%s=%s" % (name, val)
        for name, val in sorted(step.flags.items())
    ]

def _link_to_step_run(step, step_run_dir, parent_run_dir):
    link_name = _step_link_name(step)
    link_path_base = os.path.join(parent_run_dir, link_name)
    link_path = _ensure_unique_link(link_path_base)
    os.symlink(step_run_dir, link_path)

def _step_link_name(step):
    return re.sub(r"[ :/\\]", "_", str(step))

def _ensure_unique_link(path_base):
    v = 2
    path = path_base
    while True:
        assert v < 1e6
        if not os.path.exists(path):
            return path
        path = "%s_%i" % (path_base, v)
        v += 1

def _format_step_cmd(cmd):
    from six.moves import shlex_quote
    # Just show opspec on - assert front matter to catch changes to cmd.
    assert cmd[0:6] == [
        sys.executable, "-um", "guild.main_bootstrap",
        "run", "-y", "--run-dir"
    ], cmd
    return " ".join([shlex_quote(arg) for arg in cmd[7:]])

def _maybe_check_step_run(step, run):
    if not step.checks:
        return
    if _run_skipped(run):
        log.info("skipping checks for %s", step.name)
        return
    checks_passed = _check_step_run(step, run)
    if not checks_passed:
        _error(
            "stopping because a check failed",
            exit_code.TEST_FAILED)

def _run_skipped(run):
    """Returns True if run was skipped.

    We infer that a run was skipped if it's directory doesn't
    exist. The rationale relies on the assertion that the step run
    creates the specified run directory only when the run is not
    skipped.
    """
    return not os.path.exists(run.path)

def _check_step_run(step, run):
    if not step.checks:
        return True
    passed = 0
    failed = 0
    for check in step.checks:
        try:
            check.check_run(run)
        except run_check.Failed as e:
            log.error("check failed: %s", e)
            failed += 1
        else:
            passed += 1
    log.info("%i of %i checks passed", passed, passed + failed)
    if failed > 0:
        log.error("%i check(s) failed - see above for details", failed)
    return failed == 0

def _internal_error(msg):
    sys.stderr.write("guild.steps_main: %s\n" % msg)
    sys.exit(exit_code.INTERNAL_ERROR)

def _error(msg, exit_code=exit_code.DEFAULT):
    sys.stderr.write("guild: %s\n" % msg)
    sys.exit(exit_code)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># Defination of python list based Deque (Double ended queue) ADT
# A exception class define for handling empty errors

class Empty(Exception):
    '''
    Error attemping to access an element from an empty container.
    '''
    pass

class Deque:
    '''
    A special case of queue which can add and delete elements from both
    ends. Using a python list as underlying storage.'''
    DEFAULT_CAPACITY = 10
    def __init__(self):
        '''Create empty deque.'''
        self._data = [None] * DEFAULT_CAPACITY
        self._size = 0
        self._front = 0
        self._back = 0

    def __len__(self):
        '''Return the number of elements in the deque.'''
        return self._size

    def is_empty(self):
        '''Return True if the deque is empyt.'''
        return self._size == 0

    def first(self):
        '''Return (but do not remove) the element at the front of the queue.
    Raise Empty exception if the deque is empty.'''
        if self.is_empty():
            raise Empty('Deque is empty')
        return self._data[self._front]

    def last(self):
        '''Return (but do not remove) the element at the end of the queue.
    Raise Empty exception if the deque is empty.'''
        if self.is_empty():
            raise Empty('Deque is empty')
        self._back = (self._front + self._size - 1) % len(self._data)
        return self._data[self._back]

    def add_first(self, e):
        '''Add element e to the front of deque.'''
        if self._size == len(self._data):
            self._resize(2 * len(self._data))                   # extend capacity
        self._front = (self._front - 1) % len(self._data)       # cyclic shift
        self._data[self._front] = e
        self._size += 1

    def add_last(self, e):
        '''Add element e to the back of deque.'''
        if self._size == len(self._data):
            self._resize(2* len(self._data))
        self._back = (self._back + self._size) % len(self._data)
        self._data[self._back] = e
        self._size += 1

    def delete_first(self):
        '''Remove and return the first element of the deque,
    Raise Empty exception if the deque is empty.'''
        if self.is_empty():
            raise Empty('Queue is empty')
        answer = self._data[self._front]
        self._data[self._front] = None
        self._front = (self._front + 1) % len(self._data)       # i.e.,DEFAULT_CAPACITY
        self._size -= 1
        if 0 < self._size < len(self._data)//4:
            self._resize(len(self._data)//2)                    # shrink list by 1/2 after
        return answer

    def delete_last(self):
        '''Remove and return the last element of the deque,
    Raise Empty exception if the deque is empty.'''
        if self.is_empty():
            raise Empty('Queue is empty')
        answer = self._data[self._front]
        self._data[self._front] = None
        self._front = (self._front + 1) % len(self._data)       # i.e.,DEFAULT_CAPACITY
        self._size -= 1
        if 0 < self._size < len(self._data)//4:
            self._resize(len(self._data)//2)                    # shrink list by 1/2 after
        return answer

    def _resize(self, cap):
        '''Resize to a new list of capacity >= len(self)'''
        old = self._data
        self._data = [None] * cap
        walk = self._front
        for k in range(self._size):                         # self._size here is old size
            self._data[k] = old[walk]
            walk = (1 + walk) % len(old)
        self._front = 0                                     # self._front realigned

if __name__ == '__main__':

<commit_msg>some bug to fix<commit_after># Defination of python list based Deque (Double ended queue) ADT
# A exception class define for handling empty errors

class Empty(Exception):
    '''
    Error attemping to access an element from an empty container.
    '''
    pass

class Deque:
    '''
    A special case of queue which can add and delete elements from both
    ends. Using a python list as underlying storage.'''
    DEFAULT_CAPACITY = 10
    def __init__(self):
        '''Create empty deque.'''
        self._data = [None] * self.DEFAULT_CAPACITY
        self._size = 0
        self._front = 0
        self._back = 0

    def __len__(self):
        '''Return the number of elements in the deque.'''
        return self._size

    def is_empty(self):
        '''Return True if the deque is empyt.'''
        return self._size == 0

    def first(self):
        '''Return (but do not remove) the element at the front of the queue.
    Raise Empty exception if the deque is empty.'''
        if self.is_empty():
            raise Empty('Deque is empty')
        return self._data[self._front]

    def last(self):
        '''Return (but do not remove) the element at the end of the queue.
    Raise Empty exception if the deque is empty.'''
        if self.is_empty():
            raise Empty('Deque is empty')
        # self._back = (self._front + self._size - 1) % len(self._data)
        return self._data[self._back]

    def add_first(self, e):
        '''Add element e to the front of deque.'''
        if self._size == len(self._data):
            self._resize(2 * len(self._data))                   # extend capacity
        self._front = (self._front - 1) % len(self._data)       # cyclic shift
        self._data[self._front] = e
        self._size += 1

    def add_last(self, e):
        '''Add element e to the back of deque.'''
        if self._size == len(self._data):
            self._resize(2* len(self._data))
        self._back = (self._back + self._size) % len(self._data)
        self._data[self._back] = e
        self._size += 1

    def delete_first(self):
        '''Remove and return the first element of the deque,
    Raise Empty exception if the deque is empty.'''
        if self.is_empty():
            raise Empty('Queue is empty')
        answer = self._data[self._front]
        self._data[self._front] = None
        self._front = (self._front + 1) % len(self._data)       # i.e.,DEFAULT_CAPACITY
        self._size -= 1
        if 0 < self._size < len(self._data)//4:
            self._resize(len(self._data)//2)                    # shrink list by 1/2 after
        return answer

    def delete_last(self):
        '''Remove and return the last element of the deque,
    Raise Empty exception if the deque is empty.'''
        if self.is_empty():
            raise Empty('Queue is empty')
        answer = self._data[self._front]
        self._data[self._front] = None
        self._front = (self._front + 1) % len(self._data)       # i.e.,DEFAULT_CAPACITY
        self._size -= 1
        if 0 < self._size < len(self._data)//4:
            self._resize(len(self._data)//2)                    # shrink list by 1/2 after
        return answer

    def _resize(self, cap):
        '''Resize to a new list of capacity >= len(self)'''
        old = self._data
        self._data = [None] * cap
        walk = self._front
        for k in range(self._size):                             # self._size here is old size
            self._data[k] = old[walk]
            walk = (1 + walk) % len(old)
        self._front = 0                                         # self._front realigned

if __name__ == '__main__':
    def explore(deque):
        print("The deque under test --> ID: %d" % id(deque))
        print("element number --> %d" % len(deque))
        print("first element --> %d" % deque.first())
        print("last element --> %d" % deque.last())
        print("underlying data --> %s" % str(deque._data))
        print("current front index --> %d" % d._front)
        print("current back index --> %d" % d._back)
        print()

    d = Deque()
    d.add_first(1)
    d.add_last(2)
    explore(d)
    d.add_first(11)
    d.add_last(22)
    explore(d)
    d.add_first(111)
    d.add_last(222)
    explore(d)
    d.add_first(1111)
    d.add_last(2222)
    explore(d)

<|endoftext|>