<commit_before>import os, sys, uuid
import warnings
from io import BytesIO
from tempfile import NamedTemporaryFile

from ...core import HoloMap, AdjointLayout
from ...core.options import Store, StoreOptions

from .. import MIME_TYPES
from ..renderer import Renderer

from matplotlib import pyplot as plt
from matplotlib import animation
import matplotlib.tight_bbox as tight_bbox
from matplotlib.transforms import Bbox, TransformedBbox, Affine2D

from matplotlib.backends.backend_nbagg import CommSocket

import param
from param.parameterized import bothmethod

# <format name> : (animation writer, format,  anim_kwargs, extra_args)
ANIMATION_OPTS = {
    'webm': ('ffmpeg', 'webm', {},
             ['-vcodec', 'libvpx', '-b', '1000k']),
    'mp4': ('ffmpeg', 'mp4', {'codec': 'libx264'},
             ['-pix_fmt', 'yuv420p']),
    'gif': ('imagemagick', 'gif', {'fps': 10}, []),
    'scrubber': ('html', None, {'fps': 5}, None)
}

def opts(el, percent_size):
    "Returns the plot options with supplied size (if not overridden)"
    obj = el.last if isinstance(el, HoloMap) else el
    options = MPLRenderer.get_plot_size(obj, percent_size) #  Store.registry[type(el)].renderer
    options.update(Store.lookup_options(obj, 'plot').options)
    return options

class MPLRenderer(Renderer):
    """
    Exporter used to render data from matplotlib, either to a stream
    or directly to file.

    The __call__ method renders an HoloViews component to raw data of
    a specified matplotlib format.  The save method is the
    corresponding method for saving a HoloViews objects to disk.

    The save_fig and save_anim methods are used to save matplotlib
    figure and animation objects. These match the two primary return
    types of plotting class implemented with matplotlib.
    """
    drawn = {}

    def __call__(self, obj, fmt=None):
        """
        Render the supplied HoloViews component using matplotlib.
        """
        if isinstance(obj, AdjointLayout):
            obj = Layout.from_values(obj)

        element_type = obj.type if isinstance(obj, HoloMap) else type(obj)
        try:
            plotclass = Store.registry[element_type]
        except KeyError:
            raise Exception("No corresponding plot type found for %r" % type(obj))

        plot = plotclass(obj, **opts(obj,  self.size))

        if fmt is None:
            fmt = self.holomap if len(plot) > 1 else self.fig
            if fmt is None: return

        if len(plot) > 1:
            (writer, _, anim_kwargs, extra_args) = ANIMATION_OPTS[fmt]
            anim = plot.anim(fps=self.fps)
            if extra_args != []:
                anim_kwargs = dict(anim_kwargs, extra_args=extra_args)

            data = self._anim_data(anim, fmt, writer, **anim_kwargs)
        else:
            plot.update(0)
            data = self.figure_data(plot, fmt, **({'dpi':self.dpi} if self.dpi else {}))

        return data, {'file-ext':fmt,
                      'mime_type':MIME_TYPES[fmt]}

    @classmethod
    def get_plot_size(cls, obj, percent_size):
        """
        Given a holoviews object and a percentage size, apply heuristics
        to compute a suitable figure size. For instance, scaling layouts
        and grids linearly can result in unwieldy figure sizes when there
        are a large number of elements. As ad hoc heuristics are used,
        this functionality is kept separate from the plotting classes
        themselves.

        Used by the IPython Notebook display hooks and the save
        utility. Note that this can be overridden explicitly per object
        using the fig_size and size plot options.
        """
        from .plot import MPLPlot
        factor = percent_size / 100.0
        plot_type = obj.type if isinstance(obj, HoloMap) else type(obj)
        options = Store.lookup_options(obj, 'plot').options
        fig_inches = options.get('fig_inches', MPLPlot.fig_inches)
        if isinstance(fig_inches, (list, tuple)):
            fig_inches =  (fig_inches[0] * factor,
                           fig_inches[1] * factor)
        else:
            fig_inches = MPLPlot.fig_inches * factor

        return dict(fig_inches=fig_inches)

    @bothmethod
    def supported_holomap_formats(self_or_cls, optional_formats):
        "Optional formats that are actually supported by this renderer"
        supported = []
        with param.logging_level('CRITICAL'):
            self_or_cls.HOLOMAP_FORMAT_ERROR_MESSAGES = {}
        for fmt in optional_formats:
            with warnings.catch_warnings():
                warnings.simplefilter("ignore")
                try:
                    fig = plt.figure()
                    anim = animation.FuncAnimation(fig, lambda x: x, frames=[0,1])
                    (writer, fmt, anim_kwargs, extra_args) = ANIMATION_OPTS[fmt]
                    if extra_args != []:
                        anim_kwargs = dict(anim_kwargs, extra_args=extra_args)
                        renderer = self_or_cls.instance(dpi=72)
                        renderer._anim_data(anim, fmt, writer, **anim_kwargs)
                    plt.close(fig)
                    supported.append(fmt)
                except Exception as e:
                    self_or_cls.HOLOMAP_FORMAT_ERROR_MESSAGES[fmt] = str(e)
        return supported

    @bothmethod
    def save(self_or_cls, obj, basename, fmt=None, key={}, info={}, options=None, **kwargs):
        """
        Save a HoloViews object to file, either using an explicitly
        supplied format or to the appropriate default.
        """
        if info or key:
            raise Exception('MPLRenderer does not support saving metadata to file.')

        with StoreOptions.options(obj, options, **kwargs):
            rendered = self_or_cls(obj, fmt)
        if rendered is None: return
        (data, info) = rendered
        filename ='%s.%s' % (basename, info['file-ext'])
        with open(filename, 'wb') as f:
            f.write(self_or_cls.encode(rendered))

    def _anim_data(self, anim, fmt, writer, **anim_kwargs):
        """
        Render a matplotlib animation object and return the corresponding data.
        """
        anim_kwargs = dict(anim_kwargs, **({'dpi':self.dpi} if self.dpi is not None else {}))
        anim_kwargs = dict({'fps':self.fps} if fmt =='gif' else {}, **anim_kwargs)
        if not hasattr(anim, '_encoded_video'):
            with NamedTemporaryFile(suffix='.%s' % fmt) as f:
                anim.save(f.name, writer=writer,
                          **dict(anim_kwargs, **({'dpi':self.dpi} if self.dpi else {})))
                video = open(f.name, "rb").read()
        return video

    def _compute_bbox(self, fig, kw):
        """
        Compute the tight bounding box for each figure once, reducing
        number of required canvas draw calls from N*2 to N+1 as a
        function of the number of frames.

        Tight bounding box computing code here mirrors:
        matplotlib.backend_bases.FigureCanvasBase.print_figure
        as it hasn't been factored out as a function.
        """
        fig_id = id(fig)
        if kw['bbox_inches'] == 'tight' and kw['format'] == 'png':
            if not fig_id in MPLRenderer.drawn:
                fig.set_dpi(self.dpi)
                fig.canvas.draw()
                renderer = fig._cachedRenderer
                bbox_inches = fig.get_tightbbox(renderer)
                bbox_artists = fig.get_default_bbox_extra_artists()
                bbox_filtered = []
                for a in bbox_artists:
                    bbox = a.get_window_extent(renderer)
                    if a.get_clip_on():
                        clip_box = a.get_clip_box()
                        if clip_box is not None:
                            bbox = Bbox.intersection(bbox, clip_box)
                        clip_path = a.get_clip_path()
                        if clip_path is not None and bbox is not None:
                            clip_path = clip_path.get_fully_transformed_path()
                            bbox = Bbox.intersection(bbox,
                                                     clip_path.get_extents())
                    if bbox is not None and (bbox.width != 0 or
                                             bbox.height != 0):
                        bbox_filtered.append(bbox)
                if bbox_filtered:
                    _bbox = Bbox.union(bbox_filtered)
                    trans = Affine2D().scale(1.0 / self.dpi)
                    bbox_extra = TransformedBbox(_bbox, trans)
                    bbox_inches = Bbox.union([bbox_inches, bbox_extra])
                pad = plt.rcParams['savefig.pad_inches']
                bbox_inches = bbox_inches.padded(pad)
                MPLRenderer.drawn[fig_id] = bbox_inches
                kw['bbox_inches'] = bbox_inches
            else:
                kw['bbox_inches'] = MPLRenderer.drawn[fig_id]
        return kw

    def figure_data(self, plot, fmt='png', bbox_inches='tight', **kwargs):
        """
        Render matplotlib figure object and return the corresponding data.

        Similar to IPython.core.pylabtools.print_figure but without
        any IPython dependency.
        """
        fig = plot.state
        kw = dict(
            format=fmt,
            facecolor=fig.get_facecolor(),
            edgecolor=fig.get_edgecolor(),
            dpi=self.dpi,
            bbox_inches=bbox_inches,
        )
        kw.update(kwargs)

        # Attempts to precompute the tight bounding box
        try:
            kw = self._compute_bbox(fig, kw)
        except:
            pass

        bytes_io = BytesIO()
        fig.canvas.print_figure(bytes_io, **kw)
        data = bytes_io.getvalue()
        if fmt == 'svg':
            data = data.decode('utf-8')
        return data

    @classmethod
    def get_figure_manager(cls, counter, plot):
        try:
            from matplotlib.backends.backend_nbagg import new_figure_manager_given_figure
            from mpl_toolkits.mplot3d import Axes3D
        except:
            return None
        fig = plot.state
        manager = new_figure_manager_given_figure(counter, fig)
        # Need to call mouse_init on each 3D axis to enable rotation support
        for ax in fig.get_axes():
            if isinstance(ax, Axes3D):
                ax.mouse_init()
        return manager

    @bothmethod
    def get_size(self_or_cls, plot):
        w, h = plot.state.get_size_inches()
        dpi = plot.state.dpi
        return (w*dpi, h*dpi)

    @bothmethod
    def animation_data(self_or_cls, plot, holomap_format, fps, dpi):
        if sys.version_info[0] == 3 and mpl.__version__[:-2] in ['1.2', '1.3']:
            raise Exception("Python 3 matplotlib animation support broken <= 1.3")
        #renderer = Store.renderer.instance(dpi=dpi)
        anim = plot.anim(fps=fps)
        (writer, fmt, anim_kwargs, extra_args) = ANIMATION_OPTS[holomap_format]

        if extra_args != []:
            anim_kwargs = dict(anim_kwargs, extra_args=extra_args)
        if holomap_format=='gif':
            anim_kwargs['fps'] = fps
        return self_or_cls._anim_data(anim, fmt, writer, **anim_kwargs)

class WidgetCommSocket(CommSocket):
    """
    CustomCommSocket provides communication between the IPython
    kernel and a matplotlib canvas element in the notebook.
    A CustomCommSocket is required to delay communication
    between the kernel and the canvas element until the widget
    has been rendered in the notebook.
    """

    def __init__(self, manager):
        self.supports_binary = None
        self.manager = manager
        self.uuid = str(uuid.uuid4())
        self.html = "

" % self.uuid

    def start(self):
        from IPython.kernel.comm import Comm
        try:
            self.comm = Comm('matplotlib', data={'id': self.uuid})
        except AttributeError:
            raise RuntimeError('Unable to create an IPython notebook Comm '
                               'instance. Are you in the IPython notebook?')
        self.comm.on_msg(self.on_message)
        self.comm.on_close(lambda close_message: self.manager.clearup_closed())

Store.renderer = MPLRenderer

<commit_msg>Made ANIMATION_OPTS into a class attribute of MPLRenderer<commit_after>import os, sys, uuid
import warnings
from io import BytesIO
from tempfile import NamedTemporaryFile

from ...core import HoloMap, AdjointLayout
from ...core.options import Store, StoreOptions

from .. import MIME_TYPES
from ..renderer import Renderer

from matplotlib import pyplot as plt
from matplotlib import animation
import matplotlib.tight_bbox as tight_bbox
from matplotlib.transforms import Bbox, TransformedBbox, Affine2D

from matplotlib.backends.backend_nbagg import CommSocket

import param
from param.parameterized import bothmethod

def opts(el, percent_size):
    "Returns the plot options with supplied size (if not overridden)"
    obj = el.last if isinstance(el, HoloMap) else el
    options = MPLRenderer.get_plot_size(obj, percent_size) #  Store.registry[type(el)].renderer
    options.update(Store.lookup_options(obj, 'plot').options)
    return options

class MPLRenderer(Renderer):
    """
    Exporter used to render data from matplotlib, either to a stream
    or directly to file.

    The __call__ method renders an HoloViews component to raw data of
    a specified matplotlib format.  The save method is the
    corresponding method for saving a HoloViews objects to disk.

    The save_fig and save_anim methods are used to save matplotlib
    figure and animation objects. These match the two primary return
    types of plotting class implemented with matplotlib.
    """
    drawn = {}

    # <format name> : (animation writer, format,  anim_kwargs, extra_args)
    ANIMATION_OPTS = {
        'webm': ('ffmpeg', 'webm', {},
                 ['-vcodec', 'libvpx', '-b', '1000k']),
        'mp4': ('ffmpeg', 'mp4', {'codec': 'libx264'},
                 ['-pix_fmt', 'yuv420p']),
        'gif': ('imagemagick', 'gif', {'fps': 10}, []),
        'scrubber': ('html', None, {'fps': 5}, None)
    }

    def __call__(self, obj, fmt=None):
        """
        Render the supplied HoloViews component using matplotlib.
        """
        if isinstance(obj, AdjointLayout):
            obj = Layout.from_values(obj)

        element_type = obj.type if isinstance(obj, HoloMap) else type(obj)
        try:
            plotclass = Store.registry[element_type]
        except KeyError:
            raise Exception("No corresponding plot type found for %r" % type(obj))

        plot = plotclass(obj, **opts(obj,  self.size))

        if fmt is None:
            fmt = self.holomap if len(plot) > 1 else self.fig
            if fmt is None: return

        if len(plot) > 1:
            (writer, _, anim_kwargs, extra_args) = self.ANIMATION_OPTS[fmt]
            anim = plot.anim(fps=self.fps)
            if extra_args != []:
                anim_kwargs = dict(anim_kwargs, extra_args=extra_args)

            data = self._anim_data(anim, fmt, writer, **anim_kwargs)
        else:
            plot.update(0)
            data = self.figure_data(plot, fmt, **({'dpi':self.dpi} if self.dpi else {}))

        return data, {'file-ext':fmt,
                      'mime_type':MIME_TYPES[fmt]}

    @classmethod
    def get_plot_size(cls, obj, percent_size):
        """
        Given a holoviews object and a percentage size, apply heuristics
        to compute a suitable figure size. For instance, scaling layouts
        and grids linearly can result in unwieldy figure sizes when there
        are a large number of elements. As ad hoc heuristics are used,
        this functionality is kept separate from the plotting classes
        themselves.

        Used by the IPython Notebook display hooks and the save
        utility. Note that this can be overridden explicitly per object
        using the fig_size and size plot options.
        """
        from .plot import MPLPlot
        factor = percent_size / 100.0
        plot_type = obj.type if isinstance(obj, HoloMap) else type(obj)
        options = Store.lookup_options(obj, 'plot').options
        fig_inches = options.get('fig_inches', MPLPlot.fig_inches)
        if isinstance(fig_inches, (list, tuple)):
            fig_inches =  (fig_inches[0] * factor,
                           fig_inches[1] * factor)
        else:
            fig_inches = MPLPlot.fig_inches * factor

        return dict(fig_inches=fig_inches)

    @bothmethod
    def supported_holomap_formats(self_or_cls, optional_formats):
        "Optional formats that are actually supported by this renderer"
        supported = []
        with param.logging_level('CRITICAL'):
            self_or_cls.HOLOMAP_FORMAT_ERROR_MESSAGES = {}
        for fmt in optional_formats:
            with warnings.catch_warnings():
                warnings.simplefilter("ignore")
                try:
                    fig = plt.figure()
                    anim = animation.FuncAnimation(fig, lambda x: x, frames=[0,1])
                    (writer, fmt, anim_kwargs, extra_args) = self_or_cls.ANIMATION_OPTS[fmt]
                    if extra_args != []:
                        anim_kwargs = dict(anim_kwargs, extra_args=extra_args)
                        renderer = self_or_cls.instance(dpi=72)
                        renderer._anim_data(anim, fmt, writer, **anim_kwargs)
                    plt.close(fig)
                    supported.append(fmt)
                except Exception as e:
                    self_or_cls.HOLOMAP_FORMAT_ERROR_MESSAGES[fmt] = str(e)
        return supported

    @bothmethod
    def save(self_or_cls, obj, basename, fmt=None, key={}, info={}, options=None, **kwargs):
        """
        Save a HoloViews object to file, either using an explicitly
        supplied format or to the appropriate default.
        """
        if info or key:
            raise Exception('MPLRenderer does not support saving metadata to file.')

        with StoreOptions.options(obj, options, **kwargs):
            rendered = self_or_cls(obj, fmt)
        if rendered is None: return
        (data, info) = rendered
        filename ='%s.%s' % (basename, info['file-ext'])
        with open(filename, 'wb') as f:
            f.write(self_or_cls.encode(rendered))

    def _anim_data(self, anim, fmt, writer, **anim_kwargs):
        """
        Render a matplotlib animation object and return the corresponding data.
        """
        anim_kwargs = dict(anim_kwargs, **({'dpi':self.dpi} if self.dpi is not None else {}))
        anim_kwargs = dict({'fps':self.fps} if fmt =='gif' else {}, **anim_kwargs)
        if not hasattr(anim, '_encoded_video'):
            with NamedTemporaryFile(suffix='.%s' % fmt) as f:
                anim.save(f.name, writer=writer,
                          **dict(anim_kwargs, **({'dpi':self.dpi} if self.dpi else {})))
                video = open(f.name, "rb").read()
        return video

    def _compute_bbox(self, fig, kw):
        """
        Compute the tight bounding box for each figure once, reducing
        number of required canvas draw calls from N*2 to N+1 as a
        function of the number of frames.

        Tight bounding box computing code here mirrors:
        matplotlib.backend_bases.FigureCanvasBase.print_figure
        as it hasn't been factored out as a function.
        """
        fig_id = id(fig)
        if kw['bbox_inches'] == 'tight' and kw['format'] == 'png':
            if not fig_id in MPLRenderer.drawn:
                fig.set_dpi(self.dpi)
                fig.canvas.draw()
                renderer = fig._cachedRenderer
                bbox_inches = fig.get_tightbbox(renderer)
                bbox_artists = fig.get_default_bbox_extra_artists()
                bbox_filtered = []
                for a in bbox_artists:
                    bbox = a.get_window_extent(renderer)
                    if a.get_clip_on():
                        clip_box = a.get_clip_box()
                        if clip_box is not None:
                            bbox = Bbox.intersection(bbox, clip_box)
                        clip_path = a.get_clip_path()
                        if clip_path is not None and bbox is not None:
                            clip_path = clip_path.get_fully_transformed_path()
                            bbox = Bbox.intersection(bbox,
                                                     clip_path.get_extents())
                    if bbox is not None and (bbox.width != 0 or
                                             bbox.height != 0):
                        bbox_filtered.append(bbox)
                if bbox_filtered:
                    _bbox = Bbox.union(bbox_filtered)
                    trans = Affine2D().scale(1.0 / self.dpi)
                    bbox_extra = TransformedBbox(_bbox, trans)
                    bbox_inches = Bbox.union([bbox_inches, bbox_extra])
                pad = plt.rcParams['savefig.pad_inches']
                bbox_inches = bbox_inches.padded(pad)
                MPLRenderer.drawn[fig_id] = bbox_inches
                kw['bbox_inches'] = bbox_inches
            else:
                kw['bbox_inches'] = MPLRenderer.drawn[fig_id]
        return kw

    def figure_data(self, plot, fmt='png', bbox_inches='tight', **kwargs):
        """
        Render matplotlib figure object and return the corresponding data.

        Similar to IPython.core.pylabtools.print_figure but without
        any IPython dependency.
        """
        fig = plot.state
        kw = dict(
            format=fmt,
            facecolor=fig.get_facecolor(),
            edgecolor=fig.get_edgecolor(),
            dpi=self.dpi,
            bbox_inches=bbox_inches,
        )
        kw.update(kwargs)

        # Attempts to precompute the tight bounding box
        try:
            kw = self._compute_bbox(fig, kw)
        except:
            pass

        bytes_io = BytesIO()
        fig.canvas.print_figure(bytes_io, **kw)
        data = bytes_io.getvalue()
        if fmt == 'svg':
            data = data.decode('utf-8')
        return data

    @classmethod
    def get_figure_manager(cls, counter, plot):
        try:
            from matplotlib.backends.backend_nbagg import new_figure_manager_given_figure
            from mpl_toolkits.mplot3d import Axes3D
        except:
            return None
        fig = plot.state
        manager = new_figure_manager_given_figure(counter, fig)
        # Need to call mouse_init on each 3D axis to enable rotation support
        for ax in fig.get_axes():
            if isinstance(ax, Axes3D):
                ax.mouse_init()
        return manager

    @bothmethod
    def get_size(self_or_cls, plot):
        w, h = plot.state.get_size_inches()
        dpi = plot.state.dpi
        return (w*dpi, h*dpi)

    @bothmethod
    def animation_data(self_or_cls, plot, holomap_format, fps, dpi):
        if sys.version_info[0] == 3 and mpl.__version__[:-2] in ['1.2', '1.3']:
            raise Exception("Python 3 matplotlib animation support broken <= 1.3")
        #renderer = Store.renderer.instance(dpi=dpi)
        anim = plot.anim(fps=fps)
        (writer, fmt, anim_kwargs, extra_args) = self_or_cls.ANIMATION_OPTS[holomap_format]

        if extra_args != []:
            anim_kwargs = dict(anim_kwargs, extra_args=extra_args)
        if holomap_format=='gif':
            anim_kwargs['fps'] = fps
        return self_or_cls._anim_data(anim, fmt, writer, **anim_kwargs)

class WidgetCommSocket(CommSocket):
    """
    CustomCommSocket provides communication between the IPython
    kernel and a matplotlib canvas element in the notebook.
    A CustomCommSocket is required to delay communication
    between the kernel and the canvas element until the widget
    has been rendered in the notebook.
    """

    def __init__(self, manager):
        self.supports_binary = None
        self.manager = manager
        self.uuid = str(uuid.uuid4())
        self.html = "

" % self.uuid

    def start(self):
        from IPython.kernel.comm import Comm
        try:
            self.comm = Comm('matplotlib', data={'id': self.uuid})
        except AttributeError:
            raise RuntimeError('Unable to create an IPython notebook Comm '
                               'instance. Are you in the IPython notebook?')
        self.comm.on_msg(self.on_message)
        self.comm.on_close(lambda close_message: self.manager.clearup_closed())

Store.renderer = MPLRenderer

<|endoftext|>


<commit_before>##
###### Razred Igra:

##Metode znotraj razreda Igra bodo:
##* naredi_potezo(self,i,j), ki se bo razdelila na podmetodi premakni(self,i,j) in pojej(self,i,j)
##* razveljavi(self)
##* na_potezi(self)
##* je_veljavna(self,i,j)
##* zmagovalec(self)

class Figura():
    def __init__(self, x, y, barva, dama = False):
        self.x = x
        self.y = y
        self.barva = barva
        self.dama = dama

igralec_C = "C"
igralec_B = "B"

def nasprotnik(igralec):
    if igralec == igralec_C:
        return igralec_B
    elif igralec == igralec_B:
        return igralec_C
    else:
        assert False

class Igra():
    def __init__(self):
        self.deska = [[C, False, C, False, C, False, C, False],
                      [False, C, False, C, False, C, False, C],
                      [C, False, C, False, C, False, C, False],
                      [False,None,False,None,False,None,False,None],
                      [None,False,None,False,None,False,None,False],
                      [False, B, False, B, False, B, False, B],
                      [B, False, B, False, B, False, B, False],
                      [False, B, False, B, False, B, False, B]]

        self.na_potezi = igralec_C
        self.zgodovina = [(self.deska, igralec_C)]

    def shrani_potezo(self):
# seznamu zgodovina pripne par (trenutna pozicija na deski, igralec na potezi)
        a = [self.deska[j][:] for j in range(8)]
        self.zgodovina.append((a, self.na_potezi))

    def razveljavi(self):
# ce igra clovek proti cloveku se razveljavi ena poteza, sicer pa dve(SE DODATI!!)
        (self.deska, self.na_potezi) = self.zgodovina.pop()
##??? ali ta funkcija ze avtomatsko vzame zadnji element 'nove' zgodovine?
##?? ali celoten seznam zgodovine

# pop odstrani zadnji element seznama

    def kopija(self):
        copy = Igra()
        copy.deska = [self.deska[j][:] for j in range(8)]
        copy.na_potezi = self.na_potezi
        return copy

    def je_veljavna(self, i, j):
        for i in range(8):
            for j in range(8):
                if self.zgodovina[-1][i][j] == None:
                    return True
        return False

    def zmagovalec(self):
        pass

    def pojej(self, druga):
        iks = self.x
        ipsilon = self.y
        zeta = druga.x
        teta = druga.y
##        if self.dama:
##
##            if abs(self.x - i)==1 and abs(self.y - j) == 1 and 0<=i<=7 and 0<=j<=7
##            and :
##                self.x = i
##                self.y = j
##                zbrisi(iks, ipsilon)
        if self.barva == crna:
            if druga.barva == bela:
                if self.x - druga.x == 1 and (self.y + 1) == druga.y and self.deska[self.x -2 ][self.y +2] == None:
                    self.x = iks - 2
                    self.y = ipsilon + 2

                    zbrisi(iks, ipsilon)
                    zbrisi(zeta, teta)
                if self.x - druga.x == -1 and (self.y + 1) == druga.y and self.deska[self.x + 2][self.y +2] == None:
                    self.x = iks + 2
                    self.y = ipsilon + 2

                    zbrisi(iks, ipsilon)
                    zbrisi(zeta, teta)
        if self.barva == bela:
            if druga.barva == crna:
                if self.x - druga.x == 1 and (self.y - 1) == druga.y and self.deska[self.x -2 ][self.y -2] == None:
                    self.x = iks - 2
                    self.y = ipsilon - 2

                    zbrisi(iks, ipsilon)
                    zbrisi(zeta, teta)
                if self.x - druga.x == -1 and (self.y - 1) == druga.y and self.deska[self.x + 2][self.y -2] == None:
                    self.x = iks + 2
                    self.y = ipsilon - 2

                    zbrisi(iks, ipsilon)
                    zbrisi(zeta, teta)

    def zbrisi(self,i,j):
        self.deska[i][j] = None

    def premakni(self, i, j):
        iks = self.x
        ipsilon = self.y
        if not je_veljavna(self, i, j):
            return None

        if self.dama:
            if abs(self.x - i)==1 and abs(self.y - j) == 1 and 0<=i<=7 and 0<=j<=7:
                self.x = i
                self.y = j
                zbrisi(iks, ipsilon)

        elif self.barva == crna:
            if abs(self.x - i)==1 and (self.y + 1) == j and 0<=i<=7 and 0<=j<=7:
                self.x = i
                self.y = j
                zbrisi(iks, ipsilon)

        elif self.barva == bela:
            if abs(self.x - i)==1 and (self.y - 1) == j and 0<=i<=7 and 0<=j<=7:
                self.x = i
                self.y = j
                zbrisi(iks, ipsilon)

    def naredi_potezo(self, i, j):
    # če je poteza neveljavna ne naredi ničesar
        if not je_veljavna(self, i, j):
            return None

    # če je poteza veljavna jo izvede

##########################

###### Igralci in njihovi razredi:

##Igra proti računalniku bo vsebovala dva algoritma, enega namerno neizpopolnjenega in lažjega (Minimax) in pa bolj dodelanega (alfa-beta rezanje). Tretja možnost je, da je igralec človek, v tem primeru mu podelimo možnost izvajanja poteze preko uporabniškega vmesnika. Razredi igralcev bodo:
##* class Minimax
##* class Alfa_beta
##* class clovek, ki bo imel metodi:
##    * __init__(self, gui), ki se bo povezala z uporabniškim vmesnikom,
##    * premakni(self)

class Clovek():
    def __init__(self, gui):
        self.gui = gui
    def premakni(self):
        pass

class Minimax():
    pass

class Alfa_Beta():
    pass

##########################

<commit_msg>Pojej - dama<commit_after>##
###### Razred Igra:

##Metode znotraj razreda Igra bodo:
##* naredi_potezo(self,i,j), ki se bo razdelila na podmetodi premakni(self,i,j) in pojej(self,i,j)
##* razveljavi(self)
##* na_potezi(self)
##* je_veljavna(self,i,j)
##* zmagovalec(self)

class Figura():
    def __init__(self, x, y, barva, dama = False):
        self.x = x
        self.y = y
        self.barva = barva
        self.dama = dama

igralec_C = "C"
igralec_B = "B"

def nasprotnik(igralec):
    if igralec == igralec_C:
        return igralec_B
    elif igralec == igralec_B:
        return igralec_C
    else:
        assert False

class Igra():
    def __init__(self):
        self.deska = [[C, False, C, False, C, False, C, False],
                      [False, C, False, C, False, C, False, C],
                      [C, False, C, False, C, False, C, False],
                      [False,None,False,None,False,None,False,None],
                      [None,False,None,False,None,False,None,False],
                      [False, B, False, B, False, B, False, B],
                      [B, False, B, False, B, False, B, False],
                      [False, B, False, B, False, B, False, B]]

        self.na_potezi = igralec_C
        self.zgodovina = [(self.deska, igralec_C)]

    def shrani_potezo(self):
# seznamu zgodovina pripne par (trenutna pozicija na deski, igralec na potezi)
        a = [self.deska[j][:] for j in range(8)]
        self.zgodovina.append((a, self.na_potezi))

    def razveljavi(self):
# ce igra clovek proti cloveku se razveljavi ena poteza, sicer pa dve(SE DODATI!!)
        (self.deska, self.na_potezi) = self.zgodovina.pop()
##??? ali ta funkcija ze avtomatsko vzame zadnji element 'nove' zgodovine?
##?? ali celoten seznam zgodovine

# pop odstrani zadnji element seznama

    def kopija(self):
        copy = Igra()
        copy.deska = [self.deska[j][:] for j in range(8)]
        copy.na_potezi = self.na_potezi
        return copy

    def je_veljavna(self, i, j):
        for i in range(8):
            for j in range(8):
                if self.zgodovina[-1][i][j] == None:
                    return True
        return False

    def zmagovalec(self):
        pass

    def pojej(self, druga):
        iks = self.x
        ipsilon = self.y
        zeta = druga.x
        teta = druga.y
        if self.dama:
            if self.barva != druga.barva:
                if self.x - druga.x == 1 and (self.y + 1) == druga.y and self.deska[self.x -2 ][self.y +2] == None:
                    self.x = iks - 2
                    self.y = ipsilon + 2

                    zbrisi(iks, ipsilon)
                    zbrisi(zeta, teta)
                elif self.x - druga.x == -1 and (self.y + 1) == druga.y and self.deska[self.x + 2][self.y +2] == None:
                    self.x = iks + 2
                    self.y = ipsilon + 2

                    zbrisi(iks, ipsilon)
                    zbrisi(zeta, teta)
                elif self.x - druga.x == 1 and (self.y - 1) == druga.y and self.deska[self.x -2 ][self.y -2] == None:
                    self.x = iks - 2
                    self.y = ipsilon - 2

                    zbrisi(iks, ipsilon)
                    zbrisi(zeta, teta)
                elif self.x - druga.x == -1 and (self.y - 1) == druga.y and self.deska[self.x + 2][self.y -2] == None:
                    self.x = iks + 2
                    self.y = ipsilon - 2

                    zbrisi(iks, ipsilon)
                    zbrisi(zeta, teta)

        elif self.barva == crna:
            if druga.barva == bela:
                if self.x - druga.x == 1 and (self.y + 1) == druga.y and self.deska[self.x -2 ][self.y +2] == None:
                    self.x = iks - 2
                    self.y = ipsilon + 2

                    zbrisi(iks, ipsilon)
                    zbrisi(zeta, teta)
                elif self.x - druga.x == -1 and (self.y + 1) == druga.y and self.deska[self.x + 2][self.y +2] == None:
                    self.x = iks + 2
                    self.y = ipsilon + 2

                    zbrisi(iks, ipsilon)
                    zbrisi(zeta, teta)
        elif self.barva == bela:
            if druga.barva == crna:
                if self.x - druga.x == 1 and (self.y - 1) == druga.y and self.deska[self.x -2 ][self.y -2] == None:
                    self.x = iks - 2
                    self.y = ipsilon - 2

                    zbrisi(iks, ipsilon)
                    zbrisi(zeta, teta)
                elif self.x - druga.x == -1 and (self.y - 1) == druga.y and self.deska[self.x + 2][self.y -2] == None:
                    self.x = iks + 2
                    self.y = ipsilon - 2

                    zbrisi(iks, ipsilon)
                    zbrisi(zeta, teta)
        else:
            assert False, "Figura ne more pojesti"

    def zbrisi(self,i,j):
        self.deska[i][j] = None

    def premakni(self, i, j):
        iks = self.x
        ipsilon = self.y

        if self.dama:
            if abs(self.x - i)==1 and abs(self.y - j) == 1 and 0<=i<=7 and 0<=j<=7:
                self.x = i
                self.y = j
                zbrisi(iks, ipsilon)

        elif self.barva == crna:
            if abs(self.x - i)==1 and (self.y + 1) == j and 0<=i<=7 and 0<=j<=7:
                self.x = i
                self.y = j
                zbrisi(iks, ipsilon)

        elif self.barva == bela:
            if abs(self.x - i)==1 and (self.y - 1) == j and 0<=i<=7 and 0<=j<=7:
                self.x = i
                self.y = j
                zbrisi(iks, ipsilon)

    def naredi_potezo(self, druga):
    # če je poteza neveljavna ne naredi ničesar
    # če je poteza veljavna jo izvede
        if not je_veljavna(self, i, j):
            return None

##########################

###### Igralci in njihovi razredi:

##Igra proti računalniku bo vsebovala dva algoritma, enega namerno neizpopolnjenega in lažjega (Minimax) in pa bolj dodelanega (alfa-beta rezanje). Tretja možnost je, da je igralec človek, v tem primeru mu podelimo možnost izvajanja poteze preko uporabniškega vmesnika. Razredi igralcev bodo:
##* class Minimax
##* class Alfa_beta
##* class clovek, ki bo imel metodi:
##    * __init__(self, gui), ki se bo povezala z uporabniškim vmesnikom,
##    * premakni(self)

class Clovek():
    def __init__(self, gui):
        self.gui = gui
    def premakni(self):
        pass

class Minimax():
    pass

class Alfa_Beta():
    pass

##########################

<|endoftext|>


<commit_before>#!/usr/bin/env python
import os
import json
import struct
import tempfile
import shutil
import datetime
import traceback
import time
from bson.objectid import ObjectId

try:
    import Queue as queue
except ImportError:
    import queue

try:
    import SocketServer as socketserver
except ImportError:
    import socketserver

from dss.tools import buffer, thread, process, ffmpeg, show, Suppress
from dss.tools.os import set_pipe_max_size
from dss.config import config
from dss.storage import db
from dss.websocket import WebsocketBroadcast

from .enum import ContentType, DataContent
from .const import WAIT_TIMEOUT, HEADER_SIZE
from .processing.media import Media
from .processing.data import DataProc

rtmpconf = config['rtmp-server']

class MediaHandler(socketserver.BaseRequestHandler, object):
    """  Packet header description

        The packet header will provide the type of payload data, and the size.

        [ T | S | S | S | S | D | D | ... ]

        T - 1st byte              - Type of message
        S - 2nd to 5th bytes      - Size of payload
        D - Data payload

        Currently we envision just 4 types of messages:
        Metadata, Video, Audio and User generated data.
    """

    provider_prefix = 'M'
    _handlers = set()
    _handlers_lock = thread.Lock()
    _time_limit = config.getint('mobile', 'time_limit')

    def setup(self):
        self._id = None
        self.run = True
        self.buffer = None
        self.proc = None
        self.video = None
        self.audio = None
        self.destination_url = None
        self.data_processing = None
        self.thumbnail_path = None
        self.data_queue = queue.Queue()
        self.tmpdir = tempfile.mkdtemp(dir=config['mobile']['dir'])
        self._timer_alarm = False
        self.timer = thread.Timer(self._time_limit, self.timer_alarm) \
            if self._time_limit else None
        self._error = []
        self.__cleanup_executed = False

    def add_handler(self):
        with self._handlers_lock:
            self._handlers.add(self)

    def remove_handler(self):
        with self._handlers_lock:
            self._handlers.remove(self)

    @property
    def error(self):
        return self._error

    @error.setter
    def error(self, value):
        self._error.append(value)

    @classmethod
    def wait_handlers(cls):
        while cls._handlers:
            time.sleep(0.1)

    def timer_alarm(self):
        self._timer_alarm = True

    def file(self, name, open_options=None):
        """ Return the name of a file in the temp dir.
            If open_options is supplied, the file will
            be opened and returned.
        """
        fname = os.path.join(self.tmpdir, name)
        if open_options is None:
            return fname
        return open(fname, open_options)

    def cleanup(self):
        """ Close pipes, remove files and temp dir and
            kills the transcoding process.
        """
        if self.__cleanup_executed:
            return

        s = Suppress(Exception)

        # Stop timer if any
        if self.timer is not None:
            self.timer.cancel()

        # Stop all threads
        with s: self.audio.stop()
        with s: self.video.stop()
        with s: self.data_processing.stop()

        # Remove entry from database
        if self._id:
            db.mobile.update({'_id': self._id}, {'$set': {'active': False}})

        # Remove temp directory and thumbnail
        with s: shutil.rmtree(self.tmpdir)
        with s: os.remove(self.thumbnail_path)

        WebsocketBroadcast.select('mobile_location').cls.broadcast_message({
            'name': self.get_stream_name(),
            'info': 'finished'
        })

        self.__cleanup_executed = True
        if s.errors:
            show('Errors during cleanup:', *s.errors, sep='\n', end='\n\n')
        show('Mobile stream "{0}" has ended'.format(self._id))

    def finish(self):
        try:
            self.cleanup()
        except Exception as e:
            show('Exception while cleaning:', repr(e))
        finally:
            self.remove_handler()

    def handle(self):
        try:
            self._handle()
        except BaseException:
            show(traceback.format_exc())
            raise

    def _handle(self):
        self.request.settimeout(WAIT_TIMEOUT)
        self.add_handler()
        self.buffer = buffer.Buffer(self.request)
        db_data = {'start': datetime.datetime.utcnow(),
                   'active': True}

        # Read the first data block.
        # It should have complete metadata information with at least the
        # provided id, if it is known (or falsy value otherwise)
        typ, payload = self.read_data()
        typ = DataContent[typ]
        if typ is DataContent.metadata:
            action, payload = DataProc.decode_data(payload)
            try:
                self._id = ObjectId(payload['id'])
            except Exception:
                pass
        else:
            show('Received first data block of type {0.name!r}({0.value}).\n'
                 'Expected {1.name!r}({1.value})', typ, DataContent.metadata)
            return

        response = db.mobile.update({'_id': self._id}, db_data, upsert=True)
        self._id = response.get('upserted', self._id)
        self.send_data(ContentType.meta, {'id': str(self._id)})
        self.destination_url = os.path.join(
            rtmpconf['addr'], rtmpconf['app'], self.get_stream_name()
        )
        show('New mobile stream:', self.destination_url)

        audio_filename = self.file('audio.ts')
        video_filename = self.file('video.ts')
        try:
            os.mkfifo(audio_filename)
            os.mkfifo(video_filename)
        except OSError as e:
            show('Failed to create FIFO:', e)
            return

        audio_pipe = os.open(audio_filename, os.O_RDWR)
        video_pipe = os.open(video_filename, os.O_RDWR)
        set_pipe_max_size(audio_pipe, video_pipe)

        self.audio = Media(audio_pipe, self, 'audio').start()
        self.video = Media(video_pipe, self, 'video').start()
        self.data_processing = DataProc(self).start()

        thumb = config['thumbnail']
        self.thumbnail_path = os.path.join(
            thumb['dir'], self.get_stream_name()
        ) + '.' + thumb['format']

        thumb_rate = str(1 / int(thumb['mobile_interval']))

        args = ffmpeg.cmd_inputs_outputs(
            '-y -re', [audio_filename, video_filename], '',
            ['-c:v copy -c:a libfdk_aac -b:a 64k -f flv',
             '-r ' + thumb_rate + ' -update 1 -an'],
            [self.destination_url, self.thumbnail_path]
        )

        # Start the timer for alarm
        if self.timer is not None:
            self.timer.start()

        with self.file('proc_output', 'w') as out:
            with self.file('proc_err', 'w') as err:
                self.handle_proc_loop(args, out, err)

    def handle_proc_loop(self, proc_args, stdout, stderr):
        """ Open transcoding process and read data from buffer
            until the client stops sending data.
        """
        with process.Popen(proc_args, stdout=stdout, stderr=stderr) as self.proc:
            while self.server.is_running \
                    and not self._timer_alarm \
                    and not self.error \
                    and self.proc.poll() is None:
                try:
                    type, payload = self.read_data()
                except Exception:
                    show('Timeout')
                    break

                if not self.run or type is None:
                    break

                self.handle_content(type, payload)

            if self._timer_alarm:
                show('Stream finished due to time limit: {0} seconds'.format(self._time_limit))

    def handle_content(self, type, payload):
        try:
            type = DataContent[type]
        except KeyError:
            show('Invalid header type "%s"' % type)

        if type in (DataContent.metadata, DataContent.userdata):
            self.data_queue.put((type, payload))
        elif type is DataContent.video:
            self.video.add_data(payload)
        elif type is DataContent.audio:
            self.audio.add_data(payload)
        else:
            show('Unknown content received: ' +
                 'Type: {0}, Payload: {1!r}'.format(type, payload))

    def process_header(self, data):
        """ Strips out the header
            Takes away the header data and returns it as a tuple of values.
        """
        size = struct.unpack('!I', data[1:])
        return data[0], size[0]

    def read_data(self):
        """ Read next packet from buffer.
            Refer to the "MediaHandler" doc for the packet description.
        """
        buff = self.buffer.read(HEADER_SIZE)
        if not buff:
            return None, None
        typ, size = self.process_header(buff)
        payload = self.buffer.read(size)
        if not payload:
            return None, None
        return typ, payload

    def send_data(self, type, content):
        data = {'type': type.name, 'content': content}
        data = json.dumps(data).encode('utf-8')
        self.write_data(DataContent.metadata, data)

    def write_data(self, data_type, data):
        header = struct.pack('!BI', data_type.value, len(data))
        self.request.sendall(header + data)

    def get_stream_name(self):
        return self.stream_name(self._id)

    @classmethod
    def stream_name(cls, id):
        return cls.provider_prefix + '_' + str(id)
<commit_msg>Do not transcode audio when changing the bitstream format solves the packaging issue.<commit_after>#!/usr/bin/env python
import os
import json
import struct
import tempfile
import shutil
import datetime
import traceback
import time
from bson.objectid import ObjectId

try:
    import Queue as queue
except ImportError:
    import queue

try:
    import SocketServer as socketserver
except ImportError:
    import socketserver

from dss.tools import buffer, thread, process, ffmpeg, show, Suppress
from dss.tools.os import set_pipe_max_size
from dss.config import config
from dss.storage import db
from dss.websocket import WebsocketBroadcast

from .enum import ContentType, DataContent
from .const import WAIT_TIMEOUT, HEADER_SIZE
from .processing.media import Media
from .processing.data import DataProc

rtmpconf = config['rtmp-server']

class MediaHandler(socketserver.BaseRequestHandler, object):
    """  Packet header description

        The packet header will provide the type of payload data, and the size.

        [ T | S | S | S | S | D | D | ... ]

        T - 1st byte              - Type of message
        S - 2nd to 5th bytes      - Size of payload
        D - Data payload

        Currently we envision just 4 types of messages:
        Metadata, Video, Audio and User generated data.
    """

    provider_prefix = 'M'
    _handlers = set()
    _handlers_lock = thread.Lock()
    _time_limit = config.getint('mobile', 'time_limit')

    def setup(self):
        self._id = None
        self.run = True
        self.buffer = None
        self.proc = None
        self.video = None
        self.audio = None
        self.destination_url = None
        self.data_processing = None
        self.thumbnail_path = None
        self.data_queue = queue.Queue()
        self.tmpdir = tempfile.mkdtemp(dir=config['mobile']['dir'])
        self._timer_alarm = False
        self.timer = thread.Timer(self._time_limit, self.timer_alarm) \
            if self._time_limit else None
        self._error = []
        self.__cleanup_executed = False

    def add_handler(self):
        with self._handlers_lock:
            self._handlers.add(self)

    def remove_handler(self):
        with self._handlers_lock:
            self._handlers.remove(self)

    @property
    def error(self):
        return self._error

    @error.setter
    def error(self, value):
        self._error.append(value)

    @classmethod
    def wait_handlers(cls):
        while cls._handlers:
            time.sleep(0.1)

    def timer_alarm(self):
        self._timer_alarm = True

    def file(self, name, open_options=None):
        """ Return the name of a file in the temp dir.
            If open_options is supplied, the file will
            be opened and returned.
        """
        fname = os.path.join(self.tmpdir, name)
        if open_options is None:
            return fname
        return open(fname, open_options)

    def cleanup(self):
        """ Close pipes, remove files and temp dir and
            kills the transcoding process.
        """
        if self.__cleanup_executed:
            return

        s = Suppress(Exception)

        # Stop timer if any
        if self.timer is not None:
            self.timer.cancel()

        # Stop all threads
        with s: self.audio.stop()
        with s: self.video.stop()
        with s: self.data_processing.stop()

        # Remove entry from database
        if self._id:
            db.mobile.update({'_id': self._id}, {'$set': {'active': False}})

        # Remove temp directory and thumbnail
        with s: shutil.rmtree(self.tmpdir)
        with s: os.remove(self.thumbnail_path)

        WebsocketBroadcast.select('mobile_location').cls.broadcast_message({
            'name': self.get_stream_name(),
            'info': 'finished'
        })

        self.__cleanup_executed = True
        if s.errors:
            show('Errors during cleanup:', *s.errors, sep='\n', end='\n\n')
        show('Mobile stream "{0}" has ended'.format(self._id))

    def finish(self):
        try:
            self.cleanup()
        except Exception as e:
            show('Exception while cleaning:', repr(e))
        finally:
            self.remove_handler()

    def handle(self):
        try:
            self._handle()
        except BaseException:
            show(traceback.format_exc())
            raise

    def _handle(self):
        self.request.settimeout(WAIT_TIMEOUT)
        self.add_handler()
        self.buffer = buffer.Buffer(self.request)
        db_data = {'start': datetime.datetime.utcnow(),
                   'active': True}

        # Read the first data block.
        # It should have complete metadata information with at least the
        # provided id, if it is known (or falsy value otherwise)
        typ, payload = self.read_data()
        typ = DataContent[typ]
        if typ is DataContent.metadata:
            action, payload = DataProc.decode_data(payload)
            try:
                self._id = ObjectId(payload['id'])
            except Exception:
                pass
        else:
            show('Received first data block of type {0.name!r}({0.value}).\n'
                 'Expected {1.name!r}({1.value})', typ, DataContent.metadata)
            return

        response = db.mobile.update({'_id': self._id}, db_data, upsert=True)
        self._id = response.get('upserted', self._id)
        self.send_data(ContentType.meta, {'id': str(self._id)})
        self.destination_url = os.path.join(
            rtmpconf['addr'], rtmpconf['app'], self.get_stream_name()
        )
        show('New mobile stream:', self.destination_url)

        audio_filename = self.file('audio.ts')
        video_filename = self.file('video.ts')
        try:
            os.mkfifo(audio_filename)
            os.mkfifo(video_filename)
        except OSError as e:
            show('Failed to create FIFO:', e)
            return

        audio_pipe = os.open(audio_filename, os.O_RDWR)
        video_pipe = os.open(video_filename, os.O_RDWR)
        set_pipe_max_size(audio_pipe, video_pipe)

        self.audio = Media(audio_pipe, self, 'audio').start()
        self.video = Media(video_pipe, self, 'video').start()
        self.data_processing = DataProc(self).start()

        thumb = config['thumbnail']
        self.thumbnail_path = os.path.join(
            thumb['dir'], self.get_stream_name()
        ) + '.' + thumb['format']

        thumb_rate = str(1 / int(thumb['mobile_interval']))

        args = ffmpeg.cmd_inputs_outputs(
            '-y -re', [audio_filename, video_filename], '',
            ['-c:v copy -c:a copy -bsf:a aac_adtstoasc -f flv',
             '-r ' + thumb_rate + ' -update 1 -an'],
            [self.destination_url, self.thumbnail_path]
        )

        # Start the timer for alarm
        if self.timer is not None:
            self.timer.start()

        with self.file('proc_output', 'w') as out:
            with self.file('proc_err', 'w') as err:
                self.handle_proc_loop(args, out, err)

    def handle_proc_loop(self, proc_args, stdout, stderr):
        """ Open transcoding process and read data from buffer
            until the client stops sending data.
        """
        with process.Popen(proc_args, stdout=stdout, stderr=stderr) as self.proc:
            while self.server.is_running \
                    and not self._timer_alarm \
                    and not self.error \
                    and self.proc.poll() is None:
                try:
                    type, payload = self.read_data()
                except Exception:
                    show('Timeout')
                    break

                if not self.run or type is None:
                    break

                self.handle_content(type, payload)

            if self._timer_alarm:
                show('Stream finished due to time limit: {0} seconds'.format(self._time_limit))

    def handle_content(self, type, payload):
        try:
            type = DataContent[type]
        except KeyError:
            show('Invalid header type "%s"' % type)

        if type in (DataContent.metadata, DataContent.userdata):
            self.data_queue.put((type, payload))
        elif type is DataContent.video:
            self.video.add_data(payload)
        elif type is DataContent.audio:
            self.audio.add_data(payload)
        else:
            show('Unknown content received: ' +
                 'Type: {0}, Payload: {1!r}'.format(type, payload))

    def process_header(self, data):
        """ Strips out the header
            Takes away the header data and returns it as a tuple of values.
        """
        size = struct.unpack('!I', data[1:])
        return data[0], size[0]

    def read_data(self):
        """ Read next packet from buffer.
            Refer to the "MediaHandler" doc for the packet description.
        """
        buff = self.buffer.read(HEADER_SIZE)
        if not buff:
            return None, None
        typ, size = self.process_header(buff)
        payload = self.buffer.read(size)
        if not payload:
            return None, None
        return typ, payload

    def send_data(self, type, content):
        data = {'type': type.name, 'content': content}
        data = json.dumps(data).encode('utf-8')
        self.write_data(DataContent.metadata, data)

    def write_data(self, data_type, data):
        header = struct.pack('!BI', data_type.value, len(data))
        self.request.sendall(header + data)

    def get_stream_name(self):
        return self.stream_name(self._id)

    @classmethod
    def stream_name(cls, id):
        return cls.provider_prefix + '_' + str(id)
<|endoftext|>


<commit_before>"""API for creating and interacting with dtool datasets.

"""

import datetime
import uuid

from pkg_resources import iter_entry_points
from collections import defaultdict

import dtoolcore.utils

__version__ = "3.0.0"

def _generate_storage_broker_lookup():
    """Return dictionary of available storage brokers."""
    storage_broker_lookup = dict()
    for entrypoint in iter_entry_points("dtool.storage_brokers"):
        StorageBroker = entrypoint.load()
        storage_broker_lookup[StorageBroker.key] = StorageBroker
    return storage_broker_lookup

def _get_storage_broker(uri, config_path):
    """Helper function to enable use lookup of appropriate storage brokers."""
    uri = dtoolcore.utils.sanitise_uri(uri)
    storage_broker_lookup = _generate_storage_broker_lookup()
    parsed_uri = dtoolcore.utils.generous_parse_uri(uri)
    StorageBroker = storage_broker_lookup[parsed_uri.scheme]
    return StorageBroker(uri, config_path)

def _admin_metadata_from_uri(uri, config_path):
    """Helper function for getting admin metadata."""
    uri = dtoolcore.utils.sanitise_uri(uri)
    storage_broker = _get_storage_broker(uri, config_path)
    admin_metadata = storage_broker.get_admin_metadata()
    return admin_metadata

def _is_dataset(uri, config_path):
    """Helper function for determining if a URI is a dataset."""
    uri = dtoolcore.utils.sanitise_uri(uri)
    storage_broker = _get_storage_broker(uri, config_path)
    return storage_broker.has_admin_metadata()

def generate_admin_metadata(name, creator_username=None):
    """Return admin metadata as a dictionary."""
    if creator_username is None:
        creator_username = dtoolcore.utils.getuser()
    admin_metadata = {
        "uuid": str(uuid.uuid4()),
        "dtoolcore_version": __version__,
        "name": name,
        "type": "protodataset",
        "creator_username": creator_username,
    }
    return admin_metadata

def _generate_uri(admin_metadata, base_uri):
    """Return dataset URI.

    :param admin_metadata: dataset administrative metadata
    :param base_uri: base URI from which to derive dataset URI
    :returns: dataset URI
    """
    name = admin_metadata["name"]
    uuid = admin_metadata["uuid"]
    # storage_broker_lookup = _generate_storage_broker_lookup()
    # parse_result = urlparse(base_uri)
    # storage = parse_result.scheme
    StorageBroker = _get_storage_broker(base_uri, config_path=None)
    return StorageBroker.generate_uri(name, uuid, base_uri)

def generate_proto_dataset(admin_metadata, base_uri, config_path=None):
    """Return :class:`dtoolcore.ProtoDataSet` instance.

    :param admin_metadata: dataset administrative metadata
    :param base_uri: base URI for proto dataset
    :param config_path: path to dtool configuration file
    """
    uri = _generate_uri(admin_metadata, base_uri)
    return ProtoDataSet(uri, admin_metadata, config_path)

def _copy_create_proto_dataset(
    src_dataset,
    dest_base_uri,
    config_path=None,
    progressbar=None
):
    if progressbar:
        progressbar.label = "Copying dataset"

    admin_metadata = src_dataset._admin_metadata
    admin_metadata["type"] = "protodataset"
    del admin_metadata["frozen_at"]

    proto_dataset = generate_proto_dataset(
        admin_metadata=admin_metadata,
        base_uri=dest_base_uri,
        config_path=config_path
    )

    # Ensure that this bug does not get re-introduced:
    # https://github.com/jic-dtool/dtoolcore/issues/1
    assert proto_dataset._admin_metadata["type"] == "protodataset"
    assert "frozen_at" not in proto_dataset._admin_metadata

    proto_dataset.create()

    return proto_dataset

def _copy_content(src_dataset, dest_proto_dataset, progressbar=None):

    def get_dest_sizes(dest_proto_dataset):
        sizes = {}
        for handle in dest_proto_dataset._storage_broker.iter_item_handles():
            identifier = dtoolcore.utils.generate_identifier(handle)
            props = dest_proto_dataset._storage_broker.item_properties(handle)
            sizes[identifier] = props["size_in_bytes"]
        return sizes

    dest_sizes = get_dest_sizes(dest_proto_dataset)
    dest_identifiers = set(dest_sizes.keys())
    for identifier in src_dataset.identifiers:
        src_properties = src_dataset.item_properties(identifier)
        if identifier in dest_identifiers:
            src_size = src_properties["size_in_bytes"]
            dest_size = dest_sizes[identifier]
            if src_size == dest_size:
                if progressbar:
                    progressbar.update(1)
                continue
        src_abspath = src_dataset.item_content_abspath(identifier)
        relpath = src_properties["relpath"]
        dest_proto_dataset.put_item(src_abspath, relpath)
        if progressbar:
            progressbar.item_show_func = lambda x: relpath
            progressbar.update(1)

    dest_proto_dataset.put_readme(src_dataset.get_readme_content())

    for overlay_name in src_dataset.list_overlay_names():
        overlay = src_dataset.get_overlay(overlay_name)
        dest_proto_dataset._put_overlay(overlay_name, overlay)

def copy(src_uri, dest_base_uri, config_path=None, progressbar=None):
    """Copy a dataset to another location.

    :param src_uri: URI of dataset to be copied
    :param dest_base_uri: base of URI for copy target
    :param config_path: path to dtool configuration file
    :returns: URI of new dataset
    """
    dataset = DataSet.from_uri(src_uri)

    proto_dataset = _copy_create_proto_dataset(
        dataset,
        dest_base_uri,
        config_path,
        progressbar
    )
    _copy_content(dataset, proto_dataset, progressbar)
    proto_dataset.freeze(progressbar=progressbar)

    return proto_dataset.uri

def copy_resume(src_uri, dest_base_uri, config_path=None, progressbar=None):
    """Copy a dataset to another location.

    :param src_uri: URI of dataset to be copied
    :param dest_base_uri: base of URI for copy target
    :param config_path: path to dtool configuration file
    :returns: URI of new dataset
    """
    dataset = DataSet.from_uri(src_uri)

    # Generate the URI of the destination proto dataset.
    dest_uri = _generate_uri(dataset._admin_metadata, dest_base_uri)

    proto_dataset = ProtoDataSet.from_uri(dest_uri)

    _copy_content(dataset, proto_dataset, progressbar)
    proto_dataset.freeze(progressbar=progressbar)

    return proto_dataset.uri

class DtoolCoreTypeError(TypeError):
    pass

class _BaseDataSet(object):
    """Base class for datasets."""

    def __init__(self, uri, admin_metadata, config_path=None):

        uri = dtoolcore.utils.sanitise_uri(uri)

        self._admin_metadata = admin_metadata
        self._storage_broker = _get_storage_broker(uri, config_path)
        self._uri = uri

    @classmethod
    def _from_uri_with_typecheck(cls, uri, config_path, type_name):
        # Make sure that the URI refers to a dataset.
        if not _is_dataset(uri, config_path):
            raise(DtoolCoreTypeError("{} is not a dataset".format(uri)))

        # Get the admin metadata out of the URI and type check.
        admin_metadata = _admin_metadata_from_uri(uri, config_path)
        if admin_metadata['type'] != type_name:
            raise DtoolCoreTypeError(
                "{} is not a {}".format(uri, cls.__name__))

        # Instantiate and return.
        return cls(uri, admin_metadata, config_path)

    @property
    def uri(self):
        """Return the URI of the dataset."""
        return self._uri

    @property
    def uuid(self):
        """Return the UUID of the dataset."""
        return self._admin_metadata["uuid"]

    @property
    def name(self):
        """Return the name of the dataset."""
        return self._admin_metadata['name']

    def get_readme_content(self):
        """
        Return the content of the README describing the dataset.

        :returns: content of README as a string
        """
        return self._storage_broker.get_readme_content()

    def _put_overlay(self, overlay_name, overlay):
        """Store overlay so that it is accessible by the given name.

        :param overlay_name: name of the overlay
        :param overlay: overlay must be a dictionary where the keys are
                        identifiers in the dataset
        :raises: TypeError if the overlay is not a dictionary,
                 ValueError if identifiers in overlay and dataset do not match
        """
        if not isinstance(overlay, dict):
            raise TypeError("Overlay must be dict")

        if set(self._identifiers()) != set(overlay.keys()):
            raise ValueError("Overlay keys must be dataset identifiers")

        self._storage_broker.put_overlay(overlay_name, overlay)

    def generate_manifest(self, progressbar=None):
        """Return manifest generated from knowledge about contents."""
        items = dict()

        if progressbar:
            progressbar.label = "Generating manifest"

        for handle in self._storage_broker.iter_item_handles():
            key = dtoolcore.utils.generate_identifier(handle)
            value = self._storage_broker.item_properties(handle)
            items[key] = value
            if progressbar:
                progressbar.item_show_func = lambda x: handle
                progressbar.update(1)

        manifest = {
            "items": items,
            "dtoolcore_version": __version__,
            "hash_function": self._storage_broker.hasher.name
        }

        return manifest

class DataSet(_BaseDataSet):
    """
    Class for reading the contents of a dataset.
    """

    def __init__(self, uri, admin_metadata, config_path=None):
        super(DataSet, self).__init__(uri, admin_metadata, config_path)
        self._manifest_cache = None

    def _identifiers(self):
        return self._manifest["items"].keys()

    @classmethod
    def from_uri(cls, uri, config_path=None):
        """
        Return an existing :class:`dtoolcore.DataSet` from a URI.

        :params uri: unique resource identifier where the existing
                     :class:`dtoolcore.DataSet` is stored
        :returns: :class:`dtoolcore.DataSet`
        """
        return cls._from_uri_with_typecheck(uri, config_path, "dataset")

    @property
    def identifiers(self):
        """Return iterable of dataset item identifiers."""
        return self._identifiers()

    @property
    def _manifest(self):
        """Return manifest content."""
        if self._manifest_cache is None:
            self._manifest_cache = self._storage_broker.get_manifest()

        return self._manifest_cache

    def item_properties(self, identifier):
        """Return properties of the item with the given identifier.

        :param identifier: item identifier
        :returns: dictionary of item properties from the manifest
        """
        return self._manifest["items"][identifier]

    def item_content_abspath(self, identifier):
        """Return absolute path at which item content can be accessed.

        :param identifier: item identifier
        :returns: absolute path from which the item content can be accessed
        """
        return self._storage_broker.get_item_abspath(identifier)

    def list_overlay_names(self):
        """Return list of overlay names."""
        return self._storage_broker.list_overlay_names()

    def get_overlay(self, overlay_name):
        """Return overlay as a dictionary.

        :param overlay_name: name of the overlay
        :returns: overlay as a dictionary
        """
        return self._storage_broker.get_overlay(overlay_name)

    def put_overlay(self, overlay_name, overlay):
        """Store overlay so that it is accessible by the given name.

        :param overlay_name: name of the overlay
        :param overlay: overlay must be a dictionary where the keys are
                        identifiers in the dataset
        :raises: TypeError if the overlay is not a dictionary,
                 ValueError if identifiers in overlay and dataset do not match
        """
        self._put_overlay(overlay_name, overlay)

class ProtoDataSet(_BaseDataSet):
    """
    Class for building up a dataset.
    """

    @classmethod
    def from_uri(cls, uri, config_path=None):
        """
        Return an existing :class:`dtoolcore.ProtoDataSet` from a URI.

        :params uri: unique resource identifier where the existing
                     :class:`dtoolcore.ProtoDataSet` is stored
        :returns: :class:`dtoolcore.ProtoDataSet`
        """
        return cls._from_uri_with_typecheck(uri, config_path, "protodataset")

    def _identifiers(self):
        """Return iterable of dataset item identifiers."""
        for handle in self._storage_broker.iter_item_handles():
            yield dtoolcore.utils.generate_identifier(handle)

    def update_name(self, new_name):
        """Update the name of the proto dataset.

        :param new_name: the new name of the proto dataset
        """
        self._admin_metadata['name'] = new_name
        if self._storage_broker.has_admin_metadata():
            self._storage_broker.put_admin_metadata(self._admin_metadata)

    def create(self):
        """Create the required directory structure and admin metadata."""
        self._storage_broker.create_structure()
        self._storage_broker.put_admin_metadata(self._admin_metadata)

    def put_readme(self, content):
        """
        Put content into the README of the dataset.

        The client is responsible for ensuring that the content is valid YAML.

        :param content: string to put into the README
        """
        self._storage_broker.put_readme(content)

    def put_item(self, fpath, relpath):
        """
        Put an item into the dataset.

        :param fpath: path to the item on disk
        :param relpath: relative path name given to the item in the dataset as
                        a handle
        :returns: the handle given to the item
        """
        return self._storage_broker.put_item(fpath, relpath)

    def add_item_metadata(self, handle, key, value):
        """
        Add metadata to a specific item in the :class:`dtoolcore.ProtoDataSet`.

        :param handle: handle representing the relative path of the item in the
                       :class:`dtoolcore.ProtoDataSet`
        :param key: metadata key
        :param value: metadata value
        """
        self._storage_broker.add_item_metadata(handle, key, value)

    def _generate_overlays(self):
        """Return dictionary of overlays generated from added item metadata."""
        overlays = defaultdict(dict)
        for handle in self._storage_broker.iter_item_handles():
            identifier = dtoolcore.utils.generate_identifier(handle)
            item_metadata = self._storage_broker.get_item_metadata(handle)
            for k, v in item_metadata.items():
                overlays[k][identifier] = v

        return overlays

    def freeze(self, progressbar=None):
        """
        Convert :class:`dtoolcore.ProtoDataSet` to :class:`dtoolcore.DataSet`.
        """
        # Call the storage broker pre_freeze hook.
        self._storage_broker.pre_freeze_hook()

        if progressbar:
            progressbar.label = "Freezing dataset"

        # Generate and persist the manifest.
        manifest = self.generate_manifest(progressbar=progressbar)
        self._storage_broker.put_manifest(manifest)

        # Generate and persist overlays from any item metadata that has been
        # added.

        overlays = self._generate_overlays()
        for overlay_name, overlay in overlays.items():
            self._put_overlay(overlay_name, overlay)

        # Change the type of the dataset from "protodataset" to "dataset" and
        # add a "frozen_at" time stamp to the administrative metadata.
        datetime_obj = datetime.datetime.utcnow()
        metadata_update = {
            "type": "dataset",
            "frozen_at": dtoolcore.utils.timestamp(datetime_obj)
        }
        self._admin_metadata.update(metadata_update)
        self._storage_broker.put_admin_metadata(self._admin_metadata)

        # Clean up using the storage broker's post freeze hook.
        self._storage_broker.post_freeze_hook()
<commit_msg>Add explanatory comments<commit_after>"""API for creating and interacting with dtool datasets.

"""

import datetime
import uuid

from pkg_resources import iter_entry_points
from collections import defaultdict

import dtoolcore.utils

__version__ = "3.0.0"

def _generate_storage_broker_lookup():
    """Return dictionary of available storage brokers."""
    storage_broker_lookup = dict()
    for entrypoint in iter_entry_points("dtool.storage_brokers"):
        StorageBroker = entrypoint.load()
        storage_broker_lookup[StorageBroker.key] = StorageBroker
    return storage_broker_lookup

def _get_storage_broker(uri, config_path):
    """Helper function to enable use lookup of appropriate storage brokers."""
    uri = dtoolcore.utils.sanitise_uri(uri)
    storage_broker_lookup = _generate_storage_broker_lookup()
    parsed_uri = dtoolcore.utils.generous_parse_uri(uri)
    StorageBroker = storage_broker_lookup[parsed_uri.scheme]
    return StorageBroker(uri, config_path)

def _admin_metadata_from_uri(uri, config_path):
    """Helper function for getting admin metadata."""
    uri = dtoolcore.utils.sanitise_uri(uri)
    storage_broker = _get_storage_broker(uri, config_path)
    admin_metadata = storage_broker.get_admin_metadata()
    return admin_metadata

def _is_dataset(uri, config_path):
    """Helper function for determining if a URI is a dataset."""
    uri = dtoolcore.utils.sanitise_uri(uri)
    storage_broker = _get_storage_broker(uri, config_path)
    return storage_broker.has_admin_metadata()

def generate_admin_metadata(name, creator_username=None):
    """Return admin metadata as a dictionary."""
    if creator_username is None:
        creator_username = dtoolcore.utils.getuser()
    admin_metadata = {
        "uuid": str(uuid.uuid4()),
        "dtoolcore_version": __version__,
        "name": name,
        "type": "protodataset",
        "creator_username": creator_username,
    }
    return admin_metadata

def _generate_uri(admin_metadata, base_uri):
    """Return dataset URI.

    :param admin_metadata: dataset administrative metadata
    :param base_uri: base URI from which to derive dataset URI
    :returns: dataset URI
    """
    name = admin_metadata["name"]
    uuid = admin_metadata["uuid"]
    # storage_broker_lookup = _generate_storage_broker_lookup()
    # parse_result = urlparse(base_uri)
    # storage = parse_result.scheme
    StorageBroker = _get_storage_broker(base_uri, config_path=None)
    return StorageBroker.generate_uri(name, uuid, base_uri)

def generate_proto_dataset(admin_metadata, base_uri, config_path=None):
    """Return :class:`dtoolcore.ProtoDataSet` instance.

    :param admin_metadata: dataset administrative metadata
    :param base_uri: base URI for proto dataset
    :param config_path: path to dtool configuration file
    """
    uri = _generate_uri(admin_metadata, base_uri)
    return ProtoDataSet(uri, admin_metadata, config_path)

def _copy_create_proto_dataset(
    src_dataset,
    dest_base_uri,
    config_path=None,
    progressbar=None
):
    if progressbar:
        progressbar.label = "Copying dataset"

    admin_metadata = src_dataset._admin_metadata
    admin_metadata["type"] = "protodataset"
    del admin_metadata["frozen_at"]

    proto_dataset = generate_proto_dataset(
        admin_metadata=admin_metadata,
        base_uri=dest_base_uri,
        config_path=config_path
    )

    # Ensure that this bug does not get re-introduced:
    # https://github.com/jic-dtool/dtoolcore/issues/1
    assert proto_dataset._admin_metadata["type"] == "protodataset"
    assert "frozen_at" not in proto_dataset._admin_metadata

    proto_dataset.create()

    return proto_dataset

def _copy_content(src_dataset, dest_proto_dataset, progressbar=None):

    # When using ``dtoolcore.copy`` there should be no handles in the
    # destination proto dataset so this funciton will return an empty
    # dictionary.
    # When using ``dtoolcore.copy_resume`` there may be handles in the
    # destination proto dataset. We therefore need to get the properties
    # of the item associated with the handle so that we can use the size
    # to ensure that the item has been copied across successfully.
    def get_dest_sizes(dest_proto_dataset):
        sizes = {}
        for handle in dest_proto_dataset._storage_broker.iter_item_handles():
            identifier = dtoolcore.utils.generate_identifier(handle)
            props = dest_proto_dataset._storage_broker.item_properties(handle)
            sizes[identifier] = props["size_in_bytes"]
        return sizes

    dest_sizes = get_dest_sizes(dest_proto_dataset)
    dest_identifiers = set(dest_sizes.keys())
    for identifier in src_dataset.identifiers:
        src_properties = src_dataset.item_properties(identifier)

        # We don't want to redo the copy of the item if it has already been
        # done successfully.
        if identifier in dest_identifiers:
            src_size = src_properties["size_in_bytes"]
            dest_size = dest_sizes[identifier]
            if src_size == dest_size:
                if progressbar:
                    progressbar.update(1)
                continue

        src_abspath = src_dataset.item_content_abspath(identifier)
        relpath = src_properties["relpath"]
        dest_proto_dataset.put_item(src_abspath, relpath)
        if progressbar:
            progressbar.item_show_func = lambda x: relpath
            progressbar.update(1)

    dest_proto_dataset.put_readme(src_dataset.get_readme_content())

    for overlay_name in src_dataset.list_overlay_names():
        overlay = src_dataset.get_overlay(overlay_name)
        dest_proto_dataset._put_overlay(overlay_name, overlay)

def copy(src_uri, dest_base_uri, config_path=None, progressbar=None):
    """Copy a dataset to another location.

    :param src_uri: URI of dataset to be copied
    :param dest_base_uri: base of URI for copy target
    :param config_path: path to dtool configuration file
    :returns: URI of new dataset
    """
    dataset = DataSet.from_uri(src_uri)

    proto_dataset = _copy_create_proto_dataset(
        dataset,
        dest_base_uri,
        config_path,
        progressbar
    )
    _copy_content(dataset, proto_dataset, progressbar)
    proto_dataset.freeze(progressbar=progressbar)

    return proto_dataset.uri

def copy_resume(src_uri, dest_base_uri, config_path=None, progressbar=None):
    """Copy a dataset to another location.

    :param src_uri: URI of dataset to be copied
    :param dest_base_uri: base of URI for copy target
    :param config_path: path to dtool configuration file
    :returns: URI of new dataset
    """
    dataset = DataSet.from_uri(src_uri)

    # Generate the URI of the destination proto dataset.
    dest_uri = _generate_uri(dataset._admin_metadata, dest_base_uri)

    proto_dataset = ProtoDataSet.from_uri(dest_uri)

    _copy_content(dataset, proto_dataset, progressbar)
    proto_dataset.freeze(progressbar=progressbar)

    return proto_dataset.uri

class DtoolCoreTypeError(TypeError):
    pass

class _BaseDataSet(object):
    """Base class for datasets."""

    def __init__(self, uri, admin_metadata, config_path=None):

        uri = dtoolcore.utils.sanitise_uri(uri)

        self._admin_metadata = admin_metadata
        self._storage_broker = _get_storage_broker(uri, config_path)
        self._uri = uri

    @classmethod
    def _from_uri_with_typecheck(cls, uri, config_path, type_name):
        # Make sure that the URI refers to a dataset.
        if not _is_dataset(uri, config_path):
            raise(DtoolCoreTypeError("{} is not a dataset".format(uri)))

        # Get the admin metadata out of the URI and type check.
        admin_metadata = _admin_metadata_from_uri(uri, config_path)
        if admin_metadata['type'] != type_name:
            raise DtoolCoreTypeError(
                "{} is not a {}".format(uri, cls.__name__))

        # Instantiate and return.
        return cls(uri, admin_metadata, config_path)

    @property
    def uri(self):
        """Return the URI of the dataset."""
        return self._uri

    @property
    def uuid(self):
        """Return the UUID of the dataset."""
        return self._admin_metadata["uuid"]

    @property
    def name(self):
        """Return the name of the dataset."""
        return self._admin_metadata['name']

    def get_readme_content(self):
        """
        Return the content of the README describing the dataset.

        :returns: content of README as a string
        """
        return self._storage_broker.get_readme_content()

    def _put_overlay(self, overlay_name, overlay):
        """Store overlay so that it is accessible by the given name.

        :param overlay_name: name of the overlay
        :param overlay: overlay must be a dictionary where the keys are
                        identifiers in the dataset
        :raises: TypeError if the overlay is not a dictionary,
                 ValueError if identifiers in overlay and dataset do not match
        """
        if not isinstance(overlay, dict):
            raise TypeError("Overlay must be dict")

        if set(self._identifiers()) != set(overlay.keys()):
            raise ValueError("Overlay keys must be dataset identifiers")

        self._storage_broker.put_overlay(overlay_name, overlay)

    def generate_manifest(self, progressbar=None):
        """Return manifest generated from knowledge about contents."""
        items = dict()

        if progressbar:
            progressbar.label = "Generating manifest"

        for handle in self._storage_broker.iter_item_handles():
            key = dtoolcore.utils.generate_identifier(handle)
            value = self._storage_broker.item_properties(handle)
            items[key] = value
            if progressbar:
                progressbar.item_show_func = lambda x: handle
                progressbar.update(1)

        manifest = {
            "items": items,
            "dtoolcore_version": __version__,
            "hash_function": self._storage_broker.hasher.name
        }

        return manifest

class DataSet(_BaseDataSet):
    """
    Class for reading the contents of a dataset.
    """

    def __init__(self, uri, admin_metadata, config_path=None):
        super(DataSet, self).__init__(uri, admin_metadata, config_path)
        self._manifest_cache = None

    def _identifiers(self):
        return self._manifest["items"].keys()

    @classmethod
    def from_uri(cls, uri, config_path=None):
        """
        Return an existing :class:`dtoolcore.DataSet` from a URI.

        :params uri: unique resource identifier where the existing
                     :class:`dtoolcore.DataSet` is stored
        :returns: :class:`dtoolcore.DataSet`
        """
        return cls._from_uri_with_typecheck(uri, config_path, "dataset")

    @property
    def identifiers(self):
        """Return iterable of dataset item identifiers."""
        return self._identifiers()

    @property
    def _manifest(self):
        """Return manifest content."""
        if self._manifest_cache is None:
            self._manifest_cache = self._storage_broker.get_manifest()

        return self._manifest_cache

    def item_properties(self, identifier):
        """Return properties of the item with the given identifier.

        :param identifier: item identifier
        :returns: dictionary of item properties from the manifest
        """
        return self._manifest["items"][identifier]

    def item_content_abspath(self, identifier):
        """Return absolute path at which item content can be accessed.

        :param identifier: item identifier
        :returns: absolute path from which the item content can be accessed
        """
        return self._storage_broker.get_item_abspath(identifier)

    def list_overlay_names(self):
        """Return list of overlay names."""
        return self._storage_broker.list_overlay_names()

    def get_overlay(self, overlay_name):
        """Return overlay as a dictionary.

        :param overlay_name: name of the overlay
        :returns: overlay as a dictionary
        """
        return self._storage_broker.get_overlay(overlay_name)

    def put_overlay(self, overlay_name, overlay):
        """Store overlay so that it is accessible by the given name.

        :param overlay_name: name of the overlay
        :param overlay: overlay must be a dictionary where the keys are
                        identifiers in the dataset
        :raises: TypeError if the overlay is not a dictionary,
                 ValueError if identifiers in overlay and dataset do not match
        """
        self._put_overlay(overlay_name, overlay)

class ProtoDataSet(_BaseDataSet):
    """
    Class for building up a dataset.
    """

    @classmethod
    def from_uri(cls, uri, config_path=None):
        """
        Return an existing :class:`dtoolcore.ProtoDataSet` from a URI.

        :params uri: unique resource identifier where the existing
                     :class:`dtoolcore.ProtoDataSet` is stored
        :returns: :class:`dtoolcore.ProtoDataSet`
        """
        return cls._from_uri_with_typecheck(uri, config_path, "protodataset")

    def _identifiers(self):
        """Return iterable of dataset item identifiers."""
        for handle in self._storage_broker.iter_item_handles():
            yield dtoolcore.utils.generate_identifier(handle)

    def update_name(self, new_name):
        """Update the name of the proto dataset.

        :param new_name: the new name of the proto dataset
        """
        self._admin_metadata['name'] = new_name
        if self._storage_broker.has_admin_metadata():
            self._storage_broker.put_admin_metadata(self._admin_metadata)

    def create(self):
        """Create the required directory structure and admin metadata."""
        self._storage_broker.create_structure()
        self._storage_broker.put_admin_metadata(self._admin_metadata)

    def put_readme(self, content):
        """
        Put content into the README of the dataset.

        The client is responsible for ensuring that the content is valid YAML.

        :param content: string to put into the README
        """
        self._storage_broker.put_readme(content)

    def put_item(self, fpath, relpath):
        """
        Put an item into the dataset.

        :param fpath: path to the item on disk
        :param relpath: relative path name given to the item in the dataset as
                        a handle
        :returns: the handle given to the item
        """
        return self._storage_broker.put_item(fpath, relpath)

    def add_item_metadata(self, handle, key, value):
        """
        Add metadata to a specific item in the :class:`dtoolcore.ProtoDataSet`.

        :param handle: handle representing the relative path of the item in the
                       :class:`dtoolcore.ProtoDataSet`
        :param key: metadata key
        :param value: metadata value
        """
        self._storage_broker.add_item_metadata(handle, key, value)

    def _generate_overlays(self):
        """Return dictionary of overlays generated from added item metadata."""
        overlays = defaultdict(dict)
        for handle in self._storage_broker.iter_item_handles():
            identifier = dtoolcore.utils.generate_identifier(handle)
            item_metadata = self._storage_broker.get_item_metadata(handle)
            for k, v in item_metadata.items():
                overlays[k][identifier] = v

        return overlays

    def freeze(self, progressbar=None):
        """
        Convert :class:`dtoolcore.ProtoDataSet` to :class:`dtoolcore.DataSet`.
        """
        # Call the storage broker pre_freeze hook.
        self._storage_broker.pre_freeze_hook()

        if progressbar:
            progressbar.label = "Freezing dataset"

        # Generate and persist the manifest.
        manifest = self.generate_manifest(progressbar=progressbar)
        self._storage_broker.put_manifest(manifest)

        # Generate and persist overlays from any item metadata that has been
        # added.

        overlays = self._generate_overlays()
        for overlay_name, overlay in overlays.items():
            self._put_overlay(overlay_name, overlay)

        # Change the type of the dataset from "protodataset" to "dataset" and
        # add a "frozen_at" time stamp to the administrative metadata.
        datetime_obj = datetime.datetime.utcnow()
        metadata_update = {
            "type": "dataset",
            "frozen_at": dtoolcore.utils.timestamp(datetime_obj)
        }
        self._admin_metadata.update(metadata_update)
        self._storage_broker.put_admin_metadata(self._admin_metadata)

        # Clean up using the storage broker's post freeze hook.
        self._storage_broker.post_freeze_hook()
<|endoftext|>