<commit_before>#!/usr/bin/env python

"""
Produces slice images for Mango volume data stored in NetCDF files.

Currently supported data types are tomo, segmented and labels. If the
data is split into multiple files, those must sit in a common directory
and that directory given as the path argument.

Slices are taken through the center of the volume. If either of the
dimensions for a slice image would be less than 10, that slice is
suppressed. Image output is in .png format.

Typical usage:
    for (data, name, action) in slices(path):
        fp = file(name, 'wb')
        fp.write(data)
        fp.close()

(Requires Python 2.6 or higher.)
"""

import os, os.path, re, struct, sys
import numpy
import bz2

from logger import Logger, LOGGER_INFO, LOGGER_WARNING
import make_image
from nc3header import NC3File

class Histogram:
    """
    Maintains a frequency count for a series of numpy arrays. Entries
    equal to <mask_value> are counted separately. Negative values are
    ignored.
    """

    def __init__(self, mask_value = 0, minval = None, maxval = None):
        # -- save the mask value
        self.mask_value = mask_value
        # -- initialize the frequency count
        self.counts = numpy.array([], dtype = 'uint64')
        # -- initialize the count of masked and total non-negative entries
        self.total = 0
        self.masked = 0

        if maxval is not None:
            self.offset = minval
            self.binsize = (maxval - minval) * (1.0 - 1.0e-12) / 0x10000;
        else:
            self.offset = 0
            self.binsize = 1

    def update(self, slice):
        """
        Updates the frequency count with the data from the numpy array
        <slice>.
        """

        # -- process the new data
        tmp = (slice.flatten() - self.offset) / self.binsize
        flat = numpy.array(tmp, dtype = 'uint16')
        mask = (flat == self.mask_value) | (flat < 0) | (flat > 0xffff)
        new_masked = flat[mask].size
        new_counts = numpy.bincount(numpy.where(mask, 0, flat))

        # -- update the frequency count, resizing if necessary
        s = max(self.counts.size, new_counts.size)
        if s > self.counts.size:
            self.counts.resize(s)
        if s > new_counts.size:
            new_counts.resize(s)
        self.counts += new_counts

        # -- update the count of masked and total non-negative entries
        self.masked += new_masked
        self.total  += new_masked + int(new_counts.sum())

class Slice:
    def __init__(self, size, type, axis, pos):
        self.axis = axis.lower()
        self.pos  = pos
        self.slice_dims = {'x': (size[2], size[1]),
                           'y': (size[2], size[0]),
                           'z': (size[1], size[0]) }[self.axis]
        self.content = numpy.zeros(self.slice_dims, type)

    def update(self, z_slice, z_pos):
        """
        Updates this slices with data from the array <z_slice>, which
        is taken to be at z = <z_pos>.
        """

        if self.axis == 'x':
            self.content[z_pos, :] = z_slice[:, self.pos]
        elif self.axis == 'y':
            self.content[z_pos, :] = z_slice[self.pos, :]
        elif self.axis == 'z':
            if z_pos == self.pos:
                self.content[:, :] = z_slice[:, :]

def get_attribute(file, var, name):
    """
    Looks in the open NC3File <file> for the attribute <name>,
    first in the given variable <var> and then in the global
    attributes. Return the attribute's value if found, or None
    otherwise.
    """

    for attr in var.attributes:
        if attr.name == name:
            return attr.value
    for attr in file.attributes:
        if attr.name == name:
            return attr.value
    return None

def volume_variable(file, var):
    """
    Extracts information pertaining to the volume variable <var> within
    the open NC3File <file>. The name, shape, origin and data type -
    mapped to the corresponding numpy type - are stored.
    """

    # -- remember the name
    name = var.name

    # -- extract the variable's shape
    dims = var.dimensions
    size = (dims[2].value, dims[1].value, dims[0].value)
    zdim_total = get_attribute(file, var, 'zdim_total')
    if zdim_total is not None:
        size = (size[0], size[1], zdim_total[0])

    # -- determine the origin
    origin = get_attribute(file, var, 'coordinate_origin_xyz') or (0, 0, 0)

    # -- determine the data type
    (dtype, big_endian_type) = {
      'b': (numpy.uint8,  numpy.dtype(">u1")),
      'h': (numpy.uint16, numpy.dtype(">u2")),
      'i': (numpy.int32,  numpy.dtype(">i4")),
      'f': (numpy.float32, numpy.dtype(">f4")) }[var.python_type_code]

    return { 'name'  : name,
             'size'  : size,
             'origin': origin,
             'dtype' : dtype,
             'big_endian_type': big_endian_type }

def find_variable(path):
    """
    Looks for a volume variable to extract slice images from in the
    NetCDF file at location <path>. Returns an descriptor object of
    class VolumeVariable if something appropriate is found, or None
    otherwise.
    """

    # -- the default return value
    var = None

    # -- open the NetCDF file
    file = NC3File(path)

    # -- loop through all the variables in the file
    for v in file.variables:
        if (len(v.dimensions) == 3 and v.dimensions[0].value > 1 and
            v.python_type_code in 'bhif'
            ):
            var = volume_variable(file, v)
            break

    # -- close the file and return the result
    file.close()
    return var

def z_slices(variable, path):
    """
    A generator method that yields constant z slices corresponding
    to the variable <var> from the NetCDF file located at <path>.

    Each value produced is a pair containing the z coordinate of
    the slice and a two-dimensional numpy array containing the
    extracted data.

    Usage example:

        for (z, data) in z_slices(var, path):
            ... # do something with data
    """

    file = NC3File(path)

    for var in file.variables:
        if var.name == variable['name']:
            break
    else:
        return

    if volume_variable(file, var) != variable:
        raise RuntimeError("variable mismatch between files")

    (x, y, z) = variable['size']

    z_range = get_attribute(file, var, 'zdim_range')
    if z_range is None:
        z_range = range(0, z)
    else:
        z_range = range(z_range[0], z_range[1] + 1)

    bytes_per_slice = x * y * var.element_size
    offset = var.data_start

    file.close()

    if path.endswith('.bz2'):
        fp = bz2.BZ2File(path, 'r', 1024 * 1024)
    else:
        fp = open(path, "rb")
    fp.seek(var.data_start)
    for z in z_range:
        buffer = fp.read(bytes_per_slice)
        if len(buffer) < bytes_per_slice:
            yield (z, None, "insufficient data")
            break
        data = numpy.fromstring(buffer, variable['big_endian_type'])
        data.shape = (y, x)
        yield (z, data)
    fp.close()

def bottom_percentile(histogram, p):
    """
    Returns the smallest number i such that at least <p> percent of
    the non-masked entries counted so far have value i or less.
    """
    threshold = p * (histogram.total - histogram.masked) / 100.0
    count = 0
    for i in xrange(histogram.counts.size):
        count += int(histogram.counts[i])
        if count >= threshold:
            return histogram.offset + i * histogram.binsize

def top_percentile(histogram, p):
    """
    Returns the largest number i such that at least <p> percent of
    the non-masked entries counted so far have value i or more.
    """
    threshold = p * (histogram.total - histogram.masked) / 100.0
    count = 0
    for i in xrange(histogram.counts.size-1, -1, -1):
        count += int(histogram.counts[i])
        if count >= threshold:
            return histogram.offset + i * histogram.binsize

def sizeprefix(thumb_size):
    return ("__%sx%s__" % thumb_size) if thumb_size else ""

def default_slice_set(var, delta, basename):
    """
    Creates a default list of empty slice instances, paired with
    associated file names, based on the shape of the volume variable
    <var>. For each axis, a slice centered at that axis is created,
    provided that the extend of the slice in both directions would be at
    least <delta>.
    """
    pos = list((x - 1) / 2 for x in var['size'])
    size = var['size']
    dtype = var['dtype']
    origin = var['origin']

    def slice_and_name(axis, pos, offset):
        s = Slice(size, dtype, axis, pos)
        n = "slice%c%d_%s.png" % (axis.upper(), pos + offset, basename)
        return (s, n)

    slices = []
    if size[1] > delta and size[2] > delta:
        slices.append(slice_and_name('x', pos[0], origin[0]))
    if size[0] > delta and size[2] > delta:
        slices.append(slice_and_name('y', pos[1], origin[1]))
    if size[0] > delta and size[1] > delta:
        slices.append(slice_and_name('z', pos[2], origin[2]))

    return slices

def image_data(slice, lo, hi, mask_val, info, thumb_size = None):
    """
    Converts a slice into an image and returns it as a PNG encoded
    string. The values <lo> and <hi> specify the range of relevant
    data. Entries equal to <mask_val> are treated as masked.

    An appropriate encoding mode is determined from the data type
    and value range. Details of the actual encoding are deferred to
    the function 'make_image' in the package of the same name.
    """

    # -- determine the encoding mode
    content = slice.content
    if content.dtype == numpy.uint8:
        if hi <= 1:
            img_mode = make_image.BLACK_AND_WHITE
        else:
            img_mode = make_image.COLOR_CODED_FIXED
    elif content.dtype == numpy.uint16:
        img_mode = make_image.GRAYSCALE
    elif content.dtype == numpy.int32:
        img_mode = make_image.COLOR_CODED
    elif content.dtype == numpy.float32:
        img_mode = make_image.GRAYSCALE
    else:
        raise Exception("unexpected array type: %s" % content.dtype)

    # -- generate slice-specific info
    myinfo = info.copy()
    myinfo.update({ 'slice-axis': slice.axis, 'slice-pos': slice.pos })

    # -- generate and return the data
    return make_image.make_image(content, lo, hi, mask_val, img_mode,
                                 thumb_size, myinfo)

def data_range(var, entries):
    minval = maxval = None

    for filename in entries:
        for tmp in z_slices(var, filename):
            z, data = tmp[:2]
            if data is not None:
                lo = numpy.min(data)
                hi = numpy.max(data)
                if minval is None or lo < minval:
                    minval = lo
                if maxval is None or hi > maxval:
                    maxval = hi

    return (minval, maxval)

def slices(path,
           existing = [],
           replace = False,
           dry_run = False,
           sizes = (None,),
           info = {}):
    """
    A generator which extracts slice images from a Mango volume data set
    stored in a collection of NetCDF files.

    The parameter <path> specifies a single NetCDF file or a directory
    containing a single volume split into several NetCDF files.

    Basic usage:
        for (data, name, action) in slices(path):
            fp = file(name, 'wb')
            fp.write(data)
            fp.close()
    """

    path      = re.sub("/$", "", path)
    name      = re.sub("[._]nc$", "", os.path.basename(path))
    basename  = re.sub("^tomo", "tom", re.sub("^segmented", "seg", name))
    img_mode  = None
    log       = Logger()

    # -- collect the list of files to be processed
    if os.path.isdir(path):
        entries = list(os.path.join(root, f)
                       for (root, dirs, files) in os.walk(path)
                       for f in files)
        entries.sort()
    else:
        entries = [ path ]
    if not entries:
        return

    # -- find a useable volume variable and analyse it
    filename = entries[0]
    log.writeln("slices(): looking for a volume variable in %s..." %
                os.path.basename(filename))
    var = find_variable(filename)
    if var is None :
        log.writeln("No appropriate volume data found.")
        return

    # -- determine the appropriate mask value
    mask_value = { numpy.uint8:  0xff,
                   numpy.uint16: 0xffff,
                   numpy.int32:  0x7fffffff,
                   numpy.float32: 1.0e30 }[var['dtype']]

    # -- initialize the slice set to be created
    slices = default_slice_set(var, 10, basename)
    r_or_s = 'REPLACE' if replace else 'SKIP'
    actions = list((r_or_s if n in existing else 'ADD')
                   for (s, n) in slices)
    slices = list(slices[i] + (actions[i],)
                  for i in range(len(slices))
                  if actions[i] != 'SKIP')

    if len(slices) == 0:
        log.writeln("No slices are to be made.")
        return
    elif dry_run:
        for (s, n, a) in slices:
            for sz in sizes:
                yield (make_image.make_dummy(n, 256, 256, sz),
                       sizeprefix(sz) + n,
                       a)
        return

    # -- initialize the histogram
    if var['dtype'] == numpy.float32:
        log.writeln("Determining the data range...")
        (minval, maxval) = data_range(var, entries)
        hist = Histogram(mask_value, minval, maxval)
    else:
        hist = Histogram(mask_value)

    # -- loop through files and copy data into slice arrays
    for filename in entries:
        log.writeln("Processing %s..." % os.path.basename(filename))
        for tmp in z_slices(var, filename):
            z, data = tmp[:2]
            if data is None:
                log.writeln(tmp[2] + " at z = %d" % z, LOGGER_WARNING)
            else:
                hist.update(data)
                for (s, n, a) in slices:
                    s.update(data, z)

    # -- analyse histogram to determine 'lo' and 'hi' values
    log.writeln("Analysing the histogram...")
    if name.startswith("tom"):
        # -- determine 0.1 and 99.9 percentile for contrast stretching
        lo = bottom_percentile(hist, 0.1)
        hi = top_percentile(hist, 0.1)
    else:
        lo = 0
        hi = hist.counts.size - 1

    # -- encode slices as PNG images
    log.writeln("Making the images...")
    for (s, n, a) in slices:
        for sz in sizes:
            yield (image_data(s, lo, hi, mask_value, info, sz),
                   sizeprefix(sz) + n,
                   a)

    # -- report success
    log.writeln("Slice image generation finished.")

if __name__ == "__main__":
    Logger().priority = LOGGER_INFO
    source = sys.argv[1]
    s = slices(source, info = { 'source': source }, sizes = (None, (80,80)))

    for (data, name, action) in s:
        fp = file(name, 'wb')
        fp.write(data)
        fp.close()
<commit_msg>no separate loop for dry run in slice() function<commit_after>#!/usr/bin/env python

"""
Produces slice images for Mango volume data stored in NetCDF files.

Currently supported data types are tomo, segmented and labels. If the
data is split into multiple files, those must sit in a common directory
and that directory given as the path argument.

Slices are taken through the center of the volume. If either of the
dimensions for a slice image would be less than 10, that slice is
suppressed. Image output is in .png format.

Typical usage:
    for (data, name, action) in slices(path):
        fp = file(name, 'wb')
        fp.write(data)
        fp.close()

(Requires Python 2.6 or higher.)
"""

import os, os.path, re, struct, sys
import numpy
import bz2

from logger import Logger, LOGGER_INFO, LOGGER_WARNING
import make_image
from nc3header import NC3File

class Histogram:
    """
    Maintains a frequency count for a series of numpy arrays. Entries
    equal to <mask_value> are counted separately. Negative values are
    ignored.
    """

    def __init__(self, mask_value = 0, minval = None, maxval = None):
        # -- save the mask value
        self.mask_value = mask_value
        # -- initialize the frequency count
        self.counts = numpy.array([], dtype = 'uint64')
        # -- initialize the count of masked and total non-negative entries
        self.total = 0
        self.masked = 0

        if maxval is not None:
            self.offset = minval
            self.binsize = (maxval - minval) * (1.0 - 1.0e-12) / 0x10000;
        else:
            self.offset = 0
            self.binsize = 1

    def update(self, slice):
        """
        Updates the frequency count with the data from the numpy array
        <slice>.
        """

        # -- process the new data
        tmp = (slice.flatten() - self.offset) / self.binsize
        flat = numpy.array(tmp, dtype = 'uint16')
        mask = (flat == self.mask_value) | (flat < 0) | (flat > 0xffff)
        new_masked = flat[mask].size
        new_counts = numpy.bincount(numpy.where(mask, 0, flat))

        # -- update the frequency count, resizing if necessary
        s = max(self.counts.size, new_counts.size)
        if s > self.counts.size:
            self.counts.resize(s)
        if s > new_counts.size:
            new_counts.resize(s)
        self.counts += new_counts

        # -- update the count of masked and total non-negative entries
        self.masked += new_masked
        self.total  += new_masked + int(new_counts.sum())

class Slice:
    def __init__(self, size, type, axis, pos):
        self.axis = axis.lower()
        self.pos  = pos
        self.slice_dims = {'x': (size[2], size[1]),
                           'y': (size[2], size[0]),
                           'z': (size[1], size[0]) }[self.axis]
        self.content = numpy.zeros(self.slice_dims, type)

    def update(self, z_slice, z_pos):
        """
        Updates this slices with data from the array <z_slice>, which
        is taken to be at z = <z_pos>.
        """

        if self.axis == 'x':
            self.content[z_pos, :] = z_slice[:, self.pos]
        elif self.axis == 'y':
            self.content[z_pos, :] = z_slice[self.pos, :]
        elif self.axis == 'z':
            if z_pos == self.pos:
                self.content[:, :] = z_slice[:, :]

def get_attribute(file, var, name):
    """
    Looks in the open NC3File <file> for the attribute <name>,
    first in the given variable <var> and then in the global
    attributes. Return the attribute's value if found, or None
    otherwise.
    """

    for attr in var.attributes:
        if attr.name == name:
            return attr.value
    for attr in file.attributes:
        if attr.name == name:
            return attr.value
    return None

def volume_variable(file, var):
    """
    Extracts information pertaining to the volume variable <var> within
    the open NC3File <file>. The name, shape, origin and data type -
    mapped to the corresponding numpy type - are stored.
    """

    # -- remember the name
    name = var.name

    # -- extract the variable's shape
    dims = var.dimensions
    size = (dims[2].value, dims[1].value, dims[0].value)
    zdim_total = get_attribute(file, var, 'zdim_total')
    if zdim_total is not None:
        size = (size[0], size[1], zdim_total[0])

    # -- determine the origin
    origin = get_attribute(file, var, 'coordinate_origin_xyz') or (0, 0, 0)

    # -- determine the data type
    (dtype, big_endian_type) = {
      'b': (numpy.uint8,  numpy.dtype(">u1")),
      'h': (numpy.uint16, numpy.dtype(">u2")),
      'i': (numpy.int32,  numpy.dtype(">i4")),
      'f': (numpy.float32, numpy.dtype(">f4")) }[var.python_type_code]

    return { 'name'  : name,
             'size'  : size,
             'origin': origin,
             'dtype' : dtype,
             'big_endian_type': big_endian_type }

def find_variable(path):
    """
    Looks for a volume variable to extract slice images from in the
    NetCDF file at location <path>. Returns an descriptor object of
    class VolumeVariable if something appropriate is found, or None
    otherwise.
    """

    # -- the default return value
    var = None

    # -- open the NetCDF file
    file = NC3File(path)

    # -- loop through all the variables in the file
    for v in file.variables:
        if (len(v.dimensions) == 3 and v.dimensions[0].value > 1 and
            v.python_type_code in 'bhif'
            ):
            var = volume_variable(file, v)
            break

    # -- close the file and return the result
    file.close()
    return var

def z_slices(variable, path):
    """
    A generator method that yields constant z slices corresponding
    to the variable <var> from the NetCDF file located at <path>.

    Each value produced is a pair containing the z coordinate of
    the slice and a two-dimensional numpy array containing the
    extracted data.

    Usage example:

        for (z, data) in z_slices(var, path):
            ... # do something with data
    """

    file = NC3File(path)

    for var in file.variables:
        if var.name == variable['name']:
            break
    else:
        return

    if volume_variable(file, var) != variable:
        raise RuntimeError("variable mismatch between files")

    (x, y, z) = variable['size']

    z_range = get_attribute(file, var, 'zdim_range')
    if z_range is None:
        z_range = range(0, z)
    else:
        z_range = range(z_range[0], z_range[1] + 1)

    bytes_per_slice = x * y * var.element_size
    offset = var.data_start

    file.close()

    if path.endswith('.bz2'):
        fp = bz2.BZ2File(path, 'r', 1024 * 1024)
    else:
        fp = open(path, "rb")
    fp.seek(var.data_start)
    for z in z_range:
        buffer = fp.read(bytes_per_slice)
        if len(buffer) < bytes_per_slice:
            yield (z, None, "insufficient data")
            break
        data = numpy.fromstring(buffer, variable['big_endian_type'])
        data.shape = (y, x)
        yield (z, data)
    fp.close()

def bottom_percentile(histogram, p):
    """
    Returns the smallest number i such that at least <p> percent of
    the non-masked entries counted so far have value i or less.
    """
    threshold = p * (histogram.total - histogram.masked) / 100.0
    count = 0
    for i in xrange(histogram.counts.size):
        count += int(histogram.counts[i])
        if count >= threshold:
            return histogram.offset + i * histogram.binsize

def top_percentile(histogram, p):
    """
    Returns the largest number i such that at least <p> percent of
    the non-masked entries counted so far have value i or more.
    """
    threshold = p * (histogram.total - histogram.masked) / 100.0
    count = 0
    for i in xrange(histogram.counts.size-1, -1, -1):
        count += int(histogram.counts[i])
        if count >= threshold:
            return histogram.offset + i * histogram.binsize

def default_slice_set(var, delta, basename):
    """
    Creates a default list of empty slice instances, paired with
    associated file names, based on the shape of the volume variable
    <var>. For each axis, a slice centered at that axis is created,
    provided that the extend of the slice in both directions would be at
    least <delta>.
    """
    pos = list((x - 1) / 2 for x in var['size'])
    size = var['size']
    dtype = var['dtype']
    origin = var['origin']

    def slice_and_name(axis, pos, offset):
        s = Slice(size, dtype, axis, pos)
        n = "slice%c%d_%s.png" % (axis.upper(), pos + offset, basename)
        return (s, n)

    slices = []
    if size[1] > delta and size[2] > delta:
        slices.append(slice_and_name('x', pos[0], origin[0]))
    if size[0] > delta and size[2] > delta:
        slices.append(slice_and_name('y', pos[1], origin[1]))
    if size[0] > delta and size[1] > delta:
        slices.append(slice_and_name('z', pos[2], origin[2]))

    return slices

def image_data(slice, lo, hi, mask_val, info, thumb_size = None):
    """
    Converts a slice into an image and returns it as a PNG encoded
    string. The values <lo> and <hi> specify the range of relevant
    data. Entries equal to <mask_val> are treated as masked.

    An appropriate encoding mode is determined from the data type
    and value range. Details of the actual encoding are deferred to
    the function 'make_image' in the package of the same name.
    """

    # -- determine the encoding mode
    content = slice.content
    if content.dtype == numpy.uint8:
        if hi <= 1:
            img_mode = make_image.BLACK_AND_WHITE
        else:
            img_mode = make_image.COLOR_CODED_FIXED
    elif content.dtype == numpy.uint16:
        img_mode = make_image.GRAYSCALE
    elif content.dtype == numpy.int32:
        img_mode = make_image.COLOR_CODED
    elif content.dtype == numpy.float32:
        img_mode = make_image.GRAYSCALE
    else:
        raise Exception("unexpected array type: %s" % content.dtype)

    # -- generate slice-specific info
    myinfo = info.copy()
    myinfo.update({ 'slice-axis': slice.axis, 'slice-pos': slice.pos })

    # -- generate and return the data
    return make_image.make_image(content, lo, hi, mask_val, img_mode,
                                 thumb_size, myinfo)

def data_range(var, entries):
    minval = maxval = None

    for filename in entries:
        for tmp in z_slices(var, filename):
            z, data = tmp[:2]
            if data is not None:
                lo = numpy.min(data)
                hi = numpy.max(data)
                if minval is None or lo < minval:
                    minval = lo
                if maxval is None or hi > maxval:
                    maxval = hi

    return (minval, maxval)

def slices(path,
           existing = [],
           replace = False,
           dry_run = False,
           sizes = (None,),
           info = {}):
    """
    A generator which extracts slice images from a Mango volume data set
    stored in a collection of NetCDF files.

    The parameter <path> specifies a single NetCDF file or a directory
    containing a single volume split into several NetCDF files.

    Basic usage:
        for (data, name, action) in slices(path):
            fp = file(name, 'wb')
            fp.write(data)
            fp.close()
    """

    path      = re.sub("/$", "", path)
    name      = re.sub("[._]nc$", "", os.path.basename(path))
    basename  = re.sub("^tomo", "tom", re.sub("^segmented", "seg", name))
    img_mode  = None
    log       = Logger()

    # -- collect the list of files to be processed
    if os.path.isdir(path):
        entries = list(os.path.join(root, f)
                       for (root, dirs, files) in os.walk(path)
                       for f in files)
        entries.sort()
    else:
        entries = [ path ]
    if not entries:
        return

    # -- find a useable volume variable and analyse it
    filename = entries[0]
    log.writeln("slices(): looking for a volume variable in %s..." %
                os.path.basename(filename))
    var = find_variable(filename)
    if var is None :
        log.writeln("No appropriate volume data found.")
        return

    # -- determine the appropriate mask value
    mask_value = { numpy.uint8:  0xff,
                   numpy.uint16: 0xffff,
                   numpy.int32:  0x7fffffff,
                   numpy.float32: 1.0e30 }[var['dtype']]

    # -- initialize the slice set to be created
    slices = default_slice_set(var, 10, basename)
    r_or_s = 'REPLACE' if replace else 'SKIP'
    actions = list((r_or_s if n in existing else 'ADD')
                   for (s, n) in slices)
    slices = list(slices[i] + (actions[i],)
                  for i in range(len(slices))
                  if actions[i] != 'SKIP')

    if len(slices) == 0:
        log.writeln("No slices are to be made.")
        return

    if not dry_run:
        # -- initialize the histogram
        if var['dtype'] == numpy.float32:
            log.writeln("Determining the data range...")
            (minval, maxval) = data_range(var, entries)
            hist = Histogram(mask_value, minval, maxval)
        else:
            hist = Histogram(mask_value)

        # -- loop through files and copy data into slice arrays
        for filename in entries:
            log.writeln("Processing %s..." % os.path.basename(filename))
            for tmp in z_slices(var, filename):
                z, data = tmp[:2]
                if data is None:
                    log.writeln(tmp[2] + " at z = %d" % z, LOGGER_WARNING)
                else:
                    hist.update(data)
                    for (s, n, a) in slices:
                        s.update(data, z)

        # -- analyse histogram to determine 'lo' and 'hi' values
        log.writeln("Analysing the histogram...")
        if name.startswith("tom"):
            # -- determine 0.1 and 99.9 percentile for contrast stretching
            lo = bottom_percentile(hist, 0.1)
            hi = top_percentile(hist, 0.1)
        else:
            lo = 0
            hi = hist.counts.size - 1

    # -- encode slices as PNG images
    log.writeln("Making the images...")
    for (s, n, a) in slices:
        for sz in sizes:
            if dry_run:
                data = make_image.make_dummy(n, 256, 256, sz)
            else:
                data = image_data(s, lo, hi, mask_value, info, sz)
            prefix = ("__%sx%s__" % sz) if sz else ""
            yield (data, prefix + n, a)

    # -- report success
    log.writeln("Slice image generation finished.")

if __name__ == "__main__":
    Logger().priority = LOGGER_INFO
    source = sys.argv[1]
    s = slices(source, info = { 'source': source }, sizes = (None, (80,80)))

    for (data, name, action) in s:
        fp = file(name, 'wb')
        fp.write(data)
        fp.close()
<|endoftext|>


<commit_before>import datetime
from sqlalchemy.orm.exc import NoResultFound

from model import Log, LogPage

def get_or_create_log(redis, mysql, chat):
    # Find existing log or create a new one.
    try:
        log = mysql.query(Log).filter(Log.url==chat).one()
        try:
            latest_page_query = mysql.query(LogPage).filter(LogPage.log_id==log.id).order_by(LogPage.number.asc())
            latest_page = latest_page_query[-1]
            # XXX Is IndexError the right exception?
        except IndexError:
            latest_page = new_page(mysql, log)
    except NoResultFound:
        log = Log(url=chat)
        mysql.add(log)
        mysql.flush()
        latest_page = new_page(mysql, log)
    return log, latest_page

def new_page(mysql, log, last=0):
    new_page_number = last+1
    latest_page = LogPage(log_id=log.id, number=new_page_number, content=u'')
    mysql.add(latest_page)
    # XXX COMMIT HERE?
    log.page_count = new_page_number
    return latest_page

def archive_chat(redis, mysql, chat, backlog=0):
    log, latest_page = get_or_create_log(redis, mysql, chat)
    # XXX MAKE REALLY REALLY REALLY GODDAMN SURE THIS WORKS WITH MINUS NUMBERS
    # XXX FOR GOD'S SAKE
    lines = redis.lrange('chat.'+chat, 0, -1-backlog)
    for line in lines:
        # Create a new page if the line won't fit on this one.
        #if len(latest_page.content.encode('utf8'))+len(line)>65535:
        if len(latest_page.content.encode('utf8'))+len(line)>65535:
            print "creating a new page"
            latest_page = latest_page = new_page(mysql, log, latest_page.number)
            print "page "+str(latest_page.number)
        latest_page.content += unicode(line, encoding='utf8')+'\n'
    log.time_saved = datetime.datetime.now()
    mysql.commit()
    # Don't delete from redis until we've successfully committed.
    redis.ltrim('chat.'+chat, len(lines), -1)
    return log.id

def delete_chat_session(redis, chat, session_id):

    counter = redis.hget('session.'+session_id+'.meta.'+chat, 'counter')
    pipe = redis.pipeline()
    pipe.hdel('chat.'+chat+'.counters', counter)
    pipe.delete('session.'+session_id+'.chat.'+chat)
    pipe.delete('session.'+session_id+'.meta.'+chat)
    pipe.srem('session.'+session_id+'.chats', chat)
    pipe.zrem('chat-sessions', chat+'/'+session_id)
    pipe.execute()

def delete_chat(redis, mysql, chat):

    # XXX PIPELINE THIS???

    # If it's been saved before, save it again before deleting.
    if redis.hget('chat.'+chat+'.meta', 'type')!='unsaved':
        archive_chat(redis, mysql, chat, 0)

    # Delete metadata first because it's used to check whether a chat exists.
    redis.delete('chat.'+chat+'.meta')

    redis.delete('chat.'+chat+'.online')
    redis.delete('chat.'+chat+'.idle')
    redis.delete('chat.'+chat+'.characters')

    for session_id in redis.hvals('chat.'+chat+'.counters'):
        redis.srem('session.'+session_id+'.chats', chat)
        redis.delete('session.'+session_id+'.chat.'+chat)
        redis.delete('session.'+session_id+'.meta.'+chat)
        redis.zrem('chat-sessions', chat+'/'+session_id)

    redis.delete('chat.'+chat+'.counters')
    redis.delete('chat.'+chat)
    redis.zrem('delete-queue', chat)

def delete_session(redis, session_id):

    for chat in redis.smembers('session.'+session_id+'.chats'):
        delete_chat_session(redis, chat, session_id)

    redis.delete('session.'+session_id)
    redis.delete('session.'+session_id+'.meta')
    redis.zrem('all-sessions', session_id)

<commit_msg>Chat archive ignores invalid utf8 characters in case of truncation errors<commit_after>import datetime
from sqlalchemy.orm.exc import NoResultFound

from model import Log, LogPage

def get_or_create_log(redis, mysql, chat):
    # Find existing log or create a new one.
    try:
        log = mysql.query(Log).filter(Log.url==chat).one()
        try:
            latest_page_query = mysql.query(LogPage).filter(LogPage.log_id==log.id).order_by(LogPage.number.asc())
            latest_page = latest_page_query[-1]
            # XXX Is IndexError the right exception?
        except IndexError:
            latest_page = new_page(mysql, log)
    except NoResultFound:
        log = Log(url=chat)
        mysql.add(log)
        mysql.flush()
        latest_page = new_page(mysql, log)
    return log, latest_page

def new_page(mysql, log, last=0):
    new_page_number = last+1
    latest_page = LogPage(log_id=log.id, number=new_page_number, content=u'')
    mysql.add(latest_page)
    # XXX COMMIT HERE?
    log.page_count = new_page_number
    return latest_page

def archive_chat(redis, mysql, chat, backlog=0):
    log, latest_page = get_or_create_log(redis, mysql, chat)
    # XXX MAKE REALLY REALLY REALLY GODDAMN SURE THIS WORKS WITH MINUS NUMBERS
    # XXX FOR GOD'S SAKE
    lines = redis.lrange('chat.'+chat, 0, -1-backlog)
    for line in lines:
        # Create a new page if the line won't fit on this one.
        if len(latest_page.content.encode('utf8'))+len(line)>65535:
            print "creating a new page"
            latest_page = latest_page = new_page(mysql, log, latest_page.number)
            print "page "+str(latest_page.number)

        # Decode the line on ignore strip out invalid utf8 characters
        line = line.decode('utf8', 'ignore')

        # Append the line to the new page
        latest_page.content += line + '\n'

    log.time_saved = datetime.datetime.now()
    mysql.commit()
    # Don't delete from redis until we've successfully committed.
    redis.ltrim('chat.'+chat, len(lines), -1)
    return log.id

def delete_chat_session(redis, chat, session_id):

    counter = redis.hget('session.'+session_id+'.meta.'+chat, 'counter')
    pipe = redis.pipeline()
    pipe.hdel('chat.'+chat+'.counters', counter)
    pipe.delete('session.'+session_id+'.chat.'+chat)
    pipe.delete('session.'+session_id+'.meta.'+chat)
    pipe.srem('session.'+session_id+'.chats', chat)
    pipe.zrem('chat-sessions', chat+'/'+session_id)
    pipe.execute()

def delete_chat(redis, mysql, chat):

    # XXX PIPELINE THIS???

    # If it's been saved before, save it again before deleting.
    if redis.hget('chat.'+chat+'.meta', 'type')!='unsaved':
        archive_chat(redis, mysql, chat, 0)

    # Delete metadata first because it's used to check whether a chat exists.
    redis.delete('chat.'+chat+'.meta')

    redis.delete('chat.'+chat+'.online')
    redis.delete('chat.'+chat+'.idle')
    redis.delete('chat.'+chat+'.characters')

    for session_id in redis.hvals('chat.'+chat+'.counters'):
        redis.srem('session.'+session_id+'.chats', chat)
        redis.delete('session.'+session_id+'.chat.'+chat)
        redis.delete('session.'+session_id+'.meta.'+chat)
        redis.zrem('chat-sessions', chat+'/'+session_id)

    redis.delete('chat.'+chat+'.counters')
    redis.delete('chat.'+chat)
    redis.zrem('delete-queue', chat)

def delete_session(redis, session_id):

    for chat in redis.smembers('session.'+session_id+'.chats'):
        delete_chat_session(redis, chat, session_id)

    redis.delete('session.'+session_id)
    redis.delete('session.'+session_id+'.meta')
    redis.zrem('all-sessions', session_id)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright (c) Vispy Development Team. All Rights Reserved.
# Distributed under the (new) BSD License. See LICENSE.txt for more info.

from __future__ import division

from .linear import STTransform, NullTransform
from .chain import ChainTransform
from ._util import TransformCache
from ...util.event import EventEmitter

class TransformSystem(object):
    """ TransformSystem encapsulates information about the coordinate
    systems needed to draw a Visual.

    Visual rendering operates in six coordinate systems:

    * **Visual** - arbitrary local coordinate frame of the visual. Vertex
      buffers used by the visual are usually specified in this coordinate
      system.

    * **Scene** - This is an isometric coordinate system used mainly for
      lighting calculations.

    * **Document** - This coordinate system has units of _logical_ pixels, and
      should usually represent the pixel coordinates of the canvas being drawn
      to. Visuals use this coordinate system to make measurements for font
      size, line width, and in general anything that is specified in physical
      units (px, pt, mm, in, etc.). In most circumstances, this is exactly the
      same as the canvas coordinate system.

    * **Canvas** - This coordinate system represents the logical pixel
      coordinates of the canvas. It has its origin in the top-left corner of
      the canvas, and is typically the coordinate system that mouse and touch
      events are reported in. Note that, by convention, _logical_ pixels
      are not necessarily the same size as the _physical_ pixels in the
      framebuffer that is being rendered to.

    * **Framebuffer** - The buffer coordinate system has units of _physical_
      pixels, and should usually represent the coordinates of the current
      framebuffer (on the canvas or an FBO) being rendered to. Visuals use this
      coordinate system primarily for antialiasing calculations. It is also the
      coorinate system used by glFragCoord. In most cases,
      this will have the same scale as the document and canvas coordinate
      systems because the active framebuffer is the
      back buffer of the canvas, and the canvas will have _logical_ and
      _physical_ pixels of the same size. However, the scale may be different
      in the case of high-resolution displays, or when rendering to an
      off-screen framebuffer with different scaling or boundaries than the
      canvas.

    * **Render** - This coordinate system is the obligatory system for
      vertices returned by a vertex shader. It has coordinates (-1, -1) to
      (1, 1) across the current glViewport. In OpenGL terminology, this is
      called clip coordinates.

    Parameters
    ----------

    canvas : Canvas
        The canvas being drawn to.
    dpi : float
        The dot-per-inch resolution of the document coordinate system. By
        default this is set to the resolution of the canvas.

    Notes
    -----

    By default, TransformSystems are configured such that the document
    coordinate system matches the logical pixels of the canvas,

    Examples
    --------

    1. To convert local vertex coordinates to normalized device coordinates in
    the vertex shader, we first need a vertex shader that supports configurable
    transformations::

        vec4 a_position;
        void main() {
            gl_Position = $transform(a_position);
        }

    Next, we supply the complete chain of transforms when drawing the visual:

        def draw(tr_sys):
            tr = tr_sys.get_full_transform()
            self.program['transform'] = tr.shader_map()
            self.program['a_position'] = self.vertex_buffer
            self.program.draw('triangles')

    2. Draw a line whose width is given in mm. To start, we need normal vectors
    for each vertex, which tell us the direction the vertex should move in
    order to set the line width::

        vec4 a_position;
        vec4 a_normal;
        float u_line_width;
        float u_dpi;
        void main() {
            // map vertex position and normal vector to the document cs
            vec4 doc_pos = $visual_to_doc(a_position);
            vec4 doc_normal = $visual_to_doc(a_position + a_normal) - doc_pos;

            // Use DPI to convert mm line width to logical pixels
            float px_width = (u_line_width / 25.4) * dpi;

            // expand by line width
            doc_pos += normalize(doc_normal) * px_width;

            // finally, map the remainder of the way to normalized device
            // coordinates.
            gl_Position = $doc_to_render(a_position);
        }

    In this case, we need to access
    the transforms independently, so ``get_full_transform()`` is not useful
    here::

        def draw(tr_sys):
            # Send two parts of the full transform separately
            self.program['visual_to_doc'] = tr_sys.visual_to_doc.shader_map()
            doc_to_render = (tr_sys.framebuffer_transform *
                             tr_sys.document_transform)
            self.program['visual_to_doc'] = doc_to_render.shader_map()

            self.program['u_line_width'] = self.line_width
            self.program['u_dpi'] = tr_sys.dpi
            self.program['a_position'] = self.vertex_buffer
            self.program['a_normal'] = self.normal_buffer
            self.program.draw('triangles')

    3. Draw a triangle with antialiasing at the edge.

    4. Using inverse transforms in the fragment shader
    """

    def __init__(self, canvas=None, dpi=None):
        self.changed = EventEmitter(source=self, type='transform_changed')
        self._canvas = None
        self._fbo_bounds = None
        self.canvas = canvas
        self._cache = TransformCache()
        self._dpi = dpi

        # Assign a ChainTransform for each step. This allows us to always
        # return the same transform objects regardless of how the user
        # configures the system.
        self._visual_transform = ChainTransform([NullTransform()])
        self._scene_transform = ChainTransform([NullTransform()])
        self._document_transform = ChainTransform([NullTransform()])
        self._canvas_transform = ChainTransform([STTransform(),
                                                 STTransform()])
        self._framebuffer_transform = ChainTransform([STTransform()])

        for tr in (self._visual_transform, self._scene_transform,
                   self._document_transform, self._canvas_transform,
                   self._framebuffer_transform):
            tr.changed.connect(self.changed)

    def configure(self, viewport=None, fbo_size=None, fbo_rect=None,
                  canvas=None):
        """Automatically configure the TransformSystem:

        * canvas_transform maps from the Canvas logical pixel
          coordinate system to the framebuffer coordinate system, taking into
          account the logical/physical pixel scale factor, current FBO
          position, and y-axis inversion.
        * framebuffer_transform maps from the current GL viewport on the
          framebuffer coordinate system to clip coordinates (-1 to 1).

        Parameters
        ==========
        viewport : tuple or None
            The GL viewport rectangle (x, y, w, h). If None, then it
            is assumed to cover the entire canvas.
        fbo_size : tuple or None
            The size of the active FBO. If None, then it is assumed to have the
            same size as the canvas's framebuffer.
        fbo_rect : tuple or None
            The position and size (x, y, w, h) of the FBO in the coordinate
            system of the canvas's framebuffer. If None, then the bounds are
            assumed to cover the entire active framebuffer.
        canvas : Canvas instance
            Optionally set the canvas for this TransformSystem. See the
            `canvas` property.
        """
        # TODO: check that d2f and f2r transforms still contain a single
        # STTransform (if the user has modified these, then auto-config should
        # either fail or replace the transforms)
        if canvas is not None:
            self.canvas = canvas
        canvas = self._canvas
        if canvas is None:
            raise RuntimeError("No canvas assigned to this TransformSystem.")

        # By default, this should invert the y axis--canvas origin is in top
        # left, whereas framebuffer origin is in bottom left.
        map_from = [(0, 0), canvas.size]
        map_to = [(0, canvas.physical_size[1]), (canvas.physical_size[0], 0)]
        self._canvas_transform.transforms[1].set_mapping(map_from, map_to)

        if fbo_rect is None:
            self._canvas_transform.transforms[0].scale = (1, 1, 1)
            self._canvas_transform.transforms[0].translate = (0, 0, 0)
        else:
            # Map into FBO coordinates
            map_from = [(fbo_rect[0], fbo_rect[1]),
                        (fbo_rect[0] + fbo_rect[2], fbo_rect[1] + fbo_rect[3])]
            map_to = [(0, 0), fbo_size]
            self._canvas_transform.transforms[0].set_mapping(map_from,  map_to)

        if viewport is None:
            if fbo_size is None:
                # viewport covers entire canvas
                map_from = [(0, 0), canvas.physical_size]
            else:
                # viewport covers entire FBO
                map_from = [(0, 0), fbo_size]
        else:
            map_from = [viewport[:2],
                        (viewport[0] + viewport[2], viewport[1] + viewport[3])]
        map_to = [(-1, -1), (1, 1)]
        self._framebuffer_transform.transforms[0].set_mapping(map_from, map_to)

    @property
    def canvas(self):
        """ The Canvas being drawn to.
        """
        return self._canvas

    @canvas.setter
    def canvas(self, canvas):
        self._canvas = canvas

    @property
    def dpi(self):
        """ Physical resolution of the document coordinate system (dots per
        inch).
        """
        if self._dpi is None:
            if self._canvas is None:
                return None
            else:
                return self.canvas.dpi
        else:
            return self._dpi

    @dpi.setter
    def dpi(self, dpi):
        assert dpi > 0
        self._dpi = dpi

    @property
    def visual_transform(self):
        """ Transform mapping from visual local coordinate frame to scene
        coordinate frame.
        """
        return self._visual_transform

    @visual_transform.setter
    def visual_transform(self, tr):
        self._visual_transform.transforms = tr

    @property
    def scene_transform(self):
        """ Transform mapping from scene coordinate frame to document
        coordinate frame.
        """
        return self._scene_transform

    @scene_transform.setter
    def scene_transform(self, tr):
        self._scene_transform.transforms = tr

    @property
    def document_transform(self):
        """ Transform mapping from document coordinate frame to the framebuffer
        (physical pixel) coordinate frame.
        """
        return self._document_transform

    @document_transform.setter
    def document_transform(self, tr):
        self._document_transform.transforms = tr

    @property
    def canvas_transform(self):
        """ Transform mapping from canvas coordinate frame to framebuffer
        coordinate frame.
        """
        return self._canvas_transform

    @canvas_transform.setter
    def canvas_transform(self, tr):
        self._canvas_transform.transforms = tr

    @property
    def framebuffer_transform(self):
        """ Transform mapping from pixel coordinate frame to rendering
        coordinate frame.
        """
        return self._framebuffer_transform

    @framebuffer_transform.setter
    def framebuffer_transform(self, tr):
        self._framebuffer_transform.transforms = tr

    def get_transform(self, map_from='visual', map_to='render'):
        """Return a transform mapping between any two coordinate systems.

        Parameters
        ----------
        map_from : str
            The starting coordinate system to map from. Must be one of: visual,
            scene, document, canvas, framebuffer, or render.
        map_to : str
            The ending coordinate system to map to. Must be one of: visual,
            scene, document, canvas, framebuffer, or render.
        """
        tr = ['visual', 'scene', 'document', 'canvas', 'framebuffer', 'render']
        ifrom = tr.index(map_from)
        ito = tr.index(map_to)

        if ifrom < ito:
            trs = [getattr(self, '_' + t + '_transform')
                   for t in tr[ifrom:ito]][::-1]
        else:
            trs = [getattr(self, '_' + t + '_transform').inverse
                   for t in tr[ito:ifrom]]
        return self._cache.get(trs)

    @property
    def pixel_scale(self):
        tr = self._canvas_transform
        return (tr.map((1, 0)) - tr.map((0, 0)))[0]
<commit_msg>Ensure canvas and framebuffer transforms only update when a change occurs<commit_after># -*- coding: utf-8 -*-
# Copyright (c) Vispy Development Team. All Rights Reserved.
# Distributed under the (new) BSD License. See LICENSE.txt for more info.

from __future__ import division

from .linear import STTransform, NullTransform
from .chain import ChainTransform
from ._util import TransformCache
from ...util.event import EventEmitter

import numpy as np

class TransformSystem(object):
    """ TransformSystem encapsulates information about the coordinate
    systems needed to draw a Visual.

    Visual rendering operates in six coordinate systems:

    * **Visual** - arbitrary local coordinate frame of the visual. Vertex
      buffers used by the visual are usually specified in this coordinate
      system.

    * **Scene** - This is an isometric coordinate system used mainly for
      lighting calculations.

    * **Document** - This coordinate system has units of _logical_ pixels, and
      should usually represent the pixel coordinates of the canvas being drawn
      to. Visuals use this coordinate system to make measurements for font
      size, line width, and in general anything that is specified in physical
      units (px, pt, mm, in, etc.). In most circumstances, this is exactly the
      same as the canvas coordinate system.

    * **Canvas** - This coordinate system represents the logical pixel
      coordinates of the canvas. It has its origin in the top-left corner of
      the canvas, and is typically the coordinate system that mouse and touch
      events are reported in. Note that, by convention, _logical_ pixels
      are not necessarily the same size as the _physical_ pixels in the
      framebuffer that is being rendered to.

    * **Framebuffer** - The buffer coordinate system has units of _physical_
      pixels, and should usually represent the coordinates of the current
      framebuffer (on the canvas or an FBO) being rendered to. Visuals use this
      coordinate system primarily for antialiasing calculations. It is also the
      coorinate system used by glFragCoord. In most cases,
      this will have the same scale as the document and canvas coordinate
      systems because the active framebuffer is the
      back buffer of the canvas, and the canvas will have _logical_ and
      _physical_ pixels of the same size. However, the scale may be different
      in the case of high-resolution displays, or when rendering to an
      off-screen framebuffer with different scaling or boundaries than the
      canvas.

    * **Render** - This coordinate system is the obligatory system for
      vertices returned by a vertex shader. It has coordinates (-1, -1) to
      (1, 1) across the current glViewport. In OpenGL terminology, this is
      called clip coordinates.

    Parameters
    ----------

    canvas : Canvas
        The canvas being drawn to.
    dpi : float
        The dot-per-inch resolution of the document coordinate system. By
        default this is set to the resolution of the canvas.

    Notes
    -----

    By default, TransformSystems are configured such that the document
    coordinate system matches the logical pixels of the canvas,

    Examples
    --------

    1. To convert local vertex coordinates to normalized device coordinates in
    the vertex shader, we first need a vertex shader that supports configurable
    transformations::

        vec4 a_position;
        void main() {
            gl_Position = $transform(a_position);
        }

    Next, we supply the complete chain of transforms when drawing the visual:

        def draw(tr_sys):
            tr = tr_sys.get_full_transform()
            self.program['transform'] = tr.shader_map()
            self.program['a_position'] = self.vertex_buffer
            self.program.draw('triangles')

    2. Draw a line whose width is given in mm. To start, we need normal vectors
    for each vertex, which tell us the direction the vertex should move in
    order to set the line width::

        vec4 a_position;
        vec4 a_normal;
        float u_line_width;
        float u_dpi;
        void main() {
            // map vertex position and normal vector to the document cs
            vec4 doc_pos = $visual_to_doc(a_position);
            vec4 doc_normal = $visual_to_doc(a_position + a_normal) - doc_pos;

            // Use DPI to convert mm line width to logical pixels
            float px_width = (u_line_width / 25.4) * dpi;

            // expand by line width
            doc_pos += normalize(doc_normal) * px_width;

            // finally, map the remainder of the way to normalized device
            // coordinates.
            gl_Position = $doc_to_render(a_position);
        }

    In this case, we need to access
    the transforms independently, so ``get_full_transform()`` is not useful
    here::

        def draw(tr_sys):
            # Send two parts of the full transform separately
            self.program['visual_to_doc'] = tr_sys.visual_to_doc.shader_map()
            doc_to_render = (tr_sys.framebuffer_transform *
                             tr_sys.document_transform)
            self.program['visual_to_doc'] = doc_to_render.shader_map()

            self.program['u_line_width'] = self.line_width
            self.program['u_dpi'] = tr_sys.dpi
            self.program['a_position'] = self.vertex_buffer
            self.program['a_normal'] = self.normal_buffer
            self.program.draw('triangles')

    3. Draw a triangle with antialiasing at the edge.

    4. Using inverse transforms in the fragment shader
    """

    def __init__(self, canvas=None, dpi=None):
        self.changed = EventEmitter(source=self, type='transform_changed')
        self._canvas = None
        self._fbo_bounds = None
        self.canvas = canvas
        self._cache = TransformCache()
        self._dpi = dpi
        self._mappings = {'ct0': None, 'ct1': None, 'ft0': None}

        # Assign a ChainTransform for each step. This allows us to always
        # return the same transform objects regardless of how the user
        # configures the system.
        self._visual_transform = ChainTransform([NullTransform()])
        self._scene_transform = ChainTransform([NullTransform()])
        self._document_transform = ChainTransform([NullTransform()])
        self._canvas_transform = ChainTransform([STTransform(),
                                                 STTransform()])
        self._framebuffer_transform = ChainTransform([STTransform()])

        for tr in (self._visual_transform, self._scene_transform,
                   self._document_transform, self._canvas_transform,
                   self._framebuffer_transform):
            tr.changed.connect(self.changed)

    def _update_if_maps_changed(self, transform, map_key, new_maps):
        """Helper to store and check current (from, to) maps against new
        ones being provided. The new mappings are only applied if a change
        has occurred (and also stored in the current mappings).
        """
        if self._mappings[map_key] is None:
            self._mappings[map_key] = new_maps
            transform.set_mapping(new_maps[0], new_maps[1])
        else:
            if np.any(self._mappings[map_key] != new_maps):
                self._mappings[map_key] = new_maps
                transform.set_mapping(new_maps[0], new_maps[1])

    def configure(self, viewport=None, fbo_size=None, fbo_rect=None,
                  canvas=None):
        """Automatically configure the TransformSystem:

        * canvas_transform maps from the Canvas logical pixel
          coordinate system to the framebuffer coordinate system, taking into
          account the logical/physical pixel scale factor, current FBO
          position, and y-axis inversion.
        * framebuffer_transform maps from the current GL viewport on the
          framebuffer coordinate system to clip coordinates (-1 to 1).

        Parameters
        ==========
        viewport : tuple or None
            The GL viewport rectangle (x, y, w, h). If None, then it
            is assumed to cover the entire canvas.
        fbo_size : tuple or None
            The size of the active FBO. If None, then it is assumed to have the
            same size as the canvas's framebuffer.
        fbo_rect : tuple or None
            The position and size (x, y, w, h) of the FBO in the coordinate
            system of the canvas's framebuffer. If None, then the bounds are
            assumed to cover the entire active framebuffer.
        canvas : Canvas instance
            Optionally set the canvas for this TransformSystem. See the
            `canvas` property.
        """
        # TODO: check that d2f and f2r transforms still contain a single
        # STTransform (if the user has modified these, then auto-config should
        # either fail or replace the transforms)
        if canvas is not None:
            self.canvas = canvas
        canvas = self._canvas
        if canvas is None:
            raise RuntimeError("No canvas assigned to this TransformSystem.")

        # By default, this should invert the y axis--canvas origin is in top
        # left, whereas framebuffer origin is in bottom left.
        map_from = [(0, 0), canvas.size]
        map_to = [(0, canvas.physical_size[1]), (canvas.physical_size[0], 0)]
        self._update_if_maps_changed(self._canvas_transform.transforms[1],
                                     'ct1', np.array((map_from, map_to)))
        if fbo_rect is None:
            self._canvas_transform.transforms[0].scale = (1, 1, 1)
            self._canvas_transform.transforms[0].translate = (0, 0, 0)
        else:
            # Map into FBO coordinates
            map_from = [(fbo_rect[0], fbo_rect[1]),
                        (fbo_rect[0] + fbo_rect[2], fbo_rect[1] + fbo_rect[3])]
            map_to = [(0, 0), fbo_size]
            self._update_if_maps_changed(self._canvas_transform.transforms[0],
                                         'ct0', np.array((map_from, map_to)))
        if viewport is None:
            if fbo_size is None:
                # viewport covers entire canvas
                map_from = [(0, 0), canvas.physical_size]
            else:
                # viewport covers entire FBO
                map_from = [(0, 0), fbo_size]
        else:
            map_from = [viewport[:2],
                        (viewport[0] + viewport[2], viewport[1] + viewport[3])]
        map_to = [(-1, -1), (1, 1)]
        self._update_if_maps_changed(self._framebuffer_transform.transforms[0],
                                     'ft0', np.array((map_from, map_to)))

    @property
    def canvas(self):
        """ The Canvas being drawn to.
        """
        return self._canvas

    @canvas.setter
    def canvas(self, canvas):
        self._canvas = canvas

    @property
    def dpi(self):
        """ Physical resolution of the document coordinate system (dots per
        inch).
        """
        if self._dpi is None:
            if self._canvas is None:
                return None
            else:
                return self.canvas.dpi
        else:
            return self._dpi

    @dpi.setter
    def dpi(self, dpi):
        assert dpi > 0
        self._dpi = dpi

    @property
    def visual_transform(self):
        """ Transform mapping from visual local coordinate frame to scene
        coordinate frame.
        """
        return self._visual_transform

    @visual_transform.setter
    def visual_transform(self, tr):
        self._visual_transform.transforms = tr

    @property
    def scene_transform(self):
        """ Transform mapping from scene coordinate frame to document
        coordinate frame.
        """
        return self._scene_transform

    @scene_transform.setter
    def scene_transform(self, tr):
        self._scene_transform.transforms = tr

    @property
    def document_transform(self):
        """ Transform mapping from document coordinate frame to the framebuffer
        (physical pixel) coordinate frame.
        """
        return self._document_transform

    @document_transform.setter
    def document_transform(self, tr):
        self._document_transform.transforms = tr

    @property
    def canvas_transform(self):
        """ Transform mapping from canvas coordinate frame to framebuffer
        coordinate frame.
        """
        return self._canvas_transform

    @canvas_transform.setter
    def canvas_transform(self, tr):
        self._canvas_transform.transforms = tr

    @property
    def framebuffer_transform(self):
        """ Transform mapping from pixel coordinate frame to rendering
        coordinate frame.
        """
        return self._framebuffer_transform

    @framebuffer_transform.setter
    def framebuffer_transform(self, tr):
        self._framebuffer_transform.transforms = tr

    def get_transform(self, map_from='visual', map_to='render'):
        """Return a transform mapping between any two coordinate systems.

        Parameters
        ----------
        map_from : str
            The starting coordinate system to map from. Must be one of: visual,
            scene, document, canvas, framebuffer, or render.
        map_to : str
            The ending coordinate system to map to. Must be one of: visual,
            scene, document, canvas, framebuffer, or render.
        """
        tr = ['visual', 'scene', 'document', 'canvas', 'framebuffer', 'render']
        ifrom = tr.index(map_from)
        ito = tr.index(map_to)

        if ifrom < ito:
            trs = [getattr(self, '_' + t + '_transform')
                   for t in tr[ifrom:ito]][::-1]
        else:
            trs = [getattr(self, '_' + t + '_transform').inverse
                   for t in tr[ito:ifrom]]
        return self._cache.get(trs)

    @property
    def pixel_scale(self):
        tr = self._canvas_transform
        return (tr.map((1, 0)) - tr.map((0, 0)))[0]
<|endoftext|>


<commit_before># mako/lookup.py
# Copyright (C) 2006-2011 the Mako authors and contributors <see AUTHORS file>
#
# This module is part of Mako and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php

import os, stat, posixpath, re
from mako import exceptions, util
from mako.template import Template

try:
    import threading
except:
    import dummy_threading as threading

class TemplateCollection(object):
    """Represent a collection of :class:`.Template` objects,
    identifiable via uri.

    A :class:`.TemplateCollection` is linked to the usage of
    all template tags that address other templates, such
    as ``<%include>``, ``<%namespace>``, and ``<%inherit>``.
    The ``file`` attribute of each of those tags refers
    to a string URI that is passed to that :class:`.Template`
    object's :class:`.TemplateCollection` for resolution.

    :class:`.TemplateCollection` is an abstract class,
    with the usual default implementation being :class:`.TemplateLookup`.

     """

    def has_template(self, uri):
        """Return ``True`` if this :class:`.TemplateLookup` is
        capable of returning a :class:`.Template` object for the
        given URL.

        :param uri: String uri of the template to be resolved.

        """
        try:
            self.get_template(uri)
            return True
        except exceptions.TemplateLookupException:
            return False

    def get_template(self, uri, relativeto=None):
        """Return a :class:`.Template` object corresponding to the given
        URL.

        The default implementation raises
        :class:`.NotImplementedError`. Implementations should
        raise :class:`.TemplateLookupException` if the given uri
        cannot be resolved.

        :param uri: String uri of the template to be resolved.
        :param relativeto: if present, the given URI is assumed to
         be relative to this uri.

        """
        raise NotImplementedError()

    def filename_to_uri(self, uri, filename):
        """Convert the given filename to a uri relative to
           this TemplateCollection."""

        return uri

    def adjust_uri(self, uri, filename):
        """Adjust the given uri based on the calling filename.

        When this method is called from the runtime, the
        'filename' parameter is taken directly to the 'filename'
        attribute of the calling template. Therefore a custom
        TemplateCollection subclass can place any string
        identifier desired in the "filename" parameter of the
        Template objects it constructs and have them come back
        here.

        """
        return uri

class TemplateLookup(TemplateCollection):
    """Represent a collection of templates that locates template source files
    from the local filesystem.

    The primary argument is the ``directories`` argument, the list of
    directories to search::

        lookup = TemplateLookup(["/path/to/templates"])
        some_template = lookup.get_template("/index.html")

    The :class:`.TemplateLookup` can also be given :class:`.Template` objects
    programatically using :meth:`.put_string` or :meth:`.put_template`::

        lookup = TemplateLookup()
        lookup.put_string("base.html", '''
            ${self.next()}
        ''')
        lookup.put_string("hello.html", '''
            <%include file='base.html'/>

            Hello, world !
        ''')

    :param directories: A list of directory names which will be
     searched for a particular template URI. The URI is appended
     to each directory and the filesystem checked.

    :param collection_size: Approximate size of the collection used
     to store templates. If left at its default of -1, the size
     is unbounded, and a plain Python dictionary is used to
     relate URI strings to :class:`.Template` instances.
     Otherwise, a least-recently-used cache object is used which
     will maintain the size of the collection approximately to
     the number given.

    :param filesystem_checks: When at its default value of ``True``,
     each call to :meth:`TemplateLookup.get_template()` will
     compare the filesystem last modified time to the time in
     which an existing :class:`.Template` object was created.
     This allows the :class:`.TemplateLookup` to regenerate a
     new :class:`.Template` whenever the original source has
     been updated. Set this to ``False`` for a very minor
     performance increase.

    :param modulename_callable: A callable which, when present,
     is passed the path of the source file as well as the
     requested URI, and then returns the full path of the
     generated Python module file. This is used to inject
     alternate schemes for Pyhton module location. If left at
     its default of ``None``, the built in system of generation
     based on ``module_directory`` plus ``uri`` is used.

    All other keyword parameters available for
    :class:`.Template` are mirrored here. When new
    :class:`.Template` objects are created, the keywords
    established with this :class:`.TemplateLookup` are passed on
    to each new :class:`.Template`.

    """

    def __init__(self,
                        directories=None,
                        module_directory=None,
                        filesystem_checks=True,
                        collection_size=-1,
                        format_exceptions=False,
                        error_handler=None,
                        disable_unicode=False,
                        output_encoding='ascii',
                        encoding_errors='strict',
                        cache_type=None,
                        cache_dir=None, cache_url=None,
                        cache_enabled=True,
                        modulename_callable=None,
                        default_filters=None,
                        buffer_filters=(),
                        strict_undefined=False,
                        imports=None,
                        input_encoding=None,
                        preprocessor=None):

        self.directories = [posixpath.normpath(d) for d in
                            util.to_list(directories, ())
                            ]
        self.module_directory = module_directory
        self.modulename_callable = modulename_callable
        self.filesystem_checks = filesystem_checks
        self.collection_size = collection_size

        self.template_args = {
            'format_exceptions':format_exceptions,
            'error_handler':error_handler,
            'disable_unicode':disable_unicode,
            'output_encoding':output_encoding,
            'encoding_errors':encoding_errors,
            'input_encoding':input_encoding,
            'module_directory':module_directory,
            'cache_type':cache_type,
            'cache_dir':cache_dir or module_directory,
            'cache_url':cache_url,
            'cache_enabled':cache_enabled,
            'default_filters':default_filters,
            'buffer_filters':buffer_filters,
            'strict_undefined':strict_undefined,
            'imports':imports,
            'preprocessor':preprocessor}

        if collection_size == -1:
            self._collection = {}
            self._uri_cache = {}
        else:
            self._collection = util.LRUCache(collection_size)
            self._uri_cache = util.LRUCache(collection_size)
        self._mutex = threading.Lock()

    def get_template(self, uri):
        """Return a :class:`.Template` object corresponding to the given
        URL.

        Note the "relativeto" argument is not supported here at the moment.

        """

        try:
            if self.filesystem_checks:
                return self._check(uri, self._collection[uri])
            else:
                return self._collection[uri]
        except KeyError:
            u = re.sub(r'^\/+', '', uri)
            for dir in self.directories:
                srcfile = posixpath.normpath(posixpath.join(dir, u))
                if os.path.isfile(srcfile):
                    return self._load(srcfile, uri)
            else:
                raise exceptions.TopLevelLookupException(
                                    "Cant locate template for uri %r" % uri)

    def adjust_uri(self, uri, relativeto):
        """adjust the given uri based on the given relative uri."""

        if uri[0] != '/':
            if relativeto is not None:
                return posixpath.join(posixpath.dirname(relativeto), uri)
            else:
                return '/' + uri
        else:
            return uri

    def filename_to_uri(self, filename):
        """Convert the given filename to a uri relative to
           this TemplateCollection."""

        try:
            return self._uri_cache[filename]
        except KeyError:
            value = self._relativeize(filename)
            self._uri_cache[filename] = value
            return value

    def _relativeize(self, filename):
        """Return the portion of a filename that is 'relative'
           to the directories in this lookup.

        """

        filename = posixpath.normpath(filename)
        for dir in self.directories:
            if filename[0:len(dir)] == dir:
                return filename[len(dir):]
        else:
            return None

    def _load(self, filename, uri):
        self._mutex.acquire()
        try:
            try:
                # try returning from collection one
                # more time in case concurrent thread already loaded
                return self._collection[uri]
            except KeyError:
                pass
            try:
                if self.modulename_callable is not None:
                    module_filename = self.modulename_callable(filename, uri)
                else:
                    module_filename = None
                self._collection[uri] = template = Template(
                                        uri=uri,
                                        filename=posixpath.normpath(filename),
                                        lookup=self,
                                        module_filename=module_filename,
                                        **self.template_args)
                return template
            except:
                # if compilation fails etc, ensure
                # template is removed from collection,
                # re-raise
                self._collection.pop(uri, None)
                raise
        finally:
            self._mutex.release()

    def _check(self, uri, template):
        if template.filename is None:
            return template

        try:
            template_stat = os.stat(template.filename)
            if template.module._modified_time < \
                        template_stat[stat.ST_MTIME]:
                self._collection.pop(uri, None)
                return self._load(template.filename, uri)
            else:
                return template
        except OSError:
            self._collection.pop(uri, None)
            raise exceptions.TemplateLookupException(
                                "Cant locate template for uri %r" % uri)

    def put_string(self, uri, text):
        """Place a new :class:`.Template` object into this
        :class:`.TemplateLookup`, based on the given string of
        text.

        """
        self._collection[uri] = Template(
                                    text,
                                    lookup=self,
                                    uri=uri,
                                    **self.template_args)

    def put_template(self, uri, template):
        """Place a new :class:`.Template` object into this
        :class:`.TemplateLookup`, based on the given
        :class:`.Template` object.

        """
        self._collection[uri] = template

<commit_msg>- use the _uri_cache for adjust_uri(), as this is called each time there's a namespace or include<commit_after># mako/lookup.py
# Copyright (C) 2006-2011 the Mako authors and contributors <see AUTHORS file>
#
# This module is part of Mako and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php

import os, stat, posixpath, re
from mako import exceptions, util
from mako.template import Template

try:
    import threading
except:
    import dummy_threading as threading

class TemplateCollection(object):
    """Represent a collection of :class:`.Template` objects,
    identifiable via uri.

    A :class:`.TemplateCollection` is linked to the usage of
    all template tags that address other templates, such
    as ``<%include>``, ``<%namespace>``, and ``<%inherit>``.
    The ``file`` attribute of each of those tags refers
    to a string URI that is passed to that :class:`.Template`
    object's :class:`.TemplateCollection` for resolution.

    :class:`.TemplateCollection` is an abstract class,
    with the usual default implementation being :class:`.TemplateLookup`.

     """

    def has_template(self, uri):
        """Return ``True`` if this :class:`.TemplateLookup` is
        capable of returning a :class:`.Template` object for the
        given URL.

        :param uri: String uri of the template to be resolved.

        """
        try:
            self.get_template(uri)
            return True
        except exceptions.TemplateLookupException:
            return False

    def get_template(self, uri, relativeto=None):
        """Return a :class:`.Template` object corresponding to the given
        URL.

        The default implementation raises
        :class:`.NotImplementedError`. Implementations should
        raise :class:`.TemplateLookupException` if the given uri
        cannot be resolved.

        :param uri: String uri of the template to be resolved.
        :param relativeto: if present, the given URI is assumed to
         be relative to this uri.

        """
        raise NotImplementedError()

    def filename_to_uri(self, uri, filename):
        """Convert the given filename to a uri relative to
           this TemplateCollection."""

        return uri

    def adjust_uri(self, uri, filename):
        """Adjust the given uri based on the calling filename.

        When this method is called from the runtime, the
        'filename' parameter is taken directly to the 'filename'
        attribute of the calling template. Therefore a custom
        TemplateCollection subclass can place any string
        identifier desired in the "filename" parameter of the
        Template objects it constructs and have them come back
        here.

        """
        return uri

class TemplateLookup(TemplateCollection):
    """Represent a collection of templates that locates template source files
    from the local filesystem.

    The primary argument is the ``directories`` argument, the list of
    directories to search::

        lookup = TemplateLookup(["/path/to/templates"])
        some_template = lookup.get_template("/index.html")

    The :class:`.TemplateLookup` can also be given :class:`.Template` objects
    programatically using :meth:`.put_string` or :meth:`.put_template`::

        lookup = TemplateLookup()
        lookup.put_string("base.html", '''
            ${self.next()}
        ''')
        lookup.put_string("hello.html", '''
            <%include file='base.html'/>

            Hello, world !
        ''')

    :param directories: A list of directory names which will be
     searched for a particular template URI. The URI is appended
     to each directory and the filesystem checked.

    :param collection_size: Approximate size of the collection used
     to store templates. If left at its default of -1, the size
     is unbounded, and a plain Python dictionary is used to
     relate URI strings to :class:`.Template` instances.
     Otherwise, a least-recently-used cache object is used which
     will maintain the size of the collection approximately to
     the number given.

    :param filesystem_checks: When at its default value of ``True``,
     each call to :meth:`TemplateLookup.get_template()` will
     compare the filesystem last modified time to the time in
     which an existing :class:`.Template` object was created.
     This allows the :class:`.TemplateLookup` to regenerate a
     new :class:`.Template` whenever the original source has
     been updated. Set this to ``False`` for a very minor
     performance increase.

    :param modulename_callable: A callable which, when present,
     is passed the path of the source file as well as the
     requested URI, and then returns the full path of the
     generated Python module file. This is used to inject
     alternate schemes for Pyhton module location. If left at
     its default of ``None``, the built in system of generation
     based on ``module_directory`` plus ``uri`` is used.

    All other keyword parameters available for
    :class:`.Template` are mirrored here. When new
    :class:`.Template` objects are created, the keywords
    established with this :class:`.TemplateLookup` are passed on
    to each new :class:`.Template`.

    """

    def __init__(self,
                        directories=None,
                        module_directory=None,
                        filesystem_checks=True,
                        collection_size=-1,
                        format_exceptions=False,
                        error_handler=None,
                        disable_unicode=False,
                        output_encoding='ascii',
                        encoding_errors='strict',
                        cache_type=None,
                        cache_dir=None, cache_url=None,
                        cache_enabled=True,
                        modulename_callable=None,
                        default_filters=None,
                        buffer_filters=(),
                        strict_undefined=False,
                        imports=None,
                        input_encoding=None,
                        preprocessor=None):

        self.directories = [posixpath.normpath(d) for d in
                            util.to_list(directories, ())
                            ]
        self.module_directory = module_directory
        self.modulename_callable = modulename_callable
        self.filesystem_checks = filesystem_checks
        self.collection_size = collection_size

        self.template_args = {
            'format_exceptions':format_exceptions,
            'error_handler':error_handler,
            'disable_unicode':disable_unicode,
            'output_encoding':output_encoding,
            'encoding_errors':encoding_errors,
            'input_encoding':input_encoding,
            'module_directory':module_directory,
            'cache_type':cache_type,
            'cache_dir':cache_dir or module_directory,
            'cache_url':cache_url,
            'cache_enabled':cache_enabled,
            'default_filters':default_filters,
            'buffer_filters':buffer_filters,
            'strict_undefined':strict_undefined,
            'imports':imports,
            'preprocessor':preprocessor}

        if collection_size == -1:
            self._collection = {}
            self._uri_cache = {}
        else:
            self._collection = util.LRUCache(collection_size)
            self._uri_cache = util.LRUCache(collection_size)
        self._mutex = threading.Lock()

    def get_template(self, uri):
        """Return a :class:`.Template` object corresponding to the given
        URL.

        Note the "relativeto" argument is not supported here at the moment.

        """

        try:
            if self.filesystem_checks:
                return self._check(uri, self._collection[uri])
            else:
                return self._collection[uri]
        except KeyError:
            u = re.sub(r'^\/+', '', uri)
            for dir in self.directories:
                srcfile = posixpath.normpath(posixpath.join(dir, u))
                if os.path.isfile(srcfile):
                    return self._load(srcfile, uri)
            else:
                raise exceptions.TopLevelLookupException(
                                    "Cant locate template for uri %r" % uri)

    def adjust_uri(self, uri, relativeto):
        """adjust the given uri based on the given relative uri."""

        key = (uri, relativeto)
        if key in self._uri_cache:
            return self._uri_cache[key]

        if uri[0] != '/':
            if relativeto is not None:
                v = self._uri_cache[key] = posixpath.join(posixpath.dirname(relativeto), uri)
            else:
                v = self._uri_cache[key] = '/' + uri
        else:
            v = self._uri_cache[key] = uri
        return v

    def filename_to_uri(self, filename):
        """Convert the given filename to a uri relative to
           this TemplateCollection."""

        try:
            return self._uri_cache[filename]
        except KeyError:
            value = self._relativeize(filename)
            self._uri_cache[filename] = value
            return value

    def _relativeize(self, filename):
        """Return the portion of a filename that is 'relative'
           to the directories in this lookup.

        """

        filename = posixpath.normpath(filename)
        for dir in self.directories:
            if filename[0:len(dir)] == dir:
                return filename[len(dir):]
        else:
            return None

    def _load(self, filename, uri):
        self._mutex.acquire()
        try:
            try:
                # try returning from collection one
                # more time in case concurrent thread already loaded
                return self._collection[uri]
            except KeyError:
                pass
            try:
                if self.modulename_callable is not None:
                    module_filename = self.modulename_callable(filename, uri)
                else:
                    module_filename = None
                self._collection[uri] = template = Template(
                                        uri=uri,
                                        filename=posixpath.normpath(filename),
                                        lookup=self,
                                        module_filename=module_filename,
                                        **self.template_args)
                return template
            except:
                # if compilation fails etc, ensure
                # template is removed from collection,
                # re-raise
                self._collection.pop(uri, None)
                raise
        finally:
            self._mutex.release()

    def _check(self, uri, template):
        if template.filename is None:
            return template

        try:
            template_stat = os.stat(template.filename)
            if template.module._modified_time < \
                        template_stat[stat.ST_MTIME]:
                self._collection.pop(uri, None)
                return self._load(template.filename, uri)
            else:
                return template
        except OSError:
            self._collection.pop(uri, None)
            raise exceptions.TemplateLookupException(
                                "Cant locate template for uri %r" % uri)

    def put_string(self, uri, text):
        """Place a new :class:`.Template` object into this
        :class:`.TemplateLookup`, based on the given string of
        text.

        """
        self._collection[uri] = Template(
                                    text,
                                    lookup=self,
                                    uri=uri,
                                    **self.template_args)

    def put_template(self, uri, template):
        """Place a new :class:`.Template` object into this
        :class:`.TemplateLookup`, based on the given
        :class:`.Template` object.

        """
        self._collection[uri] = template

<|endoftext|>


<commit_before>import logging
import time

from mopidy import backend

import requests

from mopidy_pandora import listener

logger = logging.getLogger(__name__)

class PandoraPlaybackProvider(backend.PlaybackProvider):
    SKIP_LIMIT = 5

    def __init__(self, audio, backend):
        super(PandoraPlaybackProvider, self).__init__(audio, backend)

        # TODO: It shouldn't be necessary to keep track of the number of tracks that have been skipped in the
        # player anymore once https://github.com/mopidy/mopidy/issues/1221 has been fixed.
        self._consecutive_track_skips = 0

        # TODO: add gapless playback when it is supported in Mopidy > 1.1
        # self.audio.set_about_to_finish_callback(self.callback).get()

        # def callback(self):
        # See: https://discuss.mopidy.com/t/has-the-gapless-playback-implementation-been-completed-yet/784/2
        # self.audio.set_uri(self.translate_uri(self.get_next_track())).get()

    def change_pandora_track(self, track):
        """ Attempt to retrieve the Pandora playlist item from the buffer and verify that it is ready to be played.

        A track is playable if it has been stored in the buffer, has a URL, and the header for the Pandora URL can be
        retrieved and the status code checked.

        :param track: the track to retrieve and check the Pandora playlist item for.
        :return: True if the track is playable, False otherwise.
        """
        try:
            pandora_track = self.backend.library.lookup_pandora_track(track.uri)
            if pandora_track.get_is_playable():
                # Success, reset track skip counter.
                self._consecutive_track_skips = 0
                self._trigger_track_changed(track)
            else:
                raise Unplayable("Track with URI '{}' is not playable.".format(track.uri))

        except (AttributeError, requests.exceptions.RequestException) as e:
            logger.warning('Error changing Pandora track: {}, ({})'.format(pandora_track, e))
            # Track is not playable.
            self._consecutive_track_skips += 1

            if self._consecutive_track_skips >= self.SKIP_LIMIT:
                raise MaxSkipLimitExceeded(('Maximum track skip limit ({:d}) exceeded.'
                                            .format(self.SKIP_LIMIT)))
            raise Unplayable("Cannot change to Pandora track '{}', ({}:{}).".format(track.uri,
                                                                                type(e).__name__, e.args))

    def change_track(self, track):
        if track.uri is None:
            logger.warning("No URI for Pandora track '{}'. Track cannot be played.".format(track))
            return False
        try:
            self.change_pandora_track(track)
            return super(PandoraPlaybackProvider, self).change_track(track)

        except KeyError:
            logger.exception("Error changing Pandora track: failed to lookup '{}'.".format(track.uri))
            return False
        except Unplayable as e:
            logger.warning(e)
            self.backend.prepare_next_track()
            return False
        except MaxSkipLimitExceeded as e:
            logger.warning(e)
            self._trigger_skip_limit_exceeded()
            return False

    def translate_uri(self, uri):
        return self.backend.library.lookup_pandora_track(uri).audio_url

    def _trigger_track_changed(self, track):
        listener.PandoraPlaybackListener.send(listener.PandoraPlaybackListener.track_changed.__name__, track=track)

    def _trigger_skip_limit_exceeded(self):
        listener.PandoraPlaybackListener.send(listener.PandoraPlaybackListener.skip_limit_exceeded.__name__)

class EventHandlingPlaybackProvider(PandoraPlaybackProvider):
    def __init__(self, audio, backend):
        super(EventHandlingPlaybackProvider, self).__init__(audio, backend)

        self.double_click_interval = float(backend.config.get('double_click_interval'))
        self._click_time = 0

    def set_click_time(self, click_time=None):
        if click_time is None:
            self._click_time = time.time()
        else:
            self._click_time = click_time

    def get_click_time(self):
        return self._click_time

    def is_double_click(self):
        double_clicked = self._click_time > 0 and time.time() - self._click_time < self.double_click_interval

        if not double_clicked:
            self.set_click_time(0)

        return double_clicked

    def change_track(self, track):

        if self.is_double_click():
            self._trigger_doubleclicked()

        return super(EventHandlingPlaybackProvider, self).change_track(track)

    def resume(self):
        if self.is_double_click() and self.get_time_position() > 0:
            self._trigger_doubleclicked()

        return super(EventHandlingPlaybackProvider, self).resume()

    def pause(self):
        if self.get_time_position() > 0:
            self.set_click_time()

        return super(EventHandlingPlaybackProvider, self).pause()

    def _trigger_doubleclicked(self):
        self.set_click_time(0)
        listener.PandoraEventHandlingPlaybackListener.send(
            listener.PandoraEventHandlingPlaybackListener.doubleclicked.__name__)

class MaxSkipLimitExceeded(Exception):
    pass

class Unplayable(Exception):
    pass
<commit_msg>Fix maximum skip limit exception handling routine.<commit_after>import logging
import time

from mopidy import backend

import requests

from mopidy_pandora import listener

logger = logging.getLogger(__name__)

class PandoraPlaybackProvider(backend.PlaybackProvider):
    SKIP_LIMIT = 5

    def __init__(self, audio, backend):
        super(PandoraPlaybackProvider, self).__init__(audio, backend)

        # TODO: It shouldn't be necessary to keep track of the number of tracks that have been skipped in the
        # player anymore once https://github.com/mopidy/mopidy/issues/1221 has been fixed.
        self._consecutive_track_skips = 0

        # TODO: add gapless playback when it is supported in Mopidy > 1.1
        # self.audio.set_about_to_finish_callback(self.callback).get()

        # def callback(self):
        # See: https://discuss.mopidy.com/t/has-the-gapless-playback-implementation-been-completed-yet/784/2
        # self.audio.set_uri(self.translate_uri(self.get_next_track())).get()

    def change_pandora_track(self, track):
        """ Attempt to retrieve the Pandora playlist item from the buffer and verify that it is ready to be played.

        A track is playable if it has been stored in the buffer, has a URL, and the header for the Pandora URL can be
        retrieved and the status code checked.

        :param track: the track to retrieve and check the Pandora playlist item for.
        :return: True if the track is playable, False otherwise.
        """
        try:
            pandora_track = self.backend.library.lookup_pandora_track(track.uri)
            if pandora_track.get_is_playable():
                # Success, reset track skip counter.
                self._consecutive_track_skips = 0
                self._trigger_track_changed(track)
            else:
                raise Unplayable("Track with URI '{}' is not playable.".format(track.uri))

        except (Unplayable, AttributeError, requests.exceptions.RequestException) as e:
            logger.warning('Error changing Pandora track: {}, ({})'.format(pandora_track, e))
            # Track is not playable.
            self._consecutive_track_skips += 1

            if self._consecutive_track_skips >= self.SKIP_LIMIT:
                raise MaxSkipLimitExceeded(('Maximum track skip limit ({:d}) exceeded.'
                                            .format(self.SKIP_LIMIT)))
            raise Unplayable("Cannot change to Pandora track '{}', ({}:{}).".format(track.uri,
                                                                                type(e).__name__, e.args))

    def change_track(self, track):
        if track.uri is None:
            logger.warning("No URI for Pandora track '{}'. Track cannot be played.".format(track))
            return False
        try:
            self.change_pandora_track(track)
            return super(PandoraPlaybackProvider, self).change_track(track)

        except KeyError:
            logger.exception("Error changing Pandora track: failed to lookup '{}'.".format(track.uri))
            return False
        except Unplayable as e:
            logger.warning(e)
            self.backend.prepare_next_track()
            return False
        except MaxSkipLimitExceeded as e:
            logger.warning(e)
            self._trigger_skip_limit_exceeded()
            return False

    def translate_uri(self, uri):
        return self.backend.library.lookup_pandora_track(uri).audio_url

    def _trigger_track_changed(self, track):
        listener.PandoraPlaybackListener.send(listener.PandoraPlaybackListener.track_changed.__name__, track=track)

    def _trigger_skip_limit_exceeded(self):
        listener.PandoraPlaybackListener.send(listener.PandoraPlaybackListener.skip_limit_exceeded.__name__)

class EventHandlingPlaybackProvider(PandoraPlaybackProvider):
    def __init__(self, audio, backend):
        super(EventHandlingPlaybackProvider, self).__init__(audio, backend)

        self.double_click_interval = float(backend.config.get('double_click_interval'))
        self._click_time = 0

    def set_click_time(self, click_time=None):
        if click_time is None:
            self._click_time = time.time()
        else:
            self._click_time = click_time

    def get_click_time(self):
        return self._click_time

    def is_double_click(self):
        double_clicked = self._click_time > 0 and time.time() - self._click_time < self.double_click_interval

        if not double_clicked:
            self.set_click_time(0)

        return double_clicked

    def change_track(self, track):

        if self.is_double_click():
            self._trigger_doubleclicked()

        return super(EventHandlingPlaybackProvider, self).change_track(track)

    def resume(self):
        if self.is_double_click() and self.get_time_position() > 0:
            self._trigger_doubleclicked()

        return super(EventHandlingPlaybackProvider, self).resume()

    def pause(self):
        if self.get_time_position() > 0:
            self.set_click_time()

        return super(EventHandlingPlaybackProvider, self).pause()

    def _trigger_doubleclicked(self):
        self.set_click_time(0)
        listener.PandoraEventHandlingPlaybackListener.send(
            listener.PandoraEventHandlingPlaybackListener.doubleclicked.__name__)

class MaxSkipLimitExceeded(Exception):
    pass

class Unplayable(Exception):
    pass
<|endoftext|>