<commit_before># Copyright (c) 2015 Ultimaker B.V.
# Cura is released under the terms of the AGPLv3 or higher.

from UM.Logger import Logger
from UM.PluginObject import PluginObject

##  A type of plug-in that upgrades the configuration from an old file format to
#   a newer one.
#
#   Each version upgrade plug-in can convert machine instances, preferences and
#   profiles from one version to one other version. Which versions that are is
#   specified in the metadata of the plug-in.
class VersionUpgrade(PluginObject):
    ##  Initialises a version upgrade plugin instance.
    def __init__(self):
        super().__init__(self)

    ##  Upgrades a machine instance file from one file format to another.
    #
    #   This parses the serialised data of a machine instance and converts it to
    #   a serialised form of the new file format.
    #
    #   \param serialised A machine instance, serialised in an old file format.
    #   \return A machine instance, serialised in a newer file format.
    def upgradeMachineInstance(self, serialised):
        Logger.log("w", "This version upgrade plug-in defines no way to upgrade machine instances.") #A subclass should implement this.

    ##  Upgrades a preferences file from one file format to another.
    #
    #   This parses the serialised data of a preferences file and converts it to
    #   a serialised form of the new file format.
    #
    #   \param serialised A preferences file, serialised in an old file format.
    #   \return A preferences file, serialised in a newer file format.
    def upgradePreferences(self, serialised):
        Logger.log("w", "This version upgrade plug-in defines no way to upgrade preferences.") #A subclass should implement this.

    ##  Upgrades a profile from one file format to another.
    #
    #   This parses the serialised data of a profile and converts it to a
    #   serialised form of the new file format.
    #
    #   \param serialised A profile, serialised in an old file format.
    #   \return A profile, serialised in a newer file format.
    def upgradeProfile(self, serialised):
        Logger.log("w", "This version upgrade plug-in defines no way to upgrade profiles.") #A subclass should implement this.<commit_msg>Don't pass self to super init call<commit_after># Copyright (c) 2015 Ultimaker B.V.
# Cura is released under the terms of the AGPLv3 or higher.

from UM.Logger import Logger
from UM.PluginObject import PluginObject

##  A type of plug-in that upgrades the configuration from an old file format to
#   a newer one.
#
#   Each version upgrade plug-in can convert machine instances, preferences and
#   profiles from one version to one other version. Which versions that are is
#   specified in the metadata of the plug-in.
class VersionUpgrade(PluginObject):
    ##  Initialises a version upgrade plugin instance.
    def __init__(self):
        super().__init__()

    ##  Upgrades a machine instance file from one file format to another.
    #
    #   This parses the serialised data of a machine instance and converts it to
    #   a serialised form of the new file format.
    #
    #   \param serialised A machine instance, serialised in an old file format.
    #   \return A machine instance, serialised in a newer file format.
    def upgradeMachineInstance(self, serialised):
        Logger.log("w", "This version upgrade plug-in defines no way to upgrade machine instances.") #A subclass should implement this.

    ##  Upgrades a preferences file from one file format to another.
    #
    #   This parses the serialised data of a preferences file and converts it to
    #   a serialised form of the new file format.
    #
    #   \param serialised A preferences file, serialised in an old file format.
    #   \return A preferences file, serialised in a newer file format.
    def upgradePreferences(self, serialised):
        Logger.log("w", "This version upgrade plug-in defines no way to upgrade preferences.") #A subclass should implement this.

    ##  Upgrades a profile from one file format to another.
    #
    #   This parses the serialised data of a profile and converts it to a
    #   serialised form of the new file format.
    #
    #   \param serialised A profile, serialised in an old file format.
    #   \return A profile, serialised in a newer file format.
    def upgradeProfile(self, serialised):
        Logger.log("w", "This version upgrade plug-in defines no way to upgrade profiles.") #A subclass should implement this.<|endoftext|>


<commit_before>from numpy.core.memmap import memmap as memmap_type
import numpy as N
import scipy.io as SIO

from neuroimaging.utils.odict import odict
from neuroimaging.data_io import DataSource
from neuroimaging.data_io.formats import utils
import neuroimaging.data_io.formats.binary as bin
from neuroimaging.core.reference.axis import space, spacetime
from neuroimaging.core.reference.mapping import Affine
from neuroimaging.core.reference.grid import SamplingGrid
from neuroimaging.utils.path import path

class AnalyzeFormatError(Exception):
    """
    Analyze format error exception
    """

# datatype is a one bit flag into the datatype identification byte of the
# Analyze header.
BYTE = 2
SHORT = 4
INTEGER = 8
FLOAT = 16
COMPLEX = 32
DOUBLE = 64

# map Analyze datatype to numpy scalar type
datatype2sctype = {
  BYTE: N.uint8,
  SHORT: N.int16,
  INTEGER: N.int32,
  FLOAT: N.float32,
  DOUBLE: N.float64,
  COMPLEX: N.complex64}

# map numpy scalar type to Analyze datatype
sctype2datatype = \
  dict([(v, k) for k, v in datatype2sctype.items()])

HEADER_SIZE = 348

# ordered dictionary of header field names mapped to struct format
struct_formats = odict((
    ('sizeof_hdr','i'),
    ('data_type','10s'),
    ('db_name','18s'),
    ('extents','i'),
    ('session_error','h'),
    ('regular','c'),
    ('hkey_un0','c'),
    ('dim','8h'),
    ('vox_units','4s'),
    ('cal_units','8s'),
    ('unused1','h'),
    ('datatype','h'),
    ('bitpix','h'),
    ('dim_un0','h'),
    ('pixdim','8f'),
    ('vox_offset','f'),
    ('scale_factor','f'),
    ('funused2','f'),
    ('funused3','f'),
    ('cal_max','f'),
    ('cal_min','f'),
    ('compressed','i'),
    ('verified','i'),
    ('glmax','i'),
    ('glmin','i'),
    ('descrip','80s'),
    ('aux_file','24s'),
    ('orient','c'),
    ('origin','5h'),
    ('generated','10s'),
    ('scannum','10s'),
    ('patient_id','10s'),
    ('exp_date','10s'),
    ('exp_time','10s'),
    ('hist_un0','3s'),
    ('views','i'),
    ('vols_added','i'),
    ('start_field','i'),
    ('field_skip','i'),
    ('omax','i'),
    ('omin','i'),
    ('smax','i'),
    ('smin','i')))

field_formats = struct_formats.values()

class Analyze(bin.BinaryFormat):
    """
    A class to read and write ANALYZE format images.

    """
    _field_defaults = {
      'sizeof_hdr': HEADER_SIZE,
      'extents': 16384,
      'regular': 'r',
      'hkey_un0': ' ',
      'vox_units': 'mm',
      'scale_factor':1.}

    extensions = ('.img', '.hdr', '.mat')
    #extensions = ('.img')
    # maybe I'll implement this when I figure out how
    nvector = -1
    # always false for Analyze
    extendable = False

    def __init__(self, filename, mode="r", datasource=DataSource(), **keywords):
        """
        Constructs a Analyze binary format object with at least a filename
        possible additional keyword arguments:
        grid = Grid object
        dtype = numpy data type
        intent = meaning of data
        clobber = allowed to clobber?
        usemat = use mat file?
        """

        bin.BinaryFormat.__init__(self, filename, mode, datasource, **keywords)
        self.mat_file = self.filebase + ".mat"
        self.clobber = keywords.get('clobber', False)
        self.intent = keywords.get('intent', '')
        self.usematfile = keywords.get('usemat', True)

        self.header_formats = struct_formats

        # fill the header dictionary in order, with any default values
        self.header_defaults()
        if self.mode[0] is "w":
            # should try to populate the canonical fields and
            # corresponding header fields with info from grid?
            self.byteorder = utils.NATIVE
            self.dtype = N.dtype(keywords.get('dtype', N.float64))
            self.dtype = self.dtype.newbyteorder(self.byteorder)
            if self.grid is not None:
                self.header_from_given()
            else:
                raise NotImplementedError("Don't know how to create header " \
                                          "info without a grid object")
            self.write_header(clobber=self.clobber)
        else:
            self.byteorder = self.guess_byteorder(self.header_file,
                                                  datasource=self.datasource)
            self.read_header()
            ## make sure we have the correct byteorder
            tmpsctype = datatype2sctype[self.header['datatype']]
            tmpstr = N.dtype(tmpsctype)
            self.dtype = tmpstr.newbyteorder(self.byteorder)
            self.ndim = self.header['dim'][0]

        # fill in the canonical list as best we can for Analyze
        self.inform_canonical()

        ########## This could stand a clean-up ################################
        if not self.grid:
            if self.ndim == 3:
                axisnames = space[::-1]
                origin = tuple(self.header['origin'][0:3])
                step = tuple(self.header['pixdim'][1:4])
                shape = tuple(self.header['dim'][1:4])
            elif self.ndim == 4 and self.nvector <= 1:
                axisnames = spacetime[::-1]
                origin = tuple(self.header['origin'][0:3]) + (1,)
                step = tuple(self.header['pixdim'][1:5])
                shape = tuple(self.header['dim'][1:5])
                if shape[-1] == 1:
                    axisnames = axisnames[:-1]
                    origin = origin[:-1]
                    step = step[:-1]
                    shape = shape[:-1]
                    self.ndim -= 1

            ## Setup affine transformation
            self.grid = SamplingGrid.from_start_step(names=axisnames,
                                                shape=shape,
                                                start=-N.array(origin)*step,
                                                step=step)
            if self.usematfile:
                self.grid.transform(self.read_mat())
                # assume .mat matrix uses FORTRAN indexing
                self.grid = self.grid.matlab2python()
        #else: Grid was already assigned by Format constructor

        # get memmaped array
        self.attach_data()

    @staticmethod
    def _default_field_value(fieldname, fieldformat):
        "[STATIC] Get the default value for the given field."
        return Analyze._field_defaults.get(fieldname, None) or \
               utils.format_defaults[fieldformat[-1]]

    def header_defaults(self):
        for field, format in self.header_formats.items():
            self.header[field] = self._default_field_value(field, format)

    def header_from_given(self):
        self.header['datatype'] = sctype2datatype[self.dtype.type]
        self.header['bitpix'] = self.dtype.itemsize * 8
        self.grid = self.grid.python2matlab()
        self.ndim = self.grid.ndim

        if not isinstance(self.grid.mapping, Affine):
            raise ValueError, 'non-Affine grid in writing out ANALYZE file'

        if self.grid.mapping.isdiagonal():
            _diag = True
        else:
            # what's goin on here??
            _diag = False
            self.write_mat()

        _dim = [0]*8
        _pixdim = [0.] * 8
        _dim[0] = self.ndim
        _dim[1:self.ndim+1] = self.grid.shape[:self.ndim]
        if _diag:
            _pixdim[1:self.ndim+1] = list(N.diag(self.grid.mapping.transform))[:self.ndim]
        else:
            _pixdim[1:self.ndim+1] = [1.]*self.ndim

        self.header['dim'] = _dim
        self.header['pixdim'] = _pixdim
        if _diag:
            origin = self.grid.mapping.tovoxel(N.array([0]*self.ndim))
            self.header['origin'] = list(origin) + [0]*(5-origin.shape[0])
        if not _diag:
            self.header['origin'] = [0]*5

        self.grid = self.grid.matlab2python()

    def prewrite(self, x):
        """
        Filter the incoming data. If we're casting to an Integer type,
        record the new scale factor
        """
        # try to cast in two cases:
        # 1 - we're replacing all the data
        # 2 - the maximum of the given slice of data exceeds the
        #     global maximum under the current scaling
        if x.shape == self.data.shape or \
               x.max() > (self.header['scale_factor']*self.data).max():
            scale = utils.scale_data(x, self.dtype,
                                     self.header['scale_factor'])

            # if the scale changed, mark it down
            if scale != self.header['scale_factor']:
                self.header['scale_factor'] = scale
                self.write_header(clobber=True)

        return N.round(x/self.header['scale_factor'])

    def postread(self, x):
        """
        Might transform the data after getting it from memmap
        """
        if self.header['scale_factor']:
            return x*self.header['scale_factor']
        else:
            return x

    def __del__(self):
        if hasattr(self, 'data'):
            if isinstance(self.data, memmap_type):
                self.data.sync()
            del self.data

    def inform_canonical(self, fieldsDict=None):
        if fieldsDict is not None:
            self.canonical_fields = odict(fieldsDict)
        else:
            self.canonical_fields['datasize'] = self.header['bitpix']
            (self.canonical_fields['ndim'],
             self.canonical_fields['xdim'],
             self.canonical_fields['ydim'],
             self.canonical_fields['zdim'],
             self.canonical_fields['tdim']) = self.header['dim'][:5]
            self.canonical_fields['scaling'] = self.header['scale_factor']

    def read_mat(self):
        """
        Return affine transformation matrix, if it exists.
        For now, the format is assumed to be a tab-delimited 4 line file.
        Other formats should be added.

        Read Binary
        import scipy.io as sio
        M = sio.loadmat('my.mat')
        sio.savemat('my_new.mat', M)
        """
        if self.datasource.exists(self.mat_file):

            mat = SIO.loadmat(self.mat_file)
            # SIO.loadmat puts mat in correct order for a C-ordered array
            # no need to Affine.matlab2python to correct for ordering
            if mat.has_key('mat'):
                return Affine(mat.get('mat'))
            elif mat.has_key('M'):
                return Affine(mat.get('M'))
            else:
                print 'Mat file did not contain Transform!'
                # if all fails give them something?
                return Affine.identity(self.ndim)

        else:
            # FIXME: Do we want to use these?
            if self.ndim == 4:
                names = spacetime[::-1]
            else:
                names = space[::-1]
            return Affine.identity(self.ndim)

    def write_mat(self, matfile=None):
        "Write out the affine transformation matrix."
        if matfile is None:
            matfile = self.mat_file
        if self.clobber or not path(matfile).exists():
            self.grid.mapping.tofile(matfile)

    def _get_filenames(self):
        return self.filebase + ".hdr", self.filebase + ".img"

    @staticmethod
    def guess_byteorder(hdrfile, datasource=DataSource()):
        """
        Determine byte order of the header.  The first header element is the
        header size.  It should always be 384.  If it is not then you know you
        read it in the wrong byte order.
        """
        if type(hdrfile)==type(""):
            hdrfile = datasource.open(hdrfile)
        byteorder = utils.LITTLE_ENDIAN
        reported_length = utils.struct_unpack(hdrfile,
          byteorder, field_formats[0:1])[0]
        if reported_length != HEADER_SIZE:
            byteorder = utils.BIG_ENDIAN
        return byteorder

<commit_msg>fix error on shutdown (ticket #53)<commit_after>from numpy.core.memmap import memmap as memmap_type
import numpy as N
import scipy.io as SIO

from neuroimaging.utils.odict import odict
from neuroimaging.data_io import DataSource
from neuroimaging.data_io.formats import utils
import neuroimaging.data_io.formats.binary as bin
from neuroimaging.core.reference.axis import space, spacetime
from neuroimaging.core.reference.mapping import Affine
from neuroimaging.core.reference.grid import SamplingGrid
from neuroimaging.utils.path import path

class AnalyzeFormatError(Exception):
    """
    Analyze format error exception
    """

# datatype is a one bit flag into the datatype identification byte of the
# Analyze header.
BYTE = 2
SHORT = 4
INTEGER = 8
FLOAT = 16
COMPLEX = 32
DOUBLE = 64

# map Analyze datatype to numpy scalar type
datatype2sctype = {
  BYTE: N.uint8,
  SHORT: N.int16,
  INTEGER: N.int32,
  FLOAT: N.float32,
  DOUBLE: N.float64,
  COMPLEX: N.complex64}

# map numpy scalar type to Analyze datatype
sctype2datatype = \
  dict([(v, k) for k, v in datatype2sctype.items()])

HEADER_SIZE = 348

# ordered dictionary of header field names mapped to struct format
struct_formats = odict((
    ('sizeof_hdr','i'),
    ('data_type','10s'),
    ('db_name','18s'),
    ('extents','i'),
    ('session_error','h'),
    ('regular','c'),
    ('hkey_un0','c'),
    ('dim','8h'),
    ('vox_units','4s'),
    ('cal_units','8s'),
    ('unused1','h'),
    ('datatype','h'),
    ('bitpix','h'),
    ('dim_un0','h'),
    ('pixdim','8f'),
    ('vox_offset','f'),
    ('scale_factor','f'),
    ('funused2','f'),
    ('funused3','f'),
    ('cal_max','f'),
    ('cal_min','f'),
    ('compressed','i'),
    ('verified','i'),
    ('glmax','i'),
    ('glmin','i'),
    ('descrip','80s'),
    ('aux_file','24s'),
    ('orient','c'),
    ('origin','5h'),
    ('generated','10s'),
    ('scannum','10s'),
    ('patient_id','10s'),
    ('exp_date','10s'),
    ('exp_time','10s'),
    ('hist_un0','3s'),
    ('views','i'),
    ('vols_added','i'),
    ('start_field','i'),
    ('field_skip','i'),
    ('omax','i'),
    ('omin','i'),
    ('smax','i'),
    ('smin','i')))

field_formats = struct_formats.values()

class Analyze(bin.BinaryFormat):
    """
    A class to read and write ANALYZE format images.

    """
    _field_defaults = {
      'sizeof_hdr': HEADER_SIZE,
      'extents': 16384,
      'regular': 'r',
      'hkey_un0': ' ',
      'vox_units': 'mm',
      'scale_factor':1.}

    extensions = ('.img', '.hdr', '.mat')
    #extensions = ('.img')
    # maybe I'll implement this when I figure out how
    nvector = -1
    # always false for Analyze
    extendable = False

    def __init__(self, filename, mode="r", datasource=DataSource(), **keywords):
        """
        Constructs a Analyze binary format object with at least a filename
        possible additional keyword arguments:
        grid = Grid object
        dtype = numpy data type
        intent = meaning of data
        clobber = allowed to clobber?
        usemat = use mat file?
        """
        bin.BinaryFormat.__init__(self, filename, mode, datasource, **keywords)
        self.mat_file = self.filebase + ".mat"
        self.clobber = keywords.get('clobber', False)
        self.intent = keywords.get('intent', '')
        self.usematfile = keywords.get('usemat', True)

        self.header_formats = struct_formats

        # fill the header dictionary in order, with any default values
        self.header_defaults()
        if self.mode[0] is "w":
            # should try to populate the canonical fields and
            # corresponding header fields with info from grid?
            self.byteorder = utils.NATIVE
            self.dtype = N.dtype(keywords.get('dtype', N.float64))
            self.dtype = self.dtype.newbyteorder(self.byteorder)
            if self.grid is not None:
                self.header_from_given()
            else:
                raise NotImplementedError("Don't know how to create header " \
                                          "info without a grid object")
            self.write_header(clobber=self.clobber)
        else:
            self.byteorder = self.guess_byteorder(self.header_file,
                                                  datasource=self.datasource)
            self.read_header()
            ## make sure we have the correct byteorder
            tmpsctype = datatype2sctype[self.header['datatype']]
            tmpstr = N.dtype(tmpsctype)
            self.dtype = tmpstr.newbyteorder(self.byteorder)
            self.ndim = self.header['dim'][0]

        # fill in the canonical list as best we can for Analyze
        self.inform_canonical()

        ########## This could stand a clean-up ################################
        if not self.grid:
            if self.ndim == 3:
                axisnames = space[::-1]
                origin = tuple(self.header['origin'][0:3])
                step = tuple(self.header['pixdim'][1:4])
                shape = tuple(self.header['dim'][1:4])
            elif self.ndim == 4 and self.nvector <= 1:
                axisnames = spacetime[::-1]
                origin = tuple(self.header['origin'][0:3]) + (1,)
                step = tuple(self.header['pixdim'][1:5])
                shape = tuple(self.header['dim'][1:5])
                if shape[-1] == 1:
                    axisnames = axisnames[:-1]
                    origin = origin[:-1]
                    step = step[:-1]
                    shape = shape[:-1]
                    self.ndim -= 1

            ## Setup affine transformation
            self.grid = SamplingGrid.from_start_step(names=axisnames,
                                                shape=shape,
                                                start=-N.array(origin)*step,
                                                step=step)
            if self.usematfile:
                self.grid.transform(self.read_mat())
                # assume .mat matrix uses FORTRAN indexing
                self.grid = self.grid.matlab2python()
        #else: Grid was already assigned by Format constructor

        # get memmaped array
        self.attach_data()

    @staticmethod
    def _default_field_value(fieldname, fieldformat):
        "[STATIC] Get the default value for the given field."
        return Analyze._field_defaults.get(fieldname, None) or \
               utils.format_defaults[fieldformat[-1]]

    def header_defaults(self):
        for field, format in self.header_formats.items():
            self.header[field] = self._default_field_value(field, format)

    def header_from_given(self):
        self.header['datatype'] = sctype2datatype[self.dtype.type]
        self.header['bitpix'] = self.dtype.itemsize * 8
        self.grid = self.grid.python2matlab()
        self.ndim = self.grid.ndim

        if not isinstance(self.grid.mapping, Affine):
            raise ValueError, 'non-Affine grid in writing out ANALYZE file'

        if self.grid.mapping.isdiagonal():
            _diag = True
        else:
            # what's goin on here??
            _diag = False
            self.write_mat()

        _dim = [0]*8
        _pixdim = [0.] * 8
        _dim[0] = self.ndim
        _dim[1:self.ndim+1] = self.grid.shape[:self.ndim]
        if _diag:
            _pixdim[1:self.ndim+1] = list(N.diag(self.grid.mapping.transform))[:self.ndim]
        else:
            _pixdim[1:self.ndim+1] = [1.]*self.ndim

        self.header['dim'] = _dim
        self.header['pixdim'] = _pixdim
        if _diag:
            origin = self.grid.mapping.tovoxel(N.array([0]*self.ndim))
            self.header['origin'] = list(origin) + [0]*(5-origin.shape[0])
        if not _diag:
            self.header['origin'] = [0]*5

        self.grid = self.grid.matlab2python()

    def prewrite(self, x):
        """
        Filter the incoming data. If we're casting to an Integer type,
        record the new scale factor
        """
        # try to cast in two cases:
        # 1 - we're replacing all the data
        # 2 - the maximum of the given slice of data exceeds the
        #     global maximum under the current scaling
        if x.shape == self.data.shape or \
               x.max() > (self.header['scale_factor']*self.data).max():
            scale = utils.scale_data(x, self.dtype,
                                     self.header['scale_factor'])

            # if the scale changed, mark it down
            if scale != self.header['scale_factor']:
                self.header['scale_factor'] = scale
                self.write_header(clobber=True)

        return N.round(x/self.header['scale_factor'])

    def postread(self, x):
        """
        Might transform the data after getting it from memmap
        """
        if self.header['scale_factor']:
            return x*self.header['scale_factor']
        else:
            return x

    def __del__(self):
        if hasattr(self, 'data'):
            try:
                self.data.sync()
            except: # what do we expect to go wrong here?
                pass
            del self.data

    def inform_canonical(self, fieldsDict=None):
        if fieldsDict is not None:
            self.canonical_fields = odict(fieldsDict)
        else:
            self.canonical_fields['datasize'] = self.header['bitpix']
            (self.canonical_fields['ndim'],
             self.canonical_fields['xdim'],
             self.canonical_fields['ydim'],
             self.canonical_fields['zdim'],
             self.canonical_fields['tdim']) = self.header['dim'][:5]
            self.canonical_fields['scaling'] = self.header['scale_factor']

    def read_mat(self):
        """
        Return affine transformation matrix, if it exists.
        For now, the format is assumed to be a tab-delimited 4 line file.
        Other formats should be added.

        Read Binary
        import scipy.io as sio
        M = sio.loadmat('my.mat')
        sio.savemat('my_new.mat', M)
        """
        if self.datasource.exists(self.mat_file):

            mat = SIO.loadmat(self.mat_file)
            # SIO.loadmat puts mat in correct order for a C-ordered array
            # no need to Affine.matlab2python to correct for ordering
            if mat.has_key('mat'):
                return Affine(mat.get('mat'))
            elif mat.has_key('M'):
                return Affine(mat.get('M'))
            else:
                print 'Mat file did not contain Transform!'
                # if all fails give them something?
                return Affine.identity(self.ndim)

        else:
            # FIXME: Do we want to use these?
            if self.ndim == 4:
                names = spacetime[::-1]
            else:
                names = space[::-1]
            return Affine.identity(self.ndim)

    def write_mat(self, matfile=None):
        "Write out the affine transformation matrix."
        if matfile is None:
            matfile = self.mat_file
        if self.clobber or not path(matfile).exists():
            self.grid.mapping.tofile(matfile)

    def _get_filenames(self):
        return self.filebase + ".hdr", self.filebase + ".img"

    @staticmethod
    def guess_byteorder(hdrfile, datasource=DataSource()):
        """
        Determine byte order of the header.  The first header element is the
        header size.  It should always be 384.  If it is not then you know you
        read it in the wrong byte order.
        """
        if type(hdrfile)==type(""):
            hdrfile = datasource.open(hdrfile)
        byteorder = utils.LITTLE_ENDIAN
        reported_length = utils.struct_unpack(hdrfile,
          byteorder, field_formats[0:1])[0]
        if reported_length != HEADER_SIZE:
            byteorder = utils.BIG_ENDIAN
        return byteorder

<|endoftext|>


<commit_before># coding: utf-8

# maposmatic, the web front-end of the MapOSMatic city map generation system
# Copyright (C) 2009  David Decotigny
# Copyright (C) 2009  Frédéric Lehobey
# Copyright (C) 2009  David Mentré
# Copyright (C) 2009  Maxime Petazzoni
# Copyright (C) 2009  Thomas Petazzoni
# Copyright (C) 2009  Gaël Utard

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from django.db import models
from django.utils.translation import ugettext_lazy as _

from datetime import datetime, timedelta
import www.settings
import re
import os

import logging

class MapRenderingJobManager(models.Manager):
    def to_render(self):
        return MapRenderingJob.objects.filter(status=0).order_by('submission_time')

    def queue_size(self):
        return MapRenderingJob.objects.filter(status=0).count()

    # We try to find a rendered map from the last 15 days, which still
    # has its thumbnail present.
    def get_random_with_thumbnail(self):
        fifteen_days_before = datetime.now() - timedelta(15)
        maps = MapRenderingJob.objects.filter(status=2).filter(submission_time__gte=fifteen_days_before).order_by('?')[0:10]
        for m in maps:
            print m
            print m.get_thumbnail()
            if m.get_thumbnail():
                return m
        return None

SPACE_REDUCE = re.compile(r"\s+")
NONASCII_REMOVE = re.compile(r"[^A-Za-z0-9]+")

class MapRenderingJob(models.Model):

    STATUS_LIST = (
        (0, 'Submitted'),
        (1, 'In progress'),
        (2, 'Done'),
        )

    maptitle = models.CharField(max_length=256)
    administrative_city = models.CharField(max_length=256, blank=True)
    lat_upper_left = models.FloatField(blank=True, null=True)
    lon_upper_left = models.FloatField(blank=True, null=True)
    lat_bottom_right = models.FloatField(blank=True, null=True)
    lon_bottom_right = models.FloatField(blank=True, null=True)
    status = models.IntegerField(choices=STATUS_LIST)
    submission_time = models.DateTimeField(auto_now_add=True)
    startofrendering_time = models.DateTimeField(null=True)
    endofrendering_time = models.DateTimeField(null=True)
    resultmsg = models.CharField(max_length=256, null=True)
    submitterip = models.IPAddressField()
    index_queue_at_submission = models.IntegerField()

    objects = MapRenderingJobManager()

    def __str__(self):
        return self.maptitle.encode('utf-8')

    def maptitle_computized(self):
        t = self.maptitle.strip()
        t = SPACE_REDUCE.sub("-", t)
        t = NONASCII_REMOVE.sub("", t)
        return t

    def files_prefix(self):
        return "%06d_%s_%s" % (self.id,
                             self.startofrendering_time.strftime("%Y-%m-%d_%H-%M"),
                             self.maptitle_computized())

    def start_rendering(self):
        self.status = 1
        self.startofrendering_time = datetime.now()
        self.save()

    def end_rendering(self, resultmsg):
        self.status = 2
        self.endofrendering_time = datetime.now()
        self.resultmsg = resultmsg
        self.save()

    def is_waiting(self):
        return self.status == 0

    def is_rendering(self):
        return self.status == 1

    def is_done(self):
        return self.status == 2

    def is_done_ok(self):
        return self.is_done() and self.resultmsg == "ok"

    def is_done_failed(self):
        return self.is_done() and self.resultmsg != "ok"

    def get_map_fileurl(self, format):
        return www.settings.RENDERING_RESULT_URL + "/" + self.files_prefix() + "." + format

    def get_map_filepath(self, format):
        return os.path.join(www.settings.RENDERING_RESULT_PATH, self.files_prefix() + "." + format)

    def get_index_fileurl(self, format):
        return www.settings.RENDERING_RESULT_URL + "/" + self.files_prefix() + "_index." + format

    def get_index_filepath(self, format):
        return os.path.join(www.settings.RENDERING_RESULT_PATH, self.files_prefix() + "_index." + format)

    def output_files(self):
        allfiles = []
        for format in www.settings.RENDERING_RESULT_FORMATS:
            print format
            if format != 'csv' and os.path.exists(self.get_map_filepath(format)):
                allfiles.append((self.get_map_fileurl(format),
                                 _("%(title)s %(format)s Map") \
                                     % { 'title' : self.maptitle,
                                         'format': format.upper() } ))
            if os.path.exists(self.get_index_filepath(format)):
                allfiles.append((self.get_index_fileurl(format),
                                 _("%(title)s %(format)s Index") % \
                                     { 'title': self.maptitle,
                                       'format': format.upper() }))
        return allfiles

    def get_thumbnail(self):
        thumbnail_file = os.path.join(www.settings.RENDERING_RESULT_PATH, self.files_prefix() + "_small.png")
        thumbnail_url = www.settings.RENDERING_RESULT_URL + "/" + self.files_prefix() + "_small.png"
        if os.path.exists(thumbnail_file):
            return thumbnail_url
        else:
            return None

    def current_position_in_queue(self):
        return MapRenderingJob.objects.filter(status=0).filter(index_queue_at_submission__lte=self.index_queue_at_submission).count()

    # Estimate the date at which the rendering will be started
    def rendering_estimated_start_time(self):
        waiting_time = datetime.now() - self.submission_time
        progression = self.index_queue_at_submission - self.current_position_in_queue()
        if progression == 0:
            return datetime.now()
        mean_job_rendering_time = waiting_time // progression
        estimated_time_left = mean_job_rendering_time * self.current_position_in_queue()
        return datetime.now() + estimated_time_left
<commit_msg>Remove debug messages<commit_after># coding: utf-8

# maposmatic, the web front-end of the MapOSMatic city map generation system
# Copyright (C) 2009  David Decotigny
# Copyright (C) 2009  Frédéric Lehobey
# Copyright (C) 2009  David Mentré
# Copyright (C) 2009  Maxime Petazzoni
# Copyright (C) 2009  Thomas Petazzoni
# Copyright (C) 2009  Gaël Utard

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from django.db import models
from django.utils.translation import ugettext_lazy as _

from datetime import datetime, timedelta
import www.settings
import re
import os

import logging

class MapRenderingJobManager(models.Manager):
    def to_render(self):
        return MapRenderingJob.objects.filter(status=0).order_by('submission_time')

    def queue_size(self):
        return MapRenderingJob.objects.filter(status=0).count()

    # We try to find a rendered map from the last 15 days, which still
    # has its thumbnail present.
    def get_random_with_thumbnail(self):
        fifteen_days_before = datetime.now() - timedelta(15)
        maps = MapRenderingJob.objects.filter(status=2).filter(submission_time__gte=fifteen_days_before).order_by('?')[0:10]
        for m in maps:
            if m.get_thumbnail():
                return m
        return None

SPACE_REDUCE = re.compile(r"\s+")
NONASCII_REMOVE = re.compile(r"[^A-Za-z0-9]+")

class MapRenderingJob(models.Model):

    STATUS_LIST = (
        (0, 'Submitted'),
        (1, 'In progress'),
        (2, 'Done'),
        )

    maptitle = models.CharField(max_length=256)
    administrative_city = models.CharField(max_length=256, blank=True)
    lat_upper_left = models.FloatField(blank=True, null=True)
    lon_upper_left = models.FloatField(blank=True, null=True)
    lat_bottom_right = models.FloatField(blank=True, null=True)
    lon_bottom_right = models.FloatField(blank=True, null=True)
    status = models.IntegerField(choices=STATUS_LIST)
    submission_time = models.DateTimeField(auto_now_add=True)
    startofrendering_time = models.DateTimeField(null=True)
    endofrendering_time = models.DateTimeField(null=True)
    resultmsg = models.CharField(max_length=256, null=True)
    submitterip = models.IPAddressField()
    index_queue_at_submission = models.IntegerField()

    objects = MapRenderingJobManager()

    def __str__(self):
        return self.maptitle.encode('utf-8')

    def maptitle_computized(self):
        t = self.maptitle.strip()
        t = SPACE_REDUCE.sub("-", t)
        t = NONASCII_REMOVE.sub("", t)
        return t

    def files_prefix(self):
        return "%06d_%s_%s" % (self.id,
                             self.startofrendering_time.strftime("%Y-%m-%d_%H-%M"),
                             self.maptitle_computized())

    def start_rendering(self):
        self.status = 1
        self.startofrendering_time = datetime.now()
        self.save()

    def end_rendering(self, resultmsg):
        self.status = 2
        self.endofrendering_time = datetime.now()
        self.resultmsg = resultmsg
        self.save()

    def is_waiting(self):
        return self.status == 0

    def is_rendering(self):
        return self.status == 1

    def is_done(self):
        return self.status == 2

    def is_done_ok(self):
        return self.is_done() and self.resultmsg == "ok"

    def is_done_failed(self):
        return self.is_done() and self.resultmsg != "ok"

    def get_map_fileurl(self, format):
        return www.settings.RENDERING_RESULT_URL + "/" + self.files_prefix() + "." + format

    def get_map_filepath(self, format):
        return os.path.join(www.settings.RENDERING_RESULT_PATH, self.files_prefix() + "." + format)

    def get_index_fileurl(self, format):
        return www.settings.RENDERING_RESULT_URL + "/" + self.files_prefix() + "_index." + format

    def get_index_filepath(self, format):
        return os.path.join(www.settings.RENDERING_RESULT_PATH, self.files_prefix() + "_index." + format)

    def output_files(self):
        allfiles = []
        for format in www.settings.RENDERING_RESULT_FORMATS:
            if format != 'csv' and os.path.exists(self.get_map_filepath(format)):
                allfiles.append((self.get_map_fileurl(format),
                                 _("%(title)s %(format)s Map") \
                                     % { 'title' : self.maptitle,
                                         'format': format.upper() } ))
            if os.path.exists(self.get_index_filepath(format)):
                allfiles.append((self.get_index_fileurl(format),
                                 _("%(title)s %(format)s Index") % \
                                     { 'title': self.maptitle,
                                       'format': format.upper() }))
        return allfiles

    def get_thumbnail(self):
        thumbnail_file = os.path.join(www.settings.RENDERING_RESULT_PATH, self.files_prefix() + "_small.png")
        thumbnail_url = www.settings.RENDERING_RESULT_URL + "/" + self.files_prefix() + "_small.png"
        if os.path.exists(thumbnail_file):
            return thumbnail_url
        else:
            return None

    def current_position_in_queue(self):
        return MapRenderingJob.objects.filter(status=0).filter(index_queue_at_submission__lte=self.index_queue_at_submission).count()

    # Estimate the date at which the rendering will be started
    def rendering_estimated_start_time(self):
        waiting_time = datetime.now() - self.submission_time
        progression = self.index_queue_at_submission - self.current_position_in_queue()
        if progression == 0:
            return datetime.now()
        mean_job_rendering_time = waiting_time // progression
        estimated_time_left = mean_job_rendering_time * self.current_position_in_queue()
        return datetime.now() + estimated_time_left
<|endoftext|>


<commit_before>""" functions for logging and printing info """
import json
import logging
import logging.config
import os
import sys
from . import const

def start_logger(options):
    """ start the logger """
    config = {
        'version'                  : 1,
        'disable_existing_loggers' : False,
        'formatters': {
            'detailed': {
                'format': '%(asctime)s - %(levelname)s - %(message)s'
            },
            'mimimal': {
                'format': '%(message)s'
            }
        },
        'handlers': {
            'log_file_handler': {
                'class'        : 'logging.FileHandler',
                'level'        : 'DEBUG',
                'formatter'    : 'detailed',
                'filename'     : options['panzer']['debug'] + '.log',
                'encoding'     : const.ENCODING
            },
            'console': {
                'class'      : 'logging.StreamHandler',
                'level'      : 'INFO',
                'formatter'  : 'mimimal',
                'stream'     : 'ext://sys.stderr'
            }
        },
        'loggers': {
            __name__: {
                'handlers'   : ['console', 'log_file_handler'],
                'level'      : 'DEBUG',
                'propagate'  : True
            }
        }
    }
    # - check debug flag
    if not options['panzer']['debug']:
        config['loggers'][__name__]['handlers'].remove('log_file_handler')
        del config['handlers']['log_file_handler']
    else:
        # - delete old log file if it exists
        # - don't see value in keeping old logs here...
        filename = config['handlers']['log_file_handler']['filename']
        if os.path.exists(filename):
            os.remove(filename)
    # - set verbosity level
    verbosity = ['CRITICAL', 'WARNING', 'INFO']
    index = options['panzer'].get('verbose', 1)
    try:
        verbosity_level = verbosity[index]
    except IndexError:
        print('ERROR: Unknown setting for ---verbose. '
              'Setting ---verbose 2.', file=sys.stderr)
        verbosity_level = 'INFO'
    config['handlers']['console']['level'] = verbosity_level
    # - send configuration to logger
    logging.config.dictConfig(config)
    log('DEBUG', 'panzer', pretty_start_log('panzer starts'))
    log('DEBUG', 'panzer', pretty_title('OPTIONS'))
    log('DEBUG', 'panzer', pretty_json_repr(options))

def log(level_str, sender, message):
    """ send a log message """
    my_logger = logging.getLogger(__name__)
    # - lookup table for internal strings to logging levels
    levels = {
        'CRITICAL' : logging.CRITICAL,
        'ERROR'    : logging.ERROR,
        'WARNING'  : logging.WARNING,
        'INFO'     : logging.INFO,
        'DEBUG'    : logging.DEBUG,
        'NOTSET'   : logging.NOTSET
    }
    # - lookup table for internal strings to pretty output strings
    pretty_levels = {
        'CRITICAL' : 'FATAL:   ',
        'ERROR'    : 'ERROR:   ',
        'WARNING'  : 'WARNING: ',
        'INFO'     : '         ',
        'DEBUG'    : '         ',
        'NOTSET'   : '         '
    }
    message = str(message)
    sender_str = ''
    message_str = ''
    level = levels.get(level_str, levels['ERROR'])
    # -- level
    pretty_level_str = pretty_levels.get(level_str, pretty_levels['ERROR'])
    # -- sender
    if sender != 'panzer':
        # sender_str = '  ' + sender + ': '
        sender_str = '  '
    # -- message
    message_str = message
    output = ''
    output += pretty_level_str
    output += sender_str
    output += message_str
    my_logger.log(level, output)

def decode_stderr_json(stderr):
    """ return a list of decoded json messages in stderr """
    # - check for blank input
    if not stderr:
        # - nothing to do
        return list()
    # - split the input (based on newlines) into list of json strings
    output = list()
    for line in stderr.split('\n'):
        if not line:
            # - skip blank lines: no valid json or message to decode
            continue
        json_message = list()
        try:
            json_message = json.loads(line)
        except ValueError:
            # - if json cannot be decoded, just log as ERROR prefixed by '!'
            json_message = {'level': 'ERROR', 'message': '!' + line}
        output.append(json_message)
    return output

def log_stderr(stderr, sender=str()):
    """ send a log from external executable """
    # 1. check for blank input
    if not stderr:
        # - nothing to do
        return
    # 2. get a string with sender's name
    if sender:
        # - remove file extension from sender's name if present
        sender = os.path.splitext(sender)[0]
    # 3. now handle the messages sent by sender
    json_message = decode_stderr_json(stderr)
    for item in json_message:
        level = item['level']
        message = item['message']
        log(level, sender, message)

def pretty_keys(dictionary):
    """ return pretty printed list of dictionary keys, num per line """
    if not dictionary:
        return []
    # - number of keys printed per line
    num = 5
    # - turn into sorted list
    keys = list(dictionary.keys())
    keys.sort()
    # - fill with blank elements to width num
    missing = num - (len(keys) % num)
    keys.extend([''] * missing)
    # - turn into 2D matrix
    matrix = [[keys[i+j] for i in range(0, num)]
              for j in range(0, len(keys), num)]
    # - calculate max width for each column
    len_matrix = [[len(col) for col in row] for row in matrix]
    max_len_col = [max([row[j] for row in len_matrix])
                   for j in range(0, num)]
    # - pad with spaces
    matrix = [[row[j].ljust(max_len_col[j]) for j in range(0, num)]
              for row in matrix]
    # - return list of lines to print
    matrix = ['  '.join(row) for row in matrix]
    return matrix

def pretty_list(input_list, separator=', '):
    """ return pretty printed list """
    output = '  %s' % separator.join(input_list)
    return output

def pretty_json_repr(data):
    """ return pretty printed data as a json """
    return json.dumps(data, sort_keys=True, indent=2)

def pretty_title(title):
    """ return pretty printed section title """
    output = '-' * 5 + ' ' + title.lower() + ' ' + '-' * 5
    return output

def pretty_start_log(title):
    """ return pretty printed title for starting log """
    output = '>' * 10 + ' ' + title + ' ' + '<' * 10
    return output

def pretty_end_log(title):
    """ return pretty printed title for ending log """
    output = '>' * 10 + ' ' + title + ' ' + '<' * 10 + '\n\n'
    return output

def pretty_path(input_path):
    """ return path string replacing '~' for home directory """
    home_path = os.path.expanduser('~')
    cwd_path = os.getcwd()
    output_path = input_path.replace(home_path, '~').replace(cwd_path, './')
    return output_path

def pretty_runlist(runlist):
    """ return pretty printed runlist """
    if not runlist:
        return ['  empty']
    max_num = len(runlist)
    output = list()
    current_kind = str()
    for i, entry in enumerate(runlist):
        if current_kind != entry['kind']:
            output.append(entry['kind'] + ':')
            current_kind = entry['kind']
        path = entry['command']
        basename = os.path.splitext(os.path.basename(path))[0]
        line = '  [%d/%d] %s' % (i+1, max_num, basename)
        line = line.ljust(30)
        line += '"%s"' % pretty_path(path)
        output.append(line)
    return output

def pretty_runlist_entry(num, max_num, path):
    """ return pretty printed run list entry """
    basename = os.path.splitext(os.path.basename(path))[0]
    line = '[%d/%d] %s' % (num+1, max_num, basename)
    # line = line.ljust(30)
    # line += '"%s"' % pretty_path(path)
    return line
<commit_msg>fixed error in pretty_keys<commit_after>""" functions for logging and printing info """
import json
import logging
import logging.config
import os
import sys
from . import const

def start_logger(options):
    """ start the logger """
    config = {
        'version'                  : 1,
        'disable_existing_loggers' : False,
        'formatters': {
            'detailed': {
                'format': '%(asctime)s - %(levelname)s - %(message)s'
            },
            'mimimal': {
                'format': '%(message)s'
            }
        },
        'handlers': {
            'log_file_handler': {
                'class'        : 'logging.FileHandler',
                'level'        : 'DEBUG',
                'formatter'    : 'detailed',
                'filename'     : options['panzer']['debug'] + '.log',
                'encoding'     : const.ENCODING
            },
            'console': {
                'class'      : 'logging.StreamHandler',
                'level'      : 'INFO',
                'formatter'  : 'mimimal',
                'stream'     : 'ext://sys.stderr'
            }
        },
        'loggers': {
            __name__: {
                'handlers'   : ['console', 'log_file_handler'],
                'level'      : 'DEBUG',
                'propagate'  : True
            }
        }
    }
    # - check debug flag
    if not options['panzer']['debug']:
        config['loggers'][__name__]['handlers'].remove('log_file_handler')
        del config['handlers']['log_file_handler']
    else:
        # - delete old log file if it exists
        # - don't see value in keeping old logs here...
        filename = config['handlers']['log_file_handler']['filename']
        if os.path.exists(filename):
            os.remove(filename)
    # - set verbosity level
    verbosity = ['CRITICAL', 'WARNING', 'INFO']
    index = options['panzer'].get('verbose', 1)
    try:
        verbosity_level = verbosity[index]
    except IndexError:
        print('ERROR: Unknown setting for ---verbose. '
              'Setting ---verbose 2.', file=sys.stderr)
        verbosity_level = 'INFO'
    config['handlers']['console']['level'] = verbosity_level
    # - send configuration to logger
    logging.config.dictConfig(config)
    log('DEBUG', 'panzer', pretty_start_log('panzer starts'))
    log('DEBUG', 'panzer', pretty_title('OPTIONS'))
    log('DEBUG', 'panzer', pretty_json_repr(options))

def log(level_str, sender, message):
    """ send a log message """
    my_logger = logging.getLogger(__name__)
    # - lookup table for internal strings to logging levels
    levels = {
        'CRITICAL' : logging.CRITICAL,
        'ERROR'    : logging.ERROR,
        'WARNING'  : logging.WARNING,
        'INFO'     : logging.INFO,
        'DEBUG'    : logging.DEBUG,
        'NOTSET'   : logging.NOTSET
    }
    # - lookup table for internal strings to pretty output strings
    pretty_levels = {
        'CRITICAL' : 'FATAL:   ',
        'ERROR'    : 'ERROR:   ',
        'WARNING'  : 'WARNING: ',
        'INFO'     : '         ',
        'DEBUG'    : '         ',
        'NOTSET'   : '         '
    }
    message = str(message)
    sender_str = ''
    message_str = ''
    level = levels.get(level_str, levels['ERROR'])
    # -- level
    pretty_level_str = pretty_levels.get(level_str, pretty_levels['ERROR'])
    # -- sender
    if sender != 'panzer':
        # sender_str = '  ' + sender + ': '
        sender_str = '  '
    # -- message
    message_str = message
    output = ''
    output += pretty_level_str
    output += sender_str
    output += message_str
    my_logger.log(level, output)

def decode_stderr_json(stderr):
    """ return a list of decoded json messages in stderr """
    # - check for blank input
    if not stderr:
        # - nothing to do
        return list()
    # - split the input (based on newlines) into list of json strings
    output = list()
    for line in stderr.split('\n'):
        if not line:
            # - skip blank lines: no valid json or message to decode
            continue
        json_message = list()
        try:
            json_message = json.loads(line)
        except ValueError:
            # - if json cannot be decoded, just log as ERROR prefixed by '!'
            json_message = {'level': 'ERROR', 'message': '!' + line}
        output.append(json_message)
    return output

def log_stderr(stderr, sender=str()):
    """ send a log from external executable """
    # 1. check for blank input
    if not stderr:
        # - nothing to do
        return
    # 2. get a string with sender's name
    if sender:
        # - remove file extension from sender's name if present
        sender = os.path.splitext(sender)[0]
    # 3. now handle the messages sent by sender
    json_message = decode_stderr_json(stderr)
    for item in json_message:
        level = item['level']
        message = item['message']
        log(level, sender, message)

def pretty_keys(dictionary):
    """ return pretty printed list of dictionary keys, num per line """
    if not dictionary:
        return []
    # - number of keys printed per line
    num = 5
    # - turn into sorted list
    keys = list(dictionary.keys())
    keys.sort()
    # - fill with blank elements to width num
    missing = (len(keys) % num)
    if missing != 0:
        to_add = num - missing
        keys.extend([''] * to_add)
    # - turn into 2D matrix
    matrix = [[keys[i+j] for i in range(0, num)]
              for j in range(0, len(keys), num)]
    # - calculate max width for each column
    len_matrix = [[len(col) for col in row] for row in matrix]
    max_len_col = [max([row[j] for row in len_matrix])
                   for j in range(0, num)]
    # - pad with spaces
    matrix = [[row[j].ljust(max_len_col[j]) for j in range(0, num)]
              for row in matrix]
    # - return list of lines to print
    matrix = ['  '.join(row) for row in matrix]
    return matrix

def pretty_list(input_list, separator=', '):
    """ return pretty printed list """
    output = '  %s' % separator.join(input_list)
    return output

def pretty_json_repr(data):
    """ return pretty printed data as a json """
    return json.dumps(data, sort_keys=True, indent=2)

def pretty_title(title):
    """ return pretty printed section title """
    output = '-' * 5 + ' ' + title.lower() + ' ' + '-' * 5
    return output

def pretty_start_log(title):
    """ return pretty printed title for starting log """
    output = '>' * 10 + ' ' + title + ' ' + '<' * 10
    return output

def pretty_end_log(title):
    """ return pretty printed title for ending log """
    output = '>' * 10 + ' ' + title + ' ' + '<' * 10 + '\n\n'
    return output

def pretty_path(input_path):
    """ return path string replacing '~' for home directory """
    home_path = os.path.expanduser('~')
    cwd_path = os.getcwd()
    output_path = input_path.replace(home_path, '~').replace(cwd_path, './')
    return output_path

def pretty_runlist(runlist):
    """ return pretty printed runlist """
    if not runlist:
        return ['  empty']
    max_num = len(runlist)
    output = list()
    current_kind = str()
    for i, entry in enumerate(runlist):
        if current_kind != entry['kind']:
            output.append(entry['kind'] + ':')
            current_kind = entry['kind']
        path = entry['command']
        basename = os.path.splitext(os.path.basename(path))[0]
        line = '  [%d/%d] %s' % (i+1, max_num, basename)
        line = line.ljust(30)
        line += '"%s"' % pretty_path(path)
        output.append(line)
    return output

def pretty_runlist_entry(num, max_num, path):
    """ return pretty printed run list entry """
    basename = os.path.splitext(os.path.basename(path))[0]
    line = '[%d/%d] %s' % (num+1, max_num, basename)
    # line = line.ljust(30)
    # line += '"%s"' % pretty_path(path)
    return line
<|endoftext|>


<commit_before>"""
Plot and PlotTemplate classes.

Plot creates the 3 matrices necessary for plotting a RGB image,
from a three-tuple (channels) and a dictionnary of SheetViews.

The channels specify the key to find the SheetViews that have to be retrieved
for this purpose.

The plot objects are listed within a PlotGroup Class which then arranges
multiple plots into a single figure in a PlotGroupPanel.

Associated with this single (constructed) box of information are multiple histograms that
are stored within the base plot object. (Not yet implemented.)

$Id$
"""
__version__='$Revision$'

### JCALERT!
###
### The code in this file is still being reviewed, and may still need
### substantial changes. Particularly, the doc has to be reviewed.

### JCALERT! Are these 3 import statements really used?
import sys
import types
import MLab

from topo.base.sheetview import *
from Numeric import zeros, ones, Float, divide
from topo.base.topoobject import TopoObject
from bitmap import matrix_hsv_to_rgb, WHITE_BACKGROUND, BLACK_BACKGROUND
from topo.base.parameter import Dynamic
import palette as palette

### JC: I think it can go...
from Numeric import array

### JCALERT! WHAT about histograms? (ask Jim)
#from histogram import Histogram

### JCALERT! The histograms should be implemented and an object histograms assign to a Plot()
### WHAT does bitmap really mean here? (ask Jim and review the doc)

class Plot(TopoObject):
    """
    Class that constructs a bitmap plot from one or more
    SheetViews (optionally including a Histogram).  The bitmap is just
    stored for future use, e.g. as part of a PlotGroup of related
    plots displayed within one GUI window.
    """
    background = Dynamic(default=BLACK_BACKGROUND)
    palette_ = Dynamic(default=palette.Monochrome)

    ### JCALERT! - Pass the content of a template instead of channels?
    ### Or pass a list of three tuples? To be fixed with Jim.
    ### Also: - put normalize in PlotGroup?

    def __init__(self,(channel_1, channel_2, channel_3),sheet_view_dict,
                 normalize=False, **params):
        """
        (channel_1, channel_2, channel_3) are keys that point to a stored
        SheetView in sheet_view_dict, as specified in a PlotTemplate.
        None is also possible if the Plot is only built from one or two
        SheetViews.

	sheet_view_dict is a dictionary of SheetViews, generally belonging
        to a Sheet object but not necessarily.

	normalize specified is the Plot is normalized or not.

        a 'name' parameter is inherited from TopoObject.

        TODO: implement the possibility to pass more than three
        SheetView keys for more complicated plots (12/01/05)
        """
        ### JCALERT! That was in the doc: WHAT ABOUT BOUNDINGREGIONS?
        ### Is it important? (Ask Jim)

        super(Plot,self).__init__(**params)

      	self.view_dict = sheet_view_dict
        self.channels = (channel_1, channel_2, channel_3)

	### JC: maybe we can use the parameter name instead of having a view_info?
        self.view_info = {}

        self.cropped = False
        self.histograms = []

	### Maybe changed the name of channel_views to be view_list?
        self.channel_views = []
        self.matrices = []              # Will finally hold 3 matrices (or more?)
        self.normalize = normalize

        if self.view_dict == None:
            raise ValueError("A Plot should be passed a sheet_view_dict when created")

	### JCALERT! maybe also raise something when channels are all None?

    # JC: This function is called by load_images in PlotGroup and sub-classes.
    ### Maybe it should just be called from __init__ ?
    def plot(self):
        """
        Get the SheetViews requested from each channel passed in at
        creation, combines them as necessary, (and generates the
        Histograms, not yet implemented).  plot() is here to allow dynamic creation of plots,
        even after the creation of the Plot object.

        Returns: self (Plot)

        Important features of this object are:

	   self.matrices: Triple of Matrices mapping to the
	                  R/G/B channels.
           self.view_info: Name information that can be used to create
                           labels and filenames.
           (self.histograms: List of Histogram objects associated with
                    the matrices is self.matrices. NOT YET IMPLEMENTED)
        """

	self._get_channel_views_from_view_dict()

	### JC: write this function to cut the long code.
	#self._get_matrices_from_channel_views()

        ### JABALERT! Need to document what this code does.

        ### JCALERT! The code below can be made clearer.
        ### Especially, the passing None in channel_views can be changed.
	### Also, this can be put in another function...

	shape = (0,0)
        self.debug('self.channel_views = ' + str(self.channel_views))

	self.matrices=[]
        for each in self.channel_views:

	    if each != None:
		view_matrix = each.view()
		self.matrices.append(view_matrix[0]) # Discards boundingbox
		shape = view_matrix[0].shape
	    else:
		self.matrices.append(None)

# 	if self.matrices[0] == None:
# 	    raise ValueError("EmptyPlot")
# 	else:
# 	    shape = self.matrices[0].shape

        ### JC: For clarity, this could be another function.

	    # By this point, self.matrices should be a triple of 2D
        # matrices trimmed and ready to go to the caller of this
        # function ... after a possible conversion to a different
        # palette form.

	s,h,c = self.matrices

        zero=zeros(shape,Float)
        one=ones(shape,Float)

        ### JABHACKALERT Need to extend for white background; assumes black

        ### JCALERT! This part of the code could go. To see.
        # No plot; in the future should probably not be a warning
        if (s==None and c==None and h==None):
            self.debug('Skipping empty plot.')
            return None

            # Determine appropriate defaults for each matrix
        if s is None: s=one # Treat as full strength by default
        if c is None: c=one # Treat as full confidence by default
        if h is None: # No color -- should be changed to drop down to COLORMAP plot.
            h=zero
            c=zero

        if self.normalize and max(s.flat) > 0:
            s = divide(s,float(max(s.flat)))

        hue=h
        sat=c
        val=s

        if max(hue.flat) > 1 or max(sat.flat) > 1 or max(val.flat) > 1:
            self.cropped = True
        #self.warning('Plot: HSVMap inputs exceed 1. Clipping to 1.0')
        if max(hue.flat) > 0: hue = MLab.clip(hue,0.0,1.0)
        if max(sat.flat) > 0: sat = MLab.clip(sat,0.0,1.0)
        if max(val.flat) > 0: val = MLab.clip(val,0.0,1.0)
        else:
            self.cropped = False

        ### JCALERT! This function ought to be in Plot rather than bitmap.py
        self.matrices = matrix_hsv_to_rgb(hue,sat,val)

        return self

    def _get_channel_views_from_view_dict(self):
        """sub-function of plot() that just retrieve the views from the view_dict
           as specified by the channels, and create channel_views (list of sheet_view).

           (It just deal with the fact that channels can be None, or that the keys
            specified by channels can potentially refer to no SheetViews in the dict).
        """
	self.channel_views=[]
        for each in self.channels:
	    ### JCALERT! Presumably, if each == None then there is no key equal to None in the
            ### dict and then sv==None. Ask Jim if it is alright, because it is always possible that
            ### some mad man has done dict[None]=who_knows_what ?
	    sv = self.view_dict.get(each, None)
	    if sv == None:
		self.channel_views.append(None)
	    else:
		self.channel_views.append(sv)
	    ### JCALERT ! This is an hack so that the problem of displaying the right
	    ### name under each map in activity and orientation map panel is solved
	    ### It has to be changed so that it display what we want for each panel
	    ### Also I think the name that is displayed should always be the plot name
            ### (Also see in create_plots for each PlotGroup sub-classes)
		self.view_info['src_name'] = sv.view_info['src_name'] + self.name
		self.view_info['view_type'] = sv.view_info['view_type']

    ### JCALERT! Actually, this function can be used this way.
    ### It is called from release_sheetviews in PlotGroup and enable to
    ### free memory space when we don't need the SheetViews entry anymore.
    def release_sheetviews(self):
        """
        Delete any sheet_view_dict entries used by this plot, under
        the assumption that this Plot is the only object that use the
        the SheetView in the sheet_view_dict  with that dictionary key.
        """
        for each in self.channels:
	    del self.view_dict[each]

<commit_msg>Continued to simplified plot.py. I created one more sub-function called _make_hsv_matrices in order to cut plot() in more comprehensive part. I also simplified the code by deleting unused tests unused plot attribute (channel_views does not exist anymore as we only need to keep a list of matrices but not the sheet_views itself, therefore _get_channel_views_from_sheet_view() is now _get_matrices_from_sheet_view()). Added some doc and comments, deleted some old alerts.<commit_after>"""
Plot and PlotTemplate classes.

Plot creates the 3 matrices necessary for plotting a RGB image,
from a three-tuple (channels) and a dictionnary of SheetViews.

The channels specify the key to find the SheetViews that have to be retrieved
for this purpose.

The plot objects are listed within a PlotGroup Class which then arranges
multiple plots into a single figure in a PlotGroupPanel.

Associated with this single (constructed) box of information are multiple histograms that
are stored within the base plot object. (Not yet implemented.)

$Id$
"""
__version__='$Revision$'

### JCALERT!
###
### The code in this file is still being reviewed, and may still need
### substantial changes. Particularly, the doc has to be reviewed.

### JCALERT! Are these 3 import statements really used?
import sys
import types
import MLab

### JC: I think it can go, to check:
from topo.base.sheetview import *

from Numeric import zeros, ones, Float, divide
from topo.base.topoobject import TopoObject
from bitmap import matrix_hsv_to_rgb, WHITE_BACKGROUND, BLACK_BACKGROUND
from topo.base.parameter import Dynamic
import palette as palette

### JC: I think it can go...
from Numeric import array

### JCALERT! WHAT about histograms? (ask Jim)
#from histogram import Histogram

### JCALERT! The histograms should be implemented and an object histograms assign to a Plot()
### WHAT does bitmap really mean here? (ask Jim and review the doc)

class Plot(TopoObject):
    """
    Class that constructs a bitmap plot from one or more
    SheetViews (optionally including a Histogram).  The bitmap is just
    stored for future use, e.g. as part of a PlotGroup of related
    plots displayed within one GUI window.
    """
    background = Dynamic(default=BLACK_BACKGROUND)
    palette_ = Dynamic(default=palette.Monochrome)

    ### JCALERT! - Pass the content of a template instead of channels?
    ### Or pass a list of three tuples? To be fixed with Jim.
    ### Also: - put normalize in PlotGroup?

    def __init__(self,(channel_1, channel_2, channel_3),sheet_view_dict,
                 normalize=False, **params):
        """
        (channel_1, channel_2, channel_3) are keys that point to a stored
        SheetView in sheet_view_dict, as specified in a PlotTemplate.
        None is also possible if the Plot is only built from one or two
        SheetViews.

	sheet_view_dict is a dictionary of SheetViews, generally belonging
        to a Sheet object but not necessarily.

	normalize specified is the Plot is normalized or not.

        a 'name' parameter is inherited from TopoObject.

        TODO: implement the possibility to pass more than three
        SheetView keys for more complicated plots (12/01/05)
        """
        ### JCALERT! That was in the doc: WHAT ABOUT BOUNDINGREGIONS?
        ### Is it important? (Ask Jim)

        super(Plot,self).__init__(**params)

      	self.view_dict = sheet_view_dict
        self.channels = (channel_1, channel_2, channel_3)

	### JC: maybe we can use the parameter name instead of having a view_info?
        self.view_info = {}

        self.cropped = False
        self.histograms = []

        self.matrices = []              # Will finally hold 3 matrices (or more?)
        self.normalize = normalize

        if self.view_dict == None:
            raise ValueError("A Plot should be passed a sheet_view_dict when created")

	### JCALERT! maybe also raise something when channels are all None?

    # JC: This function is called by load_images in PlotGroup and sub-classes.
    ### Maybe it should just be called from __init__ ?
    def plot(self):
        """
        Get the SheetViews requested from each channel passed in at
        creation, combines them as necessary, (and generates the
        Histograms, not yet implemented).  plot() is here to allow dynamic creation of plots,
        even after the creation of the Plot object.

        Returns: self (Plot)

        Important features of this object are:

	   self.matrices: Triple of Matrices mapping to the
	                  R/G/B channels.
           self.view_info: Name information that can be used to create
                           labels and filenames.
           (self.histograms: List of Histogram objects associated with
                    the matrices is self.matrices. NOT YET IMPLEMENTED)
        """

	self._get_matrices_from_view_dict()

	### JCALERT ! The name should be changed...
	hsv_matrices = self._make_hsv_matrices()

	### JCALERT! Think about mae_hsv_matrices beeing a procedure
        ### directly working on self.matrices and change matrix_hsv_to_rgb to
        ### take a list of matrices or tuple of matrices.
	if hsv_matrices == None:
	    ### JCALERT! Why are there empty plot when doing orientation map?
            ### This should maybe not returned None in this case?
            ### It should come from the PlotGroup that looks into all sheets in the simulator
	    ### even the Retina...
	    #print("Empty Plot")
	    return None
	else:
	    (hue,sat,val)= hsv_matrices

	     ### JCALERT! This function ought to be in Plot rather than bitmap.py
	    self.matrices = matrix_hsv_to_rgb(hue,sat,val)
	    return self

    def _get_matrices_from_view_dict(self):
        """
	sub-function of plot() that just retrieve the views from the view_dict
	as specified by the channels, and create its matrices attribute accordingly
	(list of view matrices from SheetView.view() ).

	(It just deals with the fact that channels can be None, or that the keys
	specified by channels can potentially refer to no SheetViews in the dict).
        """
	self.matrices=[]
        for each in self.channels:
	    ### JCALERT! Presumably, if each == None then there is no key equal to None in the
            ### dict and then sv==None. Ask Jim if it is alright, because it is always possible that
            ### some mad man has done dict[None]=who_knows_what ?
	    sv = self.view_dict.get(each, None)
	    if sv == None:
		self.matrices.append(None)
	    else:
		view_matrix = sv.view()
		self.matrices.append(view_matrix[0]) # discard boudingbox

	    ### JCALERT ! This is an hack so that the problem of displaying the right
	    ### name under each map in activity and orientation map panel is solved
	    ### It has to be changed so that it display what we want for each panel
	    ### Also I think the name that is displayed should always be the plot name
            ### (Also see in create_plots for each PlotGroup sub-classes)
            ### This should not be in this function at all
		self.view_info['src_name'] = sv.view_info['src_name'] + self.name
		self.view_info['view_type'] = sv.view_info['view_type']

    ### JCALERT! The doc has to be reviewed (as well as the code by the way...)
    ### The function should be made as a procedure working on self.matrices directly...
    def _make_hsv_matrices(self):
	"""
	Sub-function that return the h,s,v matrices corresponding to
        self.matrices,
        also applying normalizing of the plot if required.
	"""
	shape=(0,0)
	for mat in self.matrices:
	    if mat != None:
		shape = mat.shape

	s,h,c = self.matrices

        zero=zeros(shape,Float)
        one=ones(shape,Float)

        ### JABHACKALERT Need to extend for white background; assumes black

        ### JCALERT! This part of the code could go. To see.
        # No plot; in the future should probably not be a warning
        if (s==None and c==None and h==None):
            self.debug('Skipping empty plot.')
            return None

            # Determine appropriate defaults for each matrix
        if s is None: s=one # Treat as full strength by default
        if c is None: c=one # Treat as full confidence by default
        if h is None: # No color -- should be changed to drop down to COLORMAP plot.
            h=zero
            c=zero

        if self.normalize and max(s.flat) > 0:
            s = divide(s,float(max(s.flat)))

        hue=h
        sat=c
        val=s

        if max(hue.flat) > 1 or max(sat.flat) > 1 or max(val.flat) > 1:
            self.cropped = True
        #self.warning('Plot: HSVMap inputs exceed 1. Clipping to 1.0')
        if max(hue.flat) > 0: hue = MLab.clip(hue,0.0,1.0)
        if max(sat.flat) > 0: sat = MLab.clip(sat,0.0,1.0)
        if max(val.flat) > 0: val = MLab.clip(val,0.0,1.0)
        else:
            self.cropped = False

	return (hue,sat,val)

    ### JCALERT! Actually, this function can be used this way.
    ### It is called from release_sheetviews in PlotGroup and enable to
    ### free memory space when we don't need the SheetViews entry anymore.
    def release_sheetviews(self):
        """
        Delete any sheet_view_dict entries used by this plot, under
        the assumption that this Plot is the only object that use the
        the SheetView in the sheet_view_dict  with that dictionary key.
        """
        for each in self.channels:
	    del self.view_dict[each]

<|endoftext|>