<commit_before>import datetime
import hashlib
import os
import re
import uuid
from exporters.writers.base_writer import BaseWriter

MD5_FILE_NAME = 'md5checksum.md5'

def md5_for_file(f, block_size=2**20):
    md5 = hashlib.md5()
    while True:
        data = f.read(block_size)
        if not data:
            break
        md5.update(data)
    return md5.hexdigest()

class FilebaseBaseWriter(BaseWriter):
    """
    This writer is a base writer providing common methods to all file based writers

    - filebase
        Path to store the exported files

    """
    supported_options = {
        'filebase': {'type': basestring},
        'generate_md5': {'type': bool, 'default': False}
    }

    def __init__(self, options, *args, **kwargs):
        super(FilebaseBaseWriter, self).__init__(options, *args, **kwargs)
        self.filebase = self.get_filebase_with_date()
        self.written_files = {}
        self.md5_file_name = None
        self.last_written_file = None
        self.generate_md5 = self.read_option('generate_md5')

    def write(self, path, key, file_name=False):
        """
        Receive path to temp dump file and group key, and write it to the proper location.
        """
        raise NotImplementedError

    def get_file_suffix(self, path, prefix):
        """
        This method is a fallback to provide valid random filenames
        """
        return str(uuid.uuid4())

    def get_filebase_with_date(self):
        filebase = self.read_option('filebase')
        filebase = filebase.format(date=datetime.datetime.now())
        filebase = datetime.datetime.now().strftime(filebase)
        return filebase

    def create_filebase_name(self, group_info, extension='gz', file_name=None):
        """
        Returns filebase and file valid name
        """
        normalized = [re.sub('\W', '_', s) for s in group_info]
        filebase = self.filebase.format(groups=normalized)
        filebase_path, prefix = os.path.split(filebase)
        if not file_name:
            file_name = prefix + self.get_file_suffix(filebase_path, prefix) + '.' + extension
        return filebase_path, file_name

    def _get_md5(self, path):
        with open(path, 'r') as f:
            return md5_for_file(f)

    def _write(self, key):
        write_info = self.write_buffer.pack_buffer(key)
        self.write(write_info.get('compressed_path'), self.write_buffer.grouping_info[key]['membership'])
        write_info['md5'] = self._get_md5(write_info.get('compressed_path'))
        self.logger.info('Checksum for file {}: {}'.format(write_info['compressed_path'], write_info['md5']))
        self.written_files[self.last_written_file] = write_info
        self.write_buffer.clean_tmp_files(key, write_info.get('compressed_path'))

    def finish_writing(self):
        if self.generate_md5:
            try:
                with open(MD5_FILE_NAME, 'a') as f:
                    for file_name, write_info in self.written_files.iteritems():
                        write_info = self.written_files[file_name]
                        f.write('{} {}'.format(write_info['md5'], file_name)+'\n')
                self.write(MD5_FILE_NAME, None, file_name=MD5_FILE_NAME)
            finally:
                os.remove(MD5_FILE_NAME)
<commit_msg>Removing .format() for dates. Let's rely on filebase = datetime.datetime.now().strftime(filebase)<commit_after>import datetime
import hashlib
import os
import re
import uuid
from exporters.writers.base_writer import BaseWriter

MD5_FILE_NAME = 'md5checksum.md5'

def md5_for_file(f, block_size=2**20):
    md5 = hashlib.md5()
    while True:
        data = f.read(block_size)
        if not data:
            break
        md5.update(data)
    return md5.hexdigest()

class FilebaseBaseWriter(BaseWriter):
    """
    This writer is a base writer providing common methods to all file based writers

    - filebase
        Path to store the exported files

    """
    supported_options = {
        'filebase': {'type': basestring},
        'generate_md5': {'type': bool, 'default': False}
    }

    def __init__(self, options, *args, **kwargs):
        super(FilebaseBaseWriter, self).__init__(options, *args, **kwargs)
        self.filebase = self.get_filebase_with_date()
        self.written_files = {}
        self.md5_file_name = None
        self.last_written_file = None
        self.generate_md5 = self.read_option('generate_md5')

    def write(self, path, key, file_name=False):
        """
        Receive path to temp dump file and group key, and write it to the proper location.
        """
        raise NotImplementedError

    def get_file_suffix(self, path, prefix):
        """
        This method is a fallback to provide valid random filenames
        """
        return str(uuid.uuid4())

    def get_filebase_with_date(self):
        filebase = self.read_option('filebase')
        filebase = datetime.datetime.now().strftime(filebase)
        return filebase

    def create_filebase_name(self, group_info, extension='gz', file_name=None):
        """
        Returns filebase and file valid name
        """
        normalized = [re.sub('\W', '_', s) for s in group_info]
        filebase = self.filebase.format(groups=normalized)
        filebase_path, prefix = os.path.split(filebase)
        if not file_name:
            file_name = prefix + self.get_file_suffix(filebase_path, prefix) + '.' + extension
        return filebase_path, file_name

    def _get_md5(self, path):
        with open(path, 'r') as f:
            return md5_for_file(f)

    def _write(self, key):
        write_info = self.write_buffer.pack_buffer(key)
        self.write(write_info.get('compressed_path'), self.write_buffer.grouping_info[key]['membership'])
        write_info['md5'] = self._get_md5(write_info.get('compressed_path'))
        self.logger.info('Checksum for file {}: {}'.format(write_info['compressed_path'], write_info['md5']))
        self.written_files[self.last_written_file] = write_info
        self.write_buffer.clean_tmp_files(key, write_info.get('compressed_path'))

    def finish_writing(self):
        if self.generate_md5:
            try:
                with open(MD5_FILE_NAME, 'a') as f:
                    for file_name, write_info in self.written_files.iteritems():
                        write_info = self.written_files[file_name]
                        f.write('{} {}'.format(write_info['md5'], file_name)+'\n')
                self.write(MD5_FILE_NAME, None, file_name=MD5_FILE_NAME)
            finally:
                os.remove(MD5_FILE_NAME)
<|endoftext|>


<commit_before>from __future__ import (absolute_import, division, print_function,)
import numpy as np
import scipy.optimize as opt
from scipy import interpolate
import os
import sys
from cannon.helpers import Table
from cannon.dataset import Dataset
import matplotlib.pyplot as plt

# python 3 special
PY3 = sys.version_info[0] > 2
if not PY3:
    range = xrange

try:
    from astropy.io import fits as pyfits
except ImportError:
    import pyfits

class LamostDataset(Dataset):
    """ A class to represent a Dataset of LAMOST spectra and labels.

    Performs the LAMOST Munging necessary for making the data "Cannonizable."
    Retrieves the data (assumse already in rest frame), creates and reads bad-
    pixel mask, builds inverse variance vectors, packages them all into
    rectangular blocks.
    """

    def __init__(self, training_dir, test_dir, label_file):
        super(self.__class__, self).__init__(training_dir, test_dir, label_file)

    def _get_pixmask(self, file_in, middle, grid, flux, ivar):
        """ Return a mask array of bad pixels for one object's spectrum

        Bad pixels are defined as follows: fluxes or ivars are not finite, or
        ivars are negative

        Major sky lines. 4046, 4358, 5460, 5577, 6300, 6363, 6863

        Where the red and blue wings join together: 5800-6000

        Read bad pix mask: file_in[0].data[4] is the ormask

        Parameters
        ----------
        fluxes: ndarray
            flux array

        flux_errs: ndarray
            measurement uncertainties on fluxes

        Returns
        -------
        mask: ndarray, dtype=bool
            array giving bad pixels as True values
        """
        npix = len(grid)

        bad_flux = (~np.isfinite(flux))
        bad_err = (~np.isfinite(ivar)) | (ivar <= 0)
        bad_pix_a = bad_err | bad_flux

        # wings = np.logical_and(grid > 5750, grid < 6050)
        # ormask = (file_in[0].data[4] > 0)[middle]
        # bad_pix_b = wings | ormask

        # max_pix_width = grid[npix-1]-grid[npix-2]
        # skylines = np.array([4046, 4358, 5460, 5577, 6300, 6363, 6863])
        # bad_pix_c = np.zeros(npix, dtype=bool)
        # for skyline in skylines:
        #     badmin = skyline-max_pix_width
        #     badmax = skyline+max_pix_width
        #     bad_pix_temp = np.logical_and(grid > badmin, grid < badmax)
        #     bad_pix_c[bad_pix_temp] = True

        # bad_pix_ab = bad_pix_a | bad_pix_b
        # bad_pix = bad_pix_ab | bad_pix_c

        # return bad_pix
        return bad_pix_a

    def _load_spectra(self, data_dir):
        """
        Extracts spectra (wavelengths, fluxes, fluxerrs) from apogee fits files

        Returns
        -------
        IDs: list of length nstars
            stellar IDs

        wl: numpy ndarray of length npixels
            rest-frame wavelength vector

        fluxes: numpy ndarray of shape (nstars, npixels)
            training set or test set pixel intensities

        ivars: numpy ndarray of shape (nstars, npixels)
            inverse variances, parallel to fluxes

        SNRs: numpy ndarray of length nstars
        """
        print("Loading spectra from directory %s" %data_dir)
        files = list(sorted([data_dir + "/" + filename
                 for filename in os.listdir(data_dir)]))
        nstars = len(files)

        for jj, fits_file in enumerate(files):
            file_in = pyfits.open(fits_file)
            wl = np.array(file_in[0].data[2])
            if jj == 0:
                npixels = len(wl_temp)
                SNRs = np.zeros(nstars, dtype=float)
                # get rid of edges
                # middle = np.logical_and(grid > 4000, grid < 8800)
                middle = np.logical_and(grid>1000,grid<10000)
                # grid = grid[middle]
                npixels = len(grid)
                fluxes = np.zeros((nstars, npixels), dtype=float)
                ivars = np.zeros(fluxes.shape, dtype=float)
            redshift = file_in[0].header['Z']
            wlshifts = redshift*wl_temp
            wl = wl_temp - wlshifts
            flux = np.array(file_in[0].data[0])
            ivar = np.array((file_in[0].data[1]))
            flux_rs = (interpolate.interp1d(wl, flux))(grid)
            ivar_rs = (interpolate.interp1d(wl, ivar))(grid)
            badpix = self._get_pixmask(file_in, middle, grid, flux_rs, ivar_rs)
            flux_rs = np.ma.array(flux_rs, mask=badpix)
            ivar_rs = np.ma.array(ivar_rs, mask=badpix)
            SNRs[jj] = np.ma.median(flux_rs*ivar_rs**0.5)
            ivar_rs = np.ma.filled(ivar_rs, fill_value=0.)
            fluxes[jj,:] = flux_rs
            ivars[jj,:] = ivar_rs

        print("Spectra loaded")
        return files, grid, fluxes, ivars, SNRs

<commit_msg>interpolation for one star<commit_after>from __future__ import (absolute_import, division, print_function,)
import numpy as np
import scipy.optimize as opt
from scipy import interpolate
import os
import sys
from cannon.helpers import Table
from cannon.dataset import Dataset
import matplotlib.pyplot as plt

# python 3 special
PY3 = sys.version_info[0] > 2
if not PY3:
    range = xrange

try:
    from astropy.io import fits as pyfits
except ImportError:
    import pyfits

class LamostDataset(Dataset):
    """ A class to represent a Dataset of LAMOST spectra and labels.

    Performs the LAMOST Munging necessary for making the data "Cannonizable."
    Retrieves the data (assumse already in rest frame), creates and reads bad-
    pixel mask, builds inverse variance vectors, packages them all into
    rectangular blocks.
    """

    def __init__(self, training_dir, test_dir, label_file):
        super(self.__class__, self).__init__(training_dir, test_dir, label_file)

    def _get_pixmask(self, file_in, middle, grid, flux, ivar):
        """ Return a mask array of bad pixels for one object's spectrum

        Bad pixels are defined as follows: fluxes or ivars are not finite, or
        ivars are negative

        Major sky lines. 4046, 4358, 5460, 5577, 6300, 6363, 6863

        Where the red and blue wings join together: 5800-6000

        Read bad pix mask: file_in[0].data[4] is the ormask

        Parameters
        ----------
        fluxes: ndarray
            flux array

        flux_errs: ndarray
            measurement uncertainties on fluxes

        Returns
        -------
        mask: ndarray, dtype=bool
            array giving bad pixels as True values
        """
        npix = len(grid)

        bad_flux = (~np.isfinite(flux))
        bad_err = (~np.isfinite(ivar)) | (ivar <= 0)
        bad_pix_a = bad_err | bad_flux

        # wings = np.logical_and(grid > 5750, grid < 6050)
        # ormask = (file_in[0].data[4] > 0)[middle]
        # bad_pix_b = wings | ormask

        # max_pix_width = grid[npix-1]-grid[npix-2]
        # skylines = np.array([4046, 4358, 5460, 5577, 6300, 6363, 6863])
        # bad_pix_c = np.zeros(npix, dtype=bool)
        # for skyline in skylines:
        #     badmin = skyline-max_pix_width
        #     badmax = skyline+max_pix_width
        #     bad_pix_temp = np.logical_and(grid > badmin, grid < badmax)
        #     bad_pix_c[bad_pix_temp] = True

        # bad_pix_ab = bad_pix_a | bad_pix_b
        # bad_pix = bad_pix_ab | bad_pix_c

        # return bad_pix
        return bad_pix_a

    def _load_spectra(self, data_dir):
        """
        Extracts spectra (wavelengths, fluxes, fluxerrs) from apogee fits files

        Returns
        -------
        IDs: list of length nstars
            stellar IDs

        wl: numpy ndarray of length npixels
            rest-frame wavelength vector

        fluxes: numpy ndarray of shape (nstars, npixels)
            training set or test set pixel intensities

        ivars: numpy ndarray of shape (nstars, npixels)
            inverse variances, parallel to fluxes

        SNRs: numpy ndarray of length nstars
        """
        print("Loading spectra from directory %s" %data_dir)
        files = list(sorted([data_dir + "/" + filename
                 for filename in os.listdir(data_dir)]))
        nstars = len(files)

        for jj, fits_file in enumerate(files):
            file_in = pyfits.open(fits_file)
            if jj == 0:
                # all stars start out on the same wavelength grid
                grid = np.array(file_in[0].data[2])
                npixels = len(wl)
                SNRs = np.zeros(nstars, dtype=float)
                fluxes = np.zeros((nstars, npixels), dtype=float)
                ivars = np.zeros(fluxes.shape, dtype=float)
            # correct for radial velocity of star
            redshift = file_in[0].header['Z']
            wlshift = redshift*grid
            wl = grid - wlshift
            flux = np.array(file_in[0].data[0])
            ivar = np.array((file_in[0].data[1]))
            # resample onto a common grid
            # can only interpolate *within* the original range
            keep = grid > wl[0]
            flux_rs = (interpolate.interp1d(wl, flux))(grid[keep])
            ivar_rs = (interpolate.interp1d(wl, ivar))(grid[keep])
            badpix = self._get_pixmask(file_in, middle, grid, flux_rs, ivar_rs)
            flux_rs = np.ma.array(flux_rs, mask=badpix)
            ivar_rs = np.ma.array(ivar_rs, mask=badpix)
            SNRs[jj] = np.ma.median(flux_rs*ivar_rs**0.5)
            ivar_rs = np.ma.filled(ivar_rs, fill_value=0.)
            fluxes[jj,:] = flux_rs
            ivars[jj,:] = ivar_rs

        print("Spectra loaded")
        return files, grid, fluxes, ivars, SNRs

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Copyright (C) 2011 Adriano Monteiro Marques
#
# Author: Piotrek Wasilewski <wasilewski.piotrek@gmail.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from django.contrib.auth.models import User
from django.db import models
from django.db.models import permalink
from django.utils.translation import ugettext as _

from netadmin.permissions.utils import SharedObject

class NetworkObject(models.Model, SharedObject):
    """
    Abstract model class for objects like host or network.
    Every object belongs to specified user
    """
    name = models.CharField(max_length=250)
    description = models.TextField(blank=True)
    user = models.ForeignKey(User, blank=False, null=False)

    def __unicode__(self):
        return self.name

    def short_description(self, word_limit=15):
        words = self.description.split()
        if len(words) > word_limit:
            return '%s...' % ' '.join(words[:word_limit])
        else:
            return self.description

    def events(self):
        return self.event_set.all().order_by('-timestamp')

    def latest_event(self):
        return self.event_set.latest('timestamp')

    def api_list(self):
        return {
            'id': self.pk,
            'name': self.name
        }

    class Meta:
        abstract = True

class Host(NetworkObject):
    """The single host in the network
    """
    ipv4 = models.IPAddressField(verbose_name=_("IPv4 address"))
    ipv6 = models.CharField(max_length=39, verbose_name=_("IPv6 address"),
                            blank=True, null=True)

    @permalink
    def get_absolute_url(self):
        return ('host_detail', [str(self.pk)])

    def delete(self, *args, **kwargs):
        # delete all events related to this host
        # TODO
        # user should be asked if events should be deleted
        # or assigned to "dummy host"
        events = self.events()
        events.delete()

        # delete network-host relations
        related = self.networkhost_set.all()
        related.delete()

        super(Host, self).delete(*args, **kwargs)

    def networks(self):
        nethost = self.networkhost_set.all().only('id')
        nets = [nh.network.pk for nh in nethost]
        return Network.objects.filter(pk__in=nets)

    def fields(self):
        from netadmin.events.models import EventFieldsNotValid
        fields_list = []
        for event in self.events():
            try:
                for field in event.fields.keys():
                    if field not in fields_list:
                        fields_list.append(field)
            except EventFieldsNotValid:
                pass
        return fields_list

    def in_network(self, network):
        try:
            nh = self.networkhost_set.get(network=network)
        except NetworkHost.DoesNotExist:
            return False
        return True

    def api_detail(self):
        return {
            'host_id': self.pk,
            'host_name': self.name,
            'host_description': self.description,
            'ipv4': self.ipv4,
            'ipv6': self.ipv6
        }

class Network(NetworkObject):

    @permalink
    def get_absolute_url(self):
        return ('network_detail', [str(self.pk)])

    def delete(self, *args, **kwargs):
        related = NetworkHost.objects.filter(network=self)
        related.delete()
        super(Network, self).delete(*args, **kwargs)

    def hosts(self):
        """Returns all hosts in the network
        """
        related = self.networkhost_set.all()
        hosts_ids = [rel.host.pk for rel in related]
        return Host.objects.filter(pk__in=hosts_ids)

    def add_host(self, host):
        """Creates relation between host and network
        """
        try:
            self.networkhost_set.get(host=host)
        except NetworkHost.DoesNotExist:
            return NetworkHost.objects.create(network=self, host=host)

    def remove_host(self, host):
        """Removes relation between host and network
        """
        try:
            relation = self.networkhost_set.get(host=host)
        except NetworkHost.DoesNotExist:
            return
        relation.delete()

    def has_host(self, host):
        return True if host in self.hosts() else False

    def events(self):
        from netadmin.events.models import Event
        hosts = self.hosts()
        events = Event.objects.filter(source_host__in=list(hosts))
        return events.order_by('-timestamp')

    def api_detail(self):
        return {
            'network_id': self.pk,
            'network_name': self.name,
            'network_description': self.description
        }

class NetworkHost(models.Model):
    """
    Since one cannot use ManyToManyField type in GAE [1], we have to
    write extra model that will provide application with many-to-many
    relationship between networks and hosts.

    To ensure that after deleting host or network its relations will
    be removed too, we have to override delete() method for both
    Host and Network classes. Those methods should look like that:

    def delete(self, *args, **kwargs):
        related = Nself.networkhost_set.all()
        related.delete()
        super(Network, self).delete(*args, **kwargs)

    for Network class, and:

    def delete(self, *args, **kwargs):
        related = self.networkhost_set.all()
        related.delete()
        super(Host, self).delete(*args, **kwargs)

    for Host class.

    [1] http://www.allbuttonspressed.com/projects/djangoappengine
    """
    network = models.ForeignKey(Network)
    host = models.ForeignKey(Host)

<commit_msg>Added timezone field in Host model<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Copyright (C) 2011 Adriano Monteiro Marques
#
# Author: Amit Pal <amix.pal@gmail.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from django.contrib.auth.models import User
from django.db import models
from django.db.models import permalink
from django.utils.translation import ugettext as _
import datetime

from netadmin.permissions.utils import SharedObject

class NetworkObject(models.Model, SharedObject):
    """
    Abstract model class for objects like host or network.
    Every object belongs to specified user
    """
    name = models.CharField(max_length=250)
    description = models.TextField(blank=True)
    user = models.ForeignKey(User, blank=False, null=False)

    def __unicode__(self):
        return self.name

    def short_description(self, word_limit=15):
        words = self.description.split()
        if len(words) > word_limit:
            return '%s...' % ' '.join(words[:word_limit])
        else:
            return self.description

    def events(self):
        return self.event_set.all().order_by('-timestamp')

    def latest_event(self):
        return self.event_set.latest('timestamp')

    def api_list(self):
        return {
            'id': self.pk,
            'name': self.name
        }

    class Meta:
        abstract = True

class Host(NetworkObject):
    """The single host in the network
    """
    timezone = models.CharField(max_length = 30, null=True, blank=True)
    ipv4 = models.IPAddressField(verbose_name=_("IPv4 address"))
    ipv6 = models.CharField(max_length=39, verbose_name=_("IPv6 address"),
                            blank=True, null=True)

    @permalink
    def get_absolute_url(self):
        return ('host_detail', [str(self.pk)])

    def delete(self, *args, **kwargs):
        # delete all events related to this host
        # TODO
        # user should be asked if events should be deleted
        # or assigned to "dummy host"
        events = self.events()
        events.delete()

        # delete network-host relations
        related = self.networkhost_set.all()
        related.delete()

        super(Host, self).delete(*args, **kwargs)

    def networks(self):
        nethost = self.networkhost_set.all().only('id')
        nets = [nh.network.pk for nh in nethost]
        return Network.objects.filter(pk__in=nets)

    def fields(self):
        from netadmin.events.models import EventFieldsNotValid
        fields_list = []
        for event in self.events():
            try:
                for field in event.fields.keys():
                    if field not in fields_list:
                        fields_list.append(field)
            except EventFieldsNotValid:
                pass
        return fields_list

    def in_network(self, network):
        try:
            nh = self.networkhost_set.get(network=network)
        except NetworkHost.DoesNotExist:
            return False
        return True

    def api_detail(self):
        return {
            'host_id': self.pk,
            'host_name': self.name,
            'host_description': self.description,
            'ipv4': self.ipv4,
            'ipv6': self.ipv6
        }

class Network(NetworkObject):

    @permalink
    def get_absolute_url(self):
        return ('network_detail', [str(self.pk)])

    def delete(self, *args, **kwargs):
        related = NetworkHost.objects.filter(network=self)
        related.delete()
        super(Network, self).delete(*args, **kwargs)

    def hosts(self):
        """Returns all hosts in the network
        """
        related = self.networkhost_set.all()
        hosts_ids = [rel.host.pk for rel in related]
        return Host.objects.filter(pk__in=hosts_ids)

    def add_host(self, host):
        """Creates relation between host and network
        """
        try:
            self.networkhost_set.get(host=host)
        except NetworkHost.DoesNotExist:
            return NetworkHost.objects.create(network=self, host=host)

    def remove_host(self, host):
        """Removes relation between host and network
        """
        try:
            relation = self.networkhost_set.get(host=host)
        except NetworkHost.DoesNotExist:
            return
        relation.delete()

    def has_host(self, host):
        return True if host in self.hosts() else False

    def events(self):
        from netadmin.events.models import Event
        hosts = self.hosts()
        events = Event.objects.filter(source_host__in=list(hosts))
        return events.order_by('-timestamp')

    def api_detail(self):
        return {
            'network_id': self.pk,
            'network_name': self.name,
            'network_description': self.description
        }

class NetworkHost(models.Model):
    """
    Since one cannot use ManyToManyField type in GAE [1], we have to
    write extra model that will provide application with many-to-many
    relationship between networks and hosts.

    To ensure that after deleting host or network its relations will
    be removed too, we have to override delete() method for both
    Host and Network classes. Those methods should look like that:

    def delete(self, *args, **kwargs):
        related = Nself.networkhost_set.all()
        related.delete()
        super(Network, self).delete(*args, **kwargs)

    for Network class, and:

    def delete(self, *args, **kwargs):
        related = self.networkhost_set.all()
        related.delete()
        super(Host, self).delete(*args, **kwargs)

    for Host class.

    [1] http://www.allbuttonspressed.com/projects/djangoappengine
    """
    network = models.ForeignKey(Network)
    host = models.ForeignKey(Host)

<|endoftext|>


<commit_before>#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from neutron_lib.api.definitions import address_group
from neutron_lib.api.definitions import address_scope
from neutron_lib.api.definitions import agent as agent_def
from neutron_lib.api.definitions import allowedaddresspairs
from neutron_lib.api.definitions import auto_allocated_topology
from neutron_lib.api.definitions import availability_zone as az_def
from neutron_lib.api.definitions import default_subnetpools
from neutron_lib.api.definitions import dns
from neutron_lib.api.definitions import dns_domain_keywords
from neutron_lib.api.definitions import dns_domain_ports
from neutron_lib.api.definitions import expose_port_forwarding_in_fip
from neutron_lib.api.definitions import external_net
from neutron_lib.api.definitions import extra_dhcp_opt
from neutron_lib.api.definitions import extraroute
from neutron_lib.api.definitions import filter_validation
from neutron_lib.api.definitions import fip_pf_description
from neutron_lib.api.definitions import fip_pf_port_range
from neutron_lib.api.definitions import fip_port_details
from neutron_lib.api.definitions import floating_ip_port_forwarding
from neutron_lib.api.definitions import floatingip_pools
from neutron_lib.api.definitions import l3
from neutron_lib.api.definitions import l3_ext_gw_mode
from neutron_lib.api.definitions import logging
from neutron_lib.api.definitions import multiprovidernet
from neutron_lib.api.definitions import network_availability_zone
from neutron_lib.api.definitions import network_ip_availability
from neutron_lib.api.definitions import network_mtu
from neutron_lib.api.definitions import network_mtu_writable
from neutron_lib.api.definitions import pagination
from neutron_lib.api.definitions import port_device_profile
from neutron_lib.api.definitions import port_mac_address_regenerate
from neutron_lib.api.definitions import port_numa_affinity_policy
from neutron_lib.api.definitions import port_resource_request
from neutron_lib.api.definitions import port_security
from neutron_lib.api.definitions import portbindings
from neutron_lib.api.definitions import portbindings_extended as pbe_ext
from neutron_lib.api.definitions import project_id
from neutron_lib.api.definitions import provider_net
from neutron_lib.api.definitions import qos
from neutron_lib.api.definitions import qos_bw_limit_direction
from neutron_lib.api.definitions import qos_default
from neutron_lib.api.definitions import qos_gateway_ip
from neutron_lib.api.definitions import qos_rule_type_details
from neutron_lib.api.definitions import qos_rule_type_filter
from neutron_lib.api.definitions import qos_rules_alias
from neutron_lib.api.definitions import quota_check_limit
from neutron_lib.api.definitions import rbac_address_scope
from neutron_lib.api.definitions import rbac_security_groups
from neutron_lib.api.definitions import router_availability_zone as raz_def
from neutron_lib.api.definitions import security_groups_normalized_cidr
from neutron_lib.api.definitions import security_groups_remote_address_group
from neutron_lib.api.definitions import security_groups_shared_filtering
from neutron_lib.api.definitions import segment as seg_def
from neutron_lib.api.definitions import sorting
from neutron_lib.api.definitions import stateful_security_group
from neutron_lib.api.definitions import subnet_dns_publish_fixed_ip
from neutron_lib.api.definitions import subnet_service_types
from neutron_lib.api.definitions import trunk
from neutron_lib.api.definitions import vlantransparent
from neutron_lib import constants

from neutron.extensions import quotasv2_detail

# NOTE(russellb) This remains in its own file (vs constants.py) because we want
# to be able to easily import it and export the info without any dependencies
# on external imports.

# NOTE(russellb) If you update these lists, please also update
# doc/source/admin/ovn/features.rst and the current release note.
ML2_SUPPORTED_API_EXTENSIONS_OVN_L3 = [
    l3.ALIAS,
    extraroute.ALIAS,
    l3_ext_gw_mode.ALIAS,
    fip_port_details.ALIAS,
    floatingip_pools.ALIAS,
    pagination.ALIAS,
    'qos-fip',
    qos_gateway_ip.ALIAS,
    sorting.ALIAS,
    project_id.ALIAS,
    dns.ALIAS,
    dns_domain_keywords.ALIAS,
    dns_domain_ports.ALIAS,
    subnet_dns_publish_fixed_ip.ALIAS,
    agent_def.ALIAS,
    az_def.ALIAS,
    raz_def.ALIAS,
]
ML2_SUPPORTED_API_EXTENSIONS = [
    address_group.ALIAS,
    address_scope.ALIAS,
    agent_def.ALIAS,
    allowedaddresspairs.ALIAS,
    auto_allocated_topology.ALIAS,
    az_def.ALIAS,
    portbindings.ALIAS,
    pbe_ext.ALIAS,
    default_subnetpools.ALIAS,
    dns.ALIAS,
    external_net.ALIAS,
    extra_dhcp_opt.ALIAS,
    filter_validation.ALIAS,
    multiprovidernet.ALIAS,
    network_mtu.ALIAS,
    network_mtu_writable.ALIAS,
    network_availability_zone.ALIAS,
    network_ip_availability.ALIAS,
    port_device_profile.ALIAS,
    port_mac_address_regenerate.ALIAS,
    port_numa_affinity_policy.ALIAS,
    port_security.ALIAS,
    provider_net.ALIAS,
    port_resource_request.ALIAS,
    qos.ALIAS,
    qos_bw_limit_direction.ALIAS,
    qos_default.ALIAS,
    qos_rule_type_details.ALIAS,
    qos_rule_type_filter.ALIAS,
    qos_rules_alias.ALIAS,
    'quotas',
    quota_check_limit.ALIAS,
    quotasv2_detail.ALIAS,
    rbac_address_scope.ALIAS,
    'rbac-policies',
    rbac_security_groups.ALIAS,
    'standard-attr-revisions',
    'security-group',
    security_groups_normalized_cidr.ALIAS,
    security_groups_remote_address_group.ALIAS,
    security_groups_shared_filtering.ALIAS,
    stateful_security_group.ALIAS,
    'standard-attr-description',
    constants.SUBNET_ALLOCATION_EXT_ALIAS,
    'standard-attr-tag',
    'standard-attr-timestamp',
    subnet_service_types.ALIAS,
    trunk.ALIAS,
    seg_def.ALIAS,
    expose_port_forwarding_in_fip.ALIAS,
    fip_pf_description.ALIAS,
    fip_pf_port_range.ALIAS,
    floating_ip_port_forwarding.ALIAS,
    vlantransparent.ALIAS,
    logging.ALIAS,
]
<commit_msg>Add vpnaas to extensions supported by ovn<commit_after>#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from neutron_lib.api.definitions import address_group
from neutron_lib.api.definitions import address_scope
from neutron_lib.api.definitions import agent as agent_def
from neutron_lib.api.definitions import allowedaddresspairs
from neutron_lib.api.definitions import auto_allocated_topology
from neutron_lib.api.definitions import availability_zone as az_def
from neutron_lib.api.definitions import default_subnetpools
from neutron_lib.api.definitions import dns
from neutron_lib.api.definitions import dns_domain_keywords
from neutron_lib.api.definitions import dns_domain_ports
from neutron_lib.api.definitions import expose_port_forwarding_in_fip
from neutron_lib.api.definitions import external_net
from neutron_lib.api.definitions import extra_dhcp_opt
from neutron_lib.api.definitions import extraroute
from neutron_lib.api.definitions import filter_validation
from neutron_lib.api.definitions import fip_pf_description
from neutron_lib.api.definitions import fip_pf_port_range
from neutron_lib.api.definitions import fip_port_details
from neutron_lib.api.definitions import floating_ip_port_forwarding
from neutron_lib.api.definitions import floatingip_pools
from neutron_lib.api.definitions import l3
from neutron_lib.api.definitions import l3_ext_gw_mode
from neutron_lib.api.definitions import logging
from neutron_lib.api.definitions import multiprovidernet
from neutron_lib.api.definitions import network_availability_zone
from neutron_lib.api.definitions import network_ip_availability
from neutron_lib.api.definitions import network_mtu
from neutron_lib.api.definitions import network_mtu_writable
from neutron_lib.api.definitions import pagination
from neutron_lib.api.definitions import port_device_profile
from neutron_lib.api.definitions import port_mac_address_regenerate
from neutron_lib.api.definitions import port_numa_affinity_policy
from neutron_lib.api.definitions import port_resource_request
from neutron_lib.api.definitions import port_security
from neutron_lib.api.definitions import portbindings
from neutron_lib.api.definitions import portbindings_extended as pbe_ext
from neutron_lib.api.definitions import project_id
from neutron_lib.api.definitions import provider_net
from neutron_lib.api.definitions import qos
from neutron_lib.api.definitions import qos_bw_limit_direction
from neutron_lib.api.definitions import qos_default
from neutron_lib.api.definitions import qos_gateway_ip
from neutron_lib.api.definitions import qos_rule_type_details
from neutron_lib.api.definitions import qos_rule_type_filter
from neutron_lib.api.definitions import qos_rules_alias
from neutron_lib.api.definitions import quota_check_limit
from neutron_lib.api.definitions import rbac_address_scope
from neutron_lib.api.definitions import rbac_security_groups
from neutron_lib.api.definitions import router_availability_zone as raz_def
from neutron_lib.api.definitions import security_groups_normalized_cidr
from neutron_lib.api.definitions import security_groups_remote_address_group
from neutron_lib.api.definitions import security_groups_shared_filtering
from neutron_lib.api.definitions import segment as seg_def
from neutron_lib.api.definitions import sorting
from neutron_lib.api.definitions import stateful_security_group
from neutron_lib.api.definitions import subnet_dns_publish_fixed_ip
from neutron_lib.api.definitions import subnet_service_types
from neutron_lib.api.definitions import trunk
from neutron_lib.api.definitions import vlantransparent
from neutron_lib.api.definitions import vpn
from neutron_lib.api.definitions import vpn_endpoint_groups
from neutron_lib import constants

from neutron.extensions import quotasv2_detail

# NOTE(russellb) This remains in its own file (vs constants.py) because we want
# to be able to easily import it and export the info without any dependencies
# on external imports.

# NOTE(russellb) If you update these lists, please also update
# doc/source/admin/ovn/features.rst and the current release note.
ML2_SUPPORTED_API_EXTENSIONS_OVN_L3 = [
    l3.ALIAS,
    extraroute.ALIAS,
    l3_ext_gw_mode.ALIAS,
    fip_port_details.ALIAS,
    floatingip_pools.ALIAS,
    pagination.ALIAS,
    'qos-fip',
    qos_gateway_ip.ALIAS,
    sorting.ALIAS,
    project_id.ALIAS,
    dns.ALIAS,
    dns_domain_keywords.ALIAS,
    dns_domain_ports.ALIAS,
    subnet_dns_publish_fixed_ip.ALIAS,
    agent_def.ALIAS,
    az_def.ALIAS,
    raz_def.ALIAS,
]
ML2_SUPPORTED_API_EXTENSIONS = [
    address_group.ALIAS,
    address_scope.ALIAS,
    agent_def.ALIAS,
    allowedaddresspairs.ALIAS,
    auto_allocated_topology.ALIAS,
    az_def.ALIAS,
    portbindings.ALIAS,
    pbe_ext.ALIAS,
    default_subnetpools.ALIAS,
    dns.ALIAS,
    external_net.ALIAS,
    extra_dhcp_opt.ALIAS,
    filter_validation.ALIAS,
    multiprovidernet.ALIAS,
    network_mtu.ALIAS,
    network_mtu_writable.ALIAS,
    network_availability_zone.ALIAS,
    network_ip_availability.ALIAS,
    port_device_profile.ALIAS,
    port_mac_address_regenerate.ALIAS,
    port_numa_affinity_policy.ALIAS,
    port_security.ALIAS,
    provider_net.ALIAS,
    port_resource_request.ALIAS,
    qos.ALIAS,
    qos_bw_limit_direction.ALIAS,
    qos_default.ALIAS,
    qos_rule_type_details.ALIAS,
    qos_rule_type_filter.ALIAS,
    qos_rules_alias.ALIAS,
    'quotas',
    quota_check_limit.ALIAS,
    quotasv2_detail.ALIAS,
    rbac_address_scope.ALIAS,
    'rbac-policies',
    rbac_security_groups.ALIAS,
    'standard-attr-revisions',
    'security-group',
    security_groups_normalized_cidr.ALIAS,
    security_groups_remote_address_group.ALIAS,
    security_groups_shared_filtering.ALIAS,
    stateful_security_group.ALIAS,
    'standard-attr-description',
    constants.SUBNET_ALLOCATION_EXT_ALIAS,
    'standard-attr-tag',
    'standard-attr-timestamp',
    subnet_service_types.ALIAS,
    trunk.ALIAS,
    seg_def.ALIAS,
    expose_port_forwarding_in_fip.ALIAS,
    fip_pf_description.ALIAS,
    fip_pf_port_range.ALIAS,
    floating_ip_port_forwarding.ALIAS,
    vlantransparent.ALIAS,
    logging.ALIAS,
    vpn.ALIAS,
    vpn_endpoint_groups.ALIAS,
]
<|endoftext|>


<commit_before>"""
Options abstract away different class of options (e.g. matplotlib
specific styles and plot specific parameters) away from View and Stack
objects, allowing these objects to share options by name.

StyleOpts is an OptionMap that allows matplotlib style options to be
defined, allowing customization of how individual View objects are
displayed if they have the appropriate style name.
"""

from collections import OrderedDict

class Cycle(object):
    """
    A simple container class to allow specification of cyclic style
    patterns. A typical use would be to cycle styles on a plot with
    multiple curves.
    """
    def __init__(self, elements):
        self.elements = elements

    def __len__(self):
        return len(self.elements)

    def __repr__(self):
        return "Cycle(%s)" % self.elements

class Opts(object):
    """
    A Options object specified keyword options. In addition to the
    functionality of a simple dictionary, Opts support cyclic indexing
    if supplied with a Cycle object.
    """

    def __init__(self, **kwargs):
        self.items = kwargs
        self.options = self._expand_styles(kwargs)

    def __call__(self, **kwargs):
        new_style = dict(self.items, **kwargs)
        return self.__class__(**new_style)

    def _expand_styles(self, kwargs):
        """
        Expand out Cycle objects into multiple sets of keyword options.
        """
        filter_static = dict((k,v) for (k,v) in kwargs.items() if not isinstance(v, Cycle))
        filter_cycles = [(k,v) for (k,v) in kwargs.items() if isinstance(v, Cycle)]

        if not filter_cycles: return [kwargs]

        filter_names, filter_values = zip(*filter_cycles)
        if not all(len(c)==len(filter_values[0]) for c in filter_values):
            raise Exception("Cycle objects supplied with different lengths")

        cyclic_tuples = zip(*[val.elements for val in filter_values])
        return [ dict(zip(filter_names, tps), **filter_static) for tps in cyclic_tuples]

    def keys(self):
        "The keyword names defined in the options."
        return self.items.keys()

    def __getitem__(self, index):
        """
        Cyclic indexing over any Cycle objects defined.
        """
        return dict(self.options[index % len(self.options)])

    @property
    def opts(self):
        if len(self.options) == 1:
            return dict(self.options[0])
        else:
            raise Exception("The opts property may only be used with non-cyclic styles")

    def __repr__(self):
        return "%s(%s)" % (self.__class__.__name__, self.keywords)

    @property
    def keywords(self):
        return ', '.join("%s=%r" % (k,v) for (k,v) in self.items.items())

class Options(object):
    """
    A Option is a collection of Opts objects that allows convenient
    attribute access and can compose styles through inheritance.
    Options are inherited by finding all matches which end in the
    same substring as the supplied style.

    For example supplying 'Example_View' as a style would match
    these styles (if they are defined):

    'View' : Opts(a=1, b=2)
    'Example_View': Opts(b=3)

    The resulting Opts object inherits a=1 from 'Options' and b=3
    from 'Example_Options'.
    """

    @classmethod
    def normalize_key(self, key):
        """
        Given a key which may contain spaces, such as a view label,
        convert it to a string suitable for attribute access.
        """
        return key.replace(' ', '_')

    def __init__(self, name, opt_type):
        if not issubclass(opt_type, Opts):
            raise Exception("The opt_type needs to be a subclass of Opts.")
        self.name = name
        self._settable = False
        self.opt_type = opt_type
        self.__dict__['_items'] = {}

    def __call__(self, obj):
        if not hasattr(obj, 'style'):
            raise Exception('Supplied object requires style attribute.')
        elif isinstance(obj.style, list):
            return self.opt_type()

        name = obj.style
        matches = sorted((len(key), style) for key, style in self._items.items()
                         if name.endswith(key))
        if matches == []:
            return self.opt_type()
        else:
            base_match = matches[0][1]
            for _, match in matches[1:]:
                base_match = base_match(**match.items)
            return base_match

    def options(self):
        """
        The full list of base Style objects in the Options, excluding
        options customized per object.
        """
        return [k for k in self.keys() if not k.startswith('Custom')]

    def __dir__(self):
        """
        Extend dir() to include base options in IPython tab completion.
        """
        default_dir = dir(type(self)) + list(self.__dict__)
        return sorted(set(default_dir + self.options()))

    def __getattr__(self, name):
        """
        Provide attribute access for the Opts in the Options.
        """
        keys = self.__dict__['_items'].keys()
        if name in keys:
            return self[name]
        raise AttributeError(name)

    def __getitem__(self, obj):
        """
        Fuzzy matching allows a more specific key to be matched
        against a general style entry that has a common suffix.
        """
        return self._items[obj]

    def __repr__(self):
        return "<OptionMap containing %d options>" % len(self._items)

    def keys(self):
        """
        The list of all options in the OptionMap, including options
        associated with individual view objects.
        """
        return self._items.keys()

    def values(self):
        """
        All the Style objects in the OptionMap.
        """
        return self._items.values()

    def __contains__(self, k):
        return k in self.keys()

    def set(self, key, value):
        if not self._settable:
            raise Exception("OptionMaps should be set via OptionGroup")
        if not isinstance(value, Opts):
            raise Exception('An OptionMap must only contain Opts.')
        self._items[self.normalize_key(key)] = value

class OptionsGroup(object):
    """
    An OptionsGroup coordinates the setting of OptionMaps to ensure
    they share a common set of keys. While it is safe to access Opts
    from OptionMaps directly, an OptionGroup object must be used to
    set Options when there are multiple different types of Options
    (plot options as distinct from style options for instance).

    When setting Options, it is important to use the appropriate
    subclass of Opts to disambiguate the OptionGroup to be set. For
    instance, PlotOpts will set plotting options while StyleOpts are
    designed for setting style options.
    """

    normalize_key = Options.normalize_key

    def __init__(self, optmaps):

        names = [o.name for o in optmaps]
        if len(set(names)) != len(names):
            raise Exception("OptionMap names must be unique")

        for optmap in optmaps:
            self.__dict__[optmap.name] = optmap

        self.__dict__['_keys'] = set()
        self.__dict__['_opttypes'] = OrderedDict([(optmap.opt_type, optmap) for optmap in optmaps])

    def __setattr__(self, k, v):
        """
        Attribute style addition of Style objects to the OptionMap.
        """
        opttypes = self.__dict__['_opttypes']
        keys = self.__dict__['_keys']
        keys.add(k)

        if type(v) not in opttypes:
            raise Exception("Options of type %s not applicable" % type(v))

        optmap = opttypes[type(v)]
        optmap._settable = True
        optmap.set(k, v)
        optmap._settable = False

    def fuzzy_match_keys(self, name):
        name = Options.normalize_key(name)
        reversed_matches = sorted((len(key), key) for key in self.keys()
                                  if name.endswith(key))[::-1]
        if reversed_matches:
            return zip(*reversed_matches)[1]
        else:
            return []

    def __getattr__(self, k):
        return self[k]

    def __getitem__(self, key):
        if key not in self._keys:
            raise IndexError('Key not available in the OptionGroup')
        retval = tuple(optmap[key] for optmap in self._opttypes.values())
        return retval[0] if len(retval) == 1 else retval

    def __setitem__(self, key, value):
        if type(value) not in self._opttypes:
            raise Exception("Options of type %s not applicable" % type(value))
        optmap = self._opttypes[type(value)]
        optmap._settable = True
        optmap.set(key, value)
        optmap._settable = False
        self._keys.add(key)

    def options(self):
        """
        The full list of option keys in the OptionGroup, excluding
        options customized per object.
        """
        return [k for k in self.keys() if not k.startswith('Custom')]

    def keys(self):
        return sorted(list(self._keys))

    def __dir__(self):
        """
        Extend dir() to include base options in IPython tab completion.
        """
        default_dir = dir(type(self)) + list(self.__dict__)
        names = [o.name for o in self._opttypes.values()]
        return sorted(set(default_dir + self.keys() + names))

class StyleOpts(Opts):
    """
    A subclass of Opts designed to hold matplotlib options to set the
    display Style of View objects.
    """

class PlotOpts(Opts):
    """
    A subclass of Opts designed to hold plotting options that set the
    parameters of the Plot class that display View objects.
    """

class ChannelOpts(Opts):
    """
    A subclass of Opts designed to hold channel mode definitions that
    control how particular labelled layer combinations in an Overlay
    are displayed.
    """
    def __init__(self, mode, pattern, **kwargs):
        self.mode = mode
        self.pattern = pattern
        self.size = len(pattern.rsplit('*'))
        self.options = self._expand_styles(kwargs)
        self._kwargs = kwargs

    def __repr__(self):
        return "%s(%s%s)" % (self.__class__.__name__,
                              self.mode+(', ' if self._kwargs else ''),
                              self.keywords)

channels = OptionsGroup([Options('definitions', ChannelOpts)])
options = OptionsGroup([Options('plotting', PlotOpts),
                        Options('style',StyleOpts)])

# Default Styles
options.Style = StyleOpts()
options.Contours = StyleOpts(color=Cycle(['k', 'w']))
options.SheetView = StyleOpts(cmap='gray', interpolation='nearest')
options.Curve = StyleOpts(color=Cycle(['r', 'g', 'b']), linewidth=2)
options.Annotation = StyleOpts()
options.Histogram = StyleOpts(ec='k', fc='w')
options.Table = StyleOpts()

# Defining the most common style options for dataviews
GrayNearest = StyleOpts(cmap='gray', interpolation='nearest')
<commit_msg>Option key normalization now replaces spaces with '' instead of '_'<commit_after>"""
Options abstract away different class of options (e.g. matplotlib
specific styles and plot specific parameters) away from View and Stack
objects, allowing these objects to share options by name.

StyleOpts is an OptionMap that allows matplotlib style options to be
defined, allowing customization of how individual View objects are
displayed if they have the appropriate style name.
"""

from collections import OrderedDict

class Cycle(object):
    """
    A simple container class to allow specification of cyclic style
    patterns. A typical use would be to cycle styles on a plot with
    multiple curves.
    """
    def __init__(self, elements):
        self.elements = elements

    def __len__(self):
        return len(self.elements)

    def __repr__(self):
        return "Cycle(%s)" % self.elements

class Opts(object):
    """
    A Options object specified keyword options. In addition to the
    functionality of a simple dictionary, Opts support cyclic indexing
    if supplied with a Cycle object.
    """

    def __init__(self, **kwargs):
        self.items = kwargs
        self.options = self._expand_styles(kwargs)

    def __call__(self, **kwargs):
        new_style = dict(self.items, **kwargs)
        return self.__class__(**new_style)

    def _expand_styles(self, kwargs):
        """
        Expand out Cycle objects into multiple sets of keyword options.
        """
        filter_static = dict((k,v) for (k,v) in kwargs.items() if not isinstance(v, Cycle))
        filter_cycles = [(k,v) for (k,v) in kwargs.items() if isinstance(v, Cycle)]

        if not filter_cycles: return [kwargs]

        filter_names, filter_values = zip(*filter_cycles)
        if not all(len(c)==len(filter_values[0]) for c in filter_values):
            raise Exception("Cycle objects supplied with different lengths")

        cyclic_tuples = zip(*[val.elements for val in filter_values])
        return [ dict(zip(filter_names, tps), **filter_static) for tps in cyclic_tuples]

    def keys(self):
        "The keyword names defined in the options."
        return self.items.keys()

    def __getitem__(self, index):
        """
        Cyclic indexing over any Cycle objects defined.
        """
        return dict(self.options[index % len(self.options)])

    @property
    def opts(self):
        if len(self.options) == 1:
            return dict(self.options[0])
        else:
            raise Exception("The opts property may only be used with non-cyclic styles")

    def __repr__(self):
        return "%s(%s)" % (self.__class__.__name__, self.keywords)

    @property
    def keywords(self):
        return ', '.join("%s=%r" % (k,v) for (k,v) in self.items.items())

class Options(object):
    """
    A Option is a collection of Opts objects that allows convenient
    attribute access and can compose styles through inheritance.
    Options are inherited by finding all matches which end in the
    same substring as the supplied style.

    For example supplying 'Example_View' as a style would match
    these styles (if they are defined):

    'View' : Opts(a=1, b=2)
    'Example_View': Opts(b=3)

    The resulting Opts object inherits a=1 from 'Options' and b=3
    from 'Example_Options'.
    """

    @classmethod
    def normalize_key(self, key):
        """
        Given a key which may contain spaces, such as a view label,
        convert it to a string suitable for attribute access.
        """
        return key.replace(' ', '')

    def __init__(self, name, opt_type):
        if not issubclass(opt_type, Opts):
            raise Exception("The opt_type needs to be a subclass of Opts.")
        self.name = name
        self._settable = False
        self.opt_type = opt_type
        self.__dict__['_items'] = {}

    def __call__(self, obj):
        if not hasattr(obj, 'style'):
            raise Exception('Supplied object requires style attribute.')
        elif isinstance(obj.style, list):
            return self.opt_type()

        name = obj.style
        matches = sorted((len(key), style) for key, style in self._items.items()
                         if name.endswith(key))
        if matches == []:
            return self.opt_type()
        else:
            base_match = matches[0][1]
            for _, match in matches[1:]:
                base_match = base_match(**match.items)
            return base_match

    def options(self):
        """
        The full list of base Style objects in the Options, excluding
        options customized per object.
        """
        return [k for k in self.keys() if not k.startswith('Custom')]

    def __dir__(self):
        """
        Extend dir() to include base options in IPython tab completion.
        """
        default_dir = dir(type(self)) + list(self.__dict__)
        return sorted(set(default_dir + self.options()))

    def __getattr__(self, name):
        """
        Provide attribute access for the Opts in the Options.
        """
        keys = self.__dict__['_items'].keys()
        if name in keys:
            return self[name]
        raise AttributeError(name)

    def __getitem__(self, obj):
        """
        Fuzzy matching allows a more specific key to be matched
        against a general style entry that has a common suffix.
        """
        return self._items[obj]

    def __repr__(self):
        return "<OptionMap containing %d options>" % len(self._items)

    def keys(self):
        """
        The list of all options in the OptionMap, including options
        associated with individual view objects.
        """
        return self._items.keys()

    def values(self):
        """
        All the Style objects in the OptionMap.
        """
        return self._items.values()

    def __contains__(self, k):
        return k in self.keys()

    def set(self, key, value):
        if not self._settable:
            raise Exception("OptionMaps should be set via OptionGroup")
        if not isinstance(value, Opts):
            raise Exception('An OptionMap must only contain Opts.')
        self._items[self.normalize_key(key)] = value

class OptionsGroup(object):
    """
    An OptionsGroup coordinates the setting of OptionMaps to ensure
    they share a common set of keys. While it is safe to access Opts
    from OptionMaps directly, an OptionGroup object must be used to
    set Options when there are multiple different types of Options
    (plot options as distinct from style options for instance).

    When setting Options, it is important to use the appropriate
    subclass of Opts to disambiguate the OptionGroup to be set. For
    instance, PlotOpts will set plotting options while StyleOpts are
    designed for setting style options.
    """

    normalize_key = Options.normalize_key

    def __init__(self, optmaps):

        names = [o.name for o in optmaps]
        if len(set(names)) != len(names):
            raise Exception("OptionMap names must be unique")

        for optmap in optmaps:
            self.__dict__[optmap.name] = optmap

        self.__dict__['_keys'] = set()
        self.__dict__['_opttypes'] = OrderedDict([(optmap.opt_type, optmap) for optmap in optmaps])

    def __setattr__(self, k, v):
        """
        Attribute style addition of Style objects to the OptionMap.
        """
        opttypes = self.__dict__['_opttypes']
        keys = self.__dict__['_keys']
        keys.add(k)

        if type(v) not in opttypes:
            raise Exception("Options of type %s not applicable" % type(v))

        optmap = opttypes[type(v)]
        optmap._settable = True
        optmap.set(k, v)
        optmap._settable = False

    def fuzzy_match_keys(self, name):
        name = Options.normalize_key(name)
        reversed_matches = sorted((len(key), key) for key in self.keys()
                                  if name.endswith(key))[::-1]
        if reversed_matches:
            return zip(*reversed_matches)[1]
        else:
            return []

    def __getattr__(self, k):
        return self[k]

    def __getitem__(self, key):
        if key not in self._keys:
            raise IndexError('Key not available in the OptionGroup')
        retval = tuple(optmap[key] for optmap in self._opttypes.values())
        return retval[0] if len(retval) == 1 else retval

    def __setitem__(self, key, value):
        if type(value) not in self._opttypes:
            raise Exception("Options of type %s not applicable" % type(value))
        optmap = self._opttypes[type(value)]
        optmap._settable = True
        optmap.set(key, value)
        optmap._settable = False
        self._keys.add(key)

    def options(self):
        """
        The full list of option keys in the OptionGroup, excluding
        options customized per object.
        """
        return [k for k in self.keys() if not k.startswith('Custom')]

    def keys(self):
        return sorted(list(self._keys))

    def __dir__(self):
        """
        Extend dir() to include base options in IPython tab completion.
        """
        default_dir = dir(type(self)) + list(self.__dict__)
        names = [o.name for o in self._opttypes.values()]
        return sorted(set(default_dir + self.keys() + names))

class StyleOpts(Opts):
    """
    A subclass of Opts designed to hold matplotlib options to set the
    display Style of View objects.
    """

class PlotOpts(Opts):
    """
    A subclass of Opts designed to hold plotting options that set the
    parameters of the Plot class that display View objects.
    """

class ChannelOpts(Opts):
    """
    A subclass of Opts designed to hold channel mode definitions that
    control how particular labelled layer combinations in an Overlay
    are displayed.
    """
    def __init__(self, mode, pattern, **kwargs):
        self.mode = mode
        self.pattern = pattern
        self.size = len(pattern.rsplit('*'))
        self.options = self._expand_styles(kwargs)
        self._kwargs = kwargs

    def __repr__(self):
        return "%s(%s%s)" % (self.__class__.__name__,
                              self.mode+(', ' if self._kwargs else ''),
                              self.keywords)

channels = OptionsGroup([Options('definitions', ChannelOpts)])
options = OptionsGroup([Options('plotting', PlotOpts),
                        Options('style',StyleOpts)])

# Default Styles
options.Style = StyleOpts()
options.Contours = StyleOpts(color=Cycle(['k', 'w']))
options.SheetView = StyleOpts(cmap='gray', interpolation='nearest')
options.Curve = StyleOpts(color=Cycle(['r', 'g', 'b']), linewidth=2)
options.Annotation = StyleOpts()
options.Histogram = StyleOpts(ec='k', fc='w')
options.Table = StyleOpts()

# Defining the most common style options for dataviews
GrayNearest = StyleOpts(cmap='gray', interpolation='nearest')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import collections

from puz import Puzzle

from crossword import CrosswordException
from crossword.compat import range
from crossword.core import Crossword

def from_puz(puzzle):
    known_keys = (
        "width",
        "height",
        "author",
        "copyright"
        "title",
        "clues",
        "solution",
    )
    result = Crossword(puzzle.width, puzzle.height)
    result._format_identifier = Crossword.PUZ
    result.meta.creator = puzzle.author
    result.meta.rights = puzzle.copyright
    result.meta.title = puzzle.title

    rows = []
    for i in range(0, len(puzzle.solution), puzzle.width):
        rows.append(puzzle.solution[i:i + puzzle.width])

    def is_across(x, y):
        if result[x, y].solution == '.':
            return False
        start = x == 0 or (0 <= x - 1 and result[x - 1, y].solution == '.')
        end = (x + 1 < puzzle.width and result[x + 1, y].solution != '.')
        return start and end

    def is_down(x, y):
        if result[x, y].solution == '.':
            return False
        start = y == 0 or (y - 1 >= 0 and result[x, y - 1].solution == '.')
        end = (y + 1 < puzzle.height and result[x, y + 1].solution != '.')
        return start and end

    clue_index = 0
    number = 0
    for y, row in enumerate(rows):
        for x, cell in enumerate(row):
            result[x, y].cell = None
            result[x, y].solution = cell

    for y, row in enumerate(rows):
        for x, cell in enumerate(row):
            is_xy_across = is_across(x, y)
            is_xy_down = is_down(x, y)
            if is_xy_across or is_xy_down:
                number += 1
            if is_xy_across:
                result.clues.across[number] = puzzle.clues[clue_index]
                clue_index += 1
            if is_xy_down:
                result.clues.down[number] = puzzle.clues[clue_index]
                clue_index += 1

    result.block = '.'

    for attr in dir(puzzle):
        if attr in known_keys:
            continue
        if attr.startswith('__'):
            continue
        if callable(getattr(puzzle, attr)):
            continue
        result._format[attr] = getattr(puzzle, attr)

    return result

def to_puz(crossword):
    result = Puzzle()
    result.width = crossword.width
    result.height = crossword.height
    result.author = crossword.meta.creator
    result.copyright = crossword.meta.rights
    result.title = crossword.meta.title

    cells = []
    for row in crossword:
        for cell in row:
            value = None
            if cell.solution is None:
                value = crossword.block if crossword.block else '.'
            else:
                value = cell.solution
            cells.append(value)
    result.solution = ''.join(cells)

    result.clues = []
    clues = collections.defaultdict(list)
    try:
        for number, clue in crossword.clues.across(sort=int):
            clues[number].append(clue)
    except ValueError:
        raise CrosswordException
    try:
        for number, clue in crossword.clues.down(sort=int):
            clues[number].append(clue)
    except ValueError:
        raise CrosswordException
    for number, clues in sorted(clues.items()):
        for clue in clues:
            result.clues.append(clue)

    if crossword._format_identifier == Crossword.PUZ:
        for key, value in crossword._format.items():
            setattr(result, key, value)

    return result
<commit_msg>Minor refactoring<commit_after># -*- coding: utf-8 -*-
import collections

from puz import Puzzle

from crossword import CrosswordException
from crossword.compat import range
from crossword.core import Crossword

def from_puz(puzzle):
    known_keys = (
        "width",
        "height",
        "author",
        "copyright"
        "title",
        "clues",
        "solution",
    )
    result = Crossword(puzzle.width, puzzle.height)
    result._format_identifier = Crossword.PUZ
    result.meta.creator = puzzle.author
    result.meta.rights = puzzle.copyright
    result.meta.title = puzzle.title
    result.block = '.'

    rows = []
    for i in range(0, len(puzzle.solution), puzzle.width):
        rows.append(puzzle.solution[i:i + puzzle.width])

    def is_across(x, y):
        if result[x, y].solution == '.':
            return False
        start = x == 0 or (0 <= x - 1 and result[x - 1, y].solution == '.')
        end = (x + 1 < puzzle.width and result[x + 1, y].solution != '.')
        return start and end

    def is_down(x, y):
        if result[x, y].solution == '.':
            return False
        start = y == 0 or (y - 1 >= 0 and result[x, y - 1].solution == '.')
        end = (y + 1 < puzzle.height and result[x, y + 1].solution != '.')
        return start and end

    clue_index = 0
    number = 0
    for y, row in enumerate(rows):
        for x, cell in enumerate(row):
            result[x, y].cell = None
            result[x, y].solution = cell

    for y, row in enumerate(rows):
        for x, cell in enumerate(row):
            is_xy_across = is_across(x, y)
            is_xy_down = is_down(x, y)
            if is_xy_across or is_xy_down:
                number += 1
            if is_xy_across:
                result.clues.across[number] = puzzle.clues[clue_index]
                clue_index += 1
            if is_xy_down:
                result.clues.down[number] = puzzle.clues[clue_index]
                clue_index += 1

    for attr in dir(puzzle):
        if attr in known_keys:
            continue
        if attr.startswith('__'):
            continue
        if callable(getattr(puzzle, attr)):
            continue
        result._format[attr] = getattr(puzzle, attr)

    return result

def to_puz(crossword):
    result = Puzzle()
    result.width = crossword.width
    result.height = crossword.height
    result.author = crossword.meta.creator
    result.copyright = crossword.meta.rights
    result.title = crossword.meta.title

    cells = []
    for row in crossword:
        for cell in row:
            value = None
            if cell.solution is None:
                value = crossword.block if crossword.block else '.'
            else:
                value = cell.solution
            cells.append(value)
    result.solution = ''.join(cells)

    result.clues = []
    clues = collections.defaultdict(list)
    try:
        for number, clue in crossword.clues.across(sort=int):
            clues[number].append(clue)
    except ValueError:
        raise CrosswordException
    try:
        for number, clue in crossword.clues.down(sort=int):
            clues[number].append(clue)
    except ValueError:
        raise CrosswordException
    for number, clues in sorted(clues.items()):
        for clue in clues:
            result.clues.append(clue)

    if crossword._format_identifier == Crossword.PUZ:
        for key, value in crossword._format.items():
            setattr(result, key, value)

    return result
<|endoftext|>


<commit_before># Copyright (c) 2011 Mathieu Turcotte
# Licensed under the MIT license.

from __future__ import with_statement  # Enable with statement in Python 2.5.
import sys

# Use unittest2 on versions older than Python 2.7.
if sys.version_info[0] < 3 and sys.version_info[1] < 7:
    from unittest2 import TestCase, main
else:
    from unittest import TestCase, main

import msparser

EMPTY_SNAPSHOTS = """
desc: --time-unit=ms
cmd: ./a.out
time_unit: ms
#-----------
snapshot=0
#-----------
time=0
mem_heap_B=0
mem_heap_extra_B=0
mem_stacks_B=0
heap_tree=empty
#-----------
snapshot=1
#-----------
time=183
mem_heap_B=1000
mem_heap_extra_B=8
mem_stacks_B=0
heap_tree=empty
#-----------
snapshot=2
#-----------
time=184
mem_heap_B=2000
mem_heap_extra_B=16
mem_stacks_B=0
heap_tree=empty"""

DETAILED_SNAPSHOTS = """
desc: --time-unit=i
cmd: ./memory-run
time_unit: i
#-----------
snapshot=0
#-----------
time=1887671
mem_heap_B=91458
mem_heap_extra_B=39022
mem_stacks_B=0
heap_tree=empty
#-----------
snapshot=1
#-----------
time=1937480
mem_heap_B=56850
mem_heap_extra_B=24190
mem_stacks_B=0
heap_tree=detailed
n3: 56850 (heap allocation functions) malloc/new/new[], --alloc-fns, etc.
 n1: 50456 0x804BFC0: DancingLinksSolver::build_cover_matrix(Sudoku&) (SudokuSolver.cpp:114)
  n2: 50456 0x804C30F: DancingLinksSolver::solve(Sudoku&) (SudokuSolver.cpp:15)
   n0: 50456 0x8049C2C: main (memory.cpp:30)
   n0: 0 in 1 place, below massif's threshold (01.00%)
 n1: 5628 0x804BE5C: DancingLinksSolver::build_cover_matrix(Sudoku&) (SudokuSolver.cpp:42)
  n2: 5628 0x804C30F: DancingLinksSolver::solve(Sudoku&) (SudokuSolver.cpp:15)
   n0: 5628 0x8049C2C: main (memory.cpp:30)
   n0: 0 in 1 place, below massif's threshold (01.00%)
 n0: 766 in 6 places, all below massif's threshold (01.00%)
#-----------
snapshot=2
#-----------
time=3068292
mem_heap_B=441458
mem_heap_extra_B=186670
mem_stacks_B=0
heap_tree=empty
#-----------
snapshot=3
#-----------
time=3194213
mem_heap_B=494014
mem_heap_extra_B=209194
mem_stacks_B=0
heap_tree=peak
n4: 494014 (heap allocation functions) malloc/new/new[], --alloc-fns, etc.
 n1: 344064 0x804BFC0: DancingLinksSolver::build_cover_matrix(Sudoku&) (SudokuSolver.cpp:114)
  n2: 344064 0x804C30F: DancingLinksSolver::solve(Sudoku&) (SudokuSolver.cpp:15)
   n0: 344064 0x8049C88: main (memory.cpp:49)
   n0: 0 in 2 places, all below massif's threshold (01.00%)
 n1: 114688 0x804BF7C: DancingLinksSolver::build_cover_matrix(Sudoku&) (SudokuSolver.cpp:91)
  n2: 114688 0x804C30F: DancingLinksSolver::solve(Sudoku&) (SudokuSolver.cpp:15)
   n0: 114688 0x8049C88: main (memory.cpp:49)
   n0: 0 in 2 places, all below massif's threshold (01.00%)
 n1: 28672 0x804BE5C: DancingLinksSolver::build_cover_matrix(Sudoku&) (SudokuSolver.cpp:42)
  n2: 28672 0x804C30F: DancingLinksSolver::solve(Sudoku&) (SudokuSolver.cpp:15)
   n0: 28672 0x8049C88: main (memory.cpp:49)
   n0: 0 in 2 places, all below massif's threshold (01.00%)
 n0: 6590 in 5 places, all below massif's threshold (01.00%)
#-----------
snapshot=4
#-----------
time=4342249
mem_heap_B=489890
mem_heap_extra_B=209174
mem_stacks_B=0
heap_tree=empty"""

class MockFile():
    def __init__(self, content):
        self.lines = content.split("\n")
        self.index = 0

    def readline(self):
        if len(self.lines) > self.index:
            line = self.lines[self.index] + "\n"
            self.index += 1
            return line
        else:
            return ""

def parse(content):
    fd = MockFile(content)
    return msparser.parse(fd)

class MassifParserEmptySnaphostTest(TestCase):
    def setUp(self):
        self.data = parse(EMPTY_SNAPSHOTS)

    def test_parse_time_unit(self):
        self.assertEqual(self.data["time_unit"], "ms")

    def test_parse_cmd(self):
        self.assertEqual(self.data["cmd"], "./a.out")

    def test_parse_desc(self):
        self.assertEqual(self.data["desc"], "--time-unit=ms")

    def test_parse_snaphosts(self):
        self.assertEqual(len(self.data["snapshots"]), 3)

    def test_parse_empty_snapshot(self):
        snapshot = self.data["snapshots"][2]
        self.assertEqual(snapshot["id"], 2, "id")
        self.assertEqual(snapshot["time"], 184, "time")
        self.assertEqual(snapshot["mem_heap"], 2000, "mem_heap")
        self.assertEqual(snapshot["mem_heap_extra"], 16, "mem_heap_extra")
        self.assertEqual(snapshot["mem_stack"], 0, "mem_stack")
        self.assertEqual(snapshot["heap_tree"], None, "heap_tree")

class MassifParserDetailedSnaphostTest(TestCase):
    def setUp(self):
        self.data = parse(DETAILED_SNAPSHOTS)

    def test_parse_snapshot_heap_tree(self):
        snapshot = self.data["snapshots"][1]

        # Check the heap tree's root.
        root = snapshot["heap_tree"]
        self.assertEqual(root["nbytes"], 56850)
        self.assertEqual(len(root["children"]), 3)

        # Check root's first child.
        root_children = sorted(root["children"], key=lambda x: x["nbytes"],
                               reverse=True)
        first_root_child = root_children[0]
        self.assertEqual(first_root_child["nbytes"], 50456)
        self.assertEqual(first_root_child["details"], {
            "function": "DancingLinksSolver::build_cover_matrix(Sudoku&)",
            "address": "0x804BFC0",
            "file": "SudokuSolver.cpp",
            "line": 114
        })

    def test_find_peak_snapshot(self):
        peak_snapshot_index = self.data["peak_snapshot_index"]
        self.assertEqual(peak_snapshot_index, 3)

class MassifParserErrorDetectionTest(TestCase):
    def test_detect_partial_file(self):
        with self.assertRaises(msparser.ParseError):
            msparser.parse(MockFile("desc: --time-unit=ms\n"
                                    "c broken  md: ./a.out\n"
                                    "time_unit: ms\n"
                                    "#-----------\n"
                                    "snapshot=0\n"
                                    "#-----------\n"
                                    "time=0\n"
                                    "mem_heap_B=0"))

    def test_detect_broken_header(self):
        with self.assertRaises(msparser.ParseError):
            msparser.parse(MockFile("desc: --time-unit=ms\n"
                                    "c broken  md: ./a.out\n"
                                    "time_unit: ms\n"
                                    "#-----------\n"
                                    "snapshot=0\n"
                                    "#-----------\n"
                                    "time=0\n"
                                    "mem_heap_B=0\n"
                                    "mem_heap_extra_B=0\n"
                                    "mem_stacks_B=0\n"
                                    "heap_tree=empty\n"))

    def test_detect_broken_snapshot(self):
        with self.assertRaises(msparser.ParseError):
            msparser.parse(MockFile("desc: --time-unit=ms\n"
                                    "cmd: ./a.out\n"
                                    "time_unit: ms\n"
                                    "#-----------\n"
                                    "snapshot=0\n"
                                    "#-----------\n"
                                    "time=0\n"
                                    "mem_heap_B=0\n"
                                    "mem_h e a p_extra_B=0\n"
                                    "mem_stacks_B=0\n"
                                    "heap_tree=empty\n"))

    def test_detect_broken_heap_tree(self):
        fd = MockFile("desc: --time-unit=ms\n"
                      "cmd: ./a.out\n"
                      "time_unit: ms\n"
                      "#-----------\n"
                      "snapshot=0\n"
                      "#-----------\n"
                      "time=0\n"
                      "mem_heap_B=0\n"
                      "mem_heap_extra_B=0\n"
                      "mem_stacks_B=0\n"
                      "heap_tree=peak\n"
                      "n4: 32000 (heap allocation functions) malloc/new/new[], --alloc-fns, etc.\n"
                      " n1: 12000 0x8048404: h (prog.c:4)\n"
                      "  n2: 12000 0x804841D: g (prog.c:9)\n"
                      "   n1: 6000 0x8048436: f (prog.c:14)\n"
                      "    n0: 6000 0x8048472: main (prog.c:24)\n"
                      "   n0: 6000 0x8048477: main (prog.c:26)\n"
                      " n0: 10000 0x8048457: main (prog.c:22)\n"
                      " n2: 8000 0x8048418: g (prog.c:8)\n"
                      "  n1: 4000 0x8048436: f (prog.c:14)\n"
                      # "   n0: 4000 0x8048472: main (prog.c:24)\n"
                      "  n0: 4000 0x8048477: main (prog.c:26)\n"
                      " n1: 2000 0x8048431: f (prog.c:13)\n"
                      "  n0: 2000 0x8048472: main (prog.c:24)\n")

        with self.assertRaises(msparser.ParseError):
            msparser.parse(fd)

if __name__ == "__main__":
    main()
<commit_msg>Mark injected errors with comments.<commit_after># Copyright (c) 2011 Mathieu Turcotte
# Licensed under the MIT license.

from __future__ import with_statement  # Enable with statement in Python 2.5.
import sys

# Use unittest2 on versions older than Python 2.7.
if sys.version_info[0] < 3 and sys.version_info[1] < 7:
    from unittest2 import TestCase, main
else:
    from unittest import TestCase, main

import msparser

EMPTY_SNAPSHOTS = """
desc: --time-unit=ms
cmd: ./a.out
time_unit: ms
#-----------
snapshot=0
#-----------
time=0
mem_heap_B=0
mem_heap_extra_B=0
mem_stacks_B=0
heap_tree=empty
#-----------
snapshot=1
#-----------
time=183
mem_heap_B=1000
mem_heap_extra_B=8
mem_stacks_B=0
heap_tree=empty
#-----------
snapshot=2
#-----------
time=184
mem_heap_B=2000
mem_heap_extra_B=16
mem_stacks_B=0
heap_tree=empty"""

DETAILED_SNAPSHOTS = """
desc: --time-unit=i
cmd: ./memory-run
time_unit: i
#-----------
snapshot=0
#-----------
time=1887671
mem_heap_B=91458
mem_heap_extra_B=39022
mem_stacks_B=0
heap_tree=empty
#-----------
snapshot=1
#-----------
time=1937480
mem_heap_B=56850
mem_heap_extra_B=24190
mem_stacks_B=0
heap_tree=detailed
n3: 56850 (heap allocation functions) malloc/new/new[], --alloc-fns, etc.
 n1: 50456 0x804BFC0: DancingLinksSolver::build_cover_matrix(Sudoku&) (SudokuSolver.cpp:114)
  n2: 50456 0x804C30F: DancingLinksSolver::solve(Sudoku&) (SudokuSolver.cpp:15)
   n0: 50456 0x8049C2C: main (memory.cpp:30)
   n0: 0 in 1 place, below massif's threshold (01.00%)
 n1: 5628 0x804BE5C: DancingLinksSolver::build_cover_matrix(Sudoku&) (SudokuSolver.cpp:42)
  n2: 5628 0x804C30F: DancingLinksSolver::solve(Sudoku&) (SudokuSolver.cpp:15)
   n0: 5628 0x8049C2C: main (memory.cpp:30)
   n0: 0 in 1 place, below massif's threshold (01.00%)
 n0: 766 in 6 places, all below massif's threshold (01.00%)
#-----------
snapshot=2
#-----------
time=3068292
mem_heap_B=441458
mem_heap_extra_B=186670
mem_stacks_B=0
heap_tree=empty
#-----------
snapshot=3
#-----------
time=3194213
mem_heap_B=494014
mem_heap_extra_B=209194
mem_stacks_B=0
heap_tree=peak
n4: 494014 (heap allocation functions) malloc/new/new[], --alloc-fns, etc.
 n1: 344064 0x804BFC0: DancingLinksSolver::build_cover_matrix(Sudoku&) (SudokuSolver.cpp:114)
  n2: 344064 0x804C30F: DancingLinksSolver::solve(Sudoku&) (SudokuSolver.cpp:15)
   n0: 344064 0x8049C88: main (memory.cpp:49)
   n0: 0 in 2 places, all below massif's threshold (01.00%)
 n1: 114688 0x804BF7C: DancingLinksSolver::build_cover_matrix(Sudoku&) (SudokuSolver.cpp:91)
  n2: 114688 0x804C30F: DancingLinksSolver::solve(Sudoku&) (SudokuSolver.cpp:15)
   n0: 114688 0x8049C88: main (memory.cpp:49)
   n0: 0 in 2 places, all below massif's threshold (01.00%)
 n1: 28672 0x804BE5C: DancingLinksSolver::build_cover_matrix(Sudoku&) (SudokuSolver.cpp:42)
  n2: 28672 0x804C30F: DancingLinksSolver::solve(Sudoku&) (SudokuSolver.cpp:15)
   n0: 28672 0x8049C88: main (memory.cpp:49)
   n0: 0 in 2 places, all below massif's threshold (01.00%)
 n0: 6590 in 5 places, all below massif's threshold (01.00%)
#-----------
snapshot=4
#-----------
time=4342249
mem_heap_B=489890
mem_heap_extra_B=209174
mem_stacks_B=0
heap_tree=empty"""

class MockFile():
    def __init__(self, content):
        self.lines = content.split("\n")
        self.index = 0

    def readline(self):
        if len(self.lines) > self.index:
            line = self.lines[self.index] + "\n"
            self.index += 1
            return line
        else:
            return ""

def parse(content):
    fd = MockFile(content)
    return msparser.parse(fd)

class MassifParserEmptySnaphostTest(TestCase):
    def setUp(self):
        self.data = parse(EMPTY_SNAPSHOTS)

    def test_parse_time_unit(self):
        self.assertEqual(self.data["time_unit"], "ms")

    def test_parse_cmd(self):
        self.assertEqual(self.data["cmd"], "./a.out")

    def test_parse_desc(self):
        self.assertEqual(self.data["desc"], "--time-unit=ms")

    def test_parse_snaphosts(self):
        self.assertEqual(len(self.data["snapshots"]), 3)

    def test_parse_empty_snapshot(self):
        snapshot = self.data["snapshots"][2]
        self.assertEqual(snapshot["id"], 2, "id")
        self.assertEqual(snapshot["time"], 184, "time")
        self.assertEqual(snapshot["mem_heap"], 2000, "mem_heap")
        self.assertEqual(snapshot["mem_heap_extra"], 16, "mem_heap_extra")
        self.assertEqual(snapshot["mem_stack"], 0, "mem_stack")
        self.assertEqual(snapshot["heap_tree"], None, "heap_tree")

class MassifParserDetailedSnaphostTest(TestCase):
    def setUp(self):
        self.data = parse(DETAILED_SNAPSHOTS)

    def test_parse_snapshot_heap_tree(self):
        snapshot = self.data["snapshots"][1]

        # Check the heap tree's root.
        root = snapshot["heap_tree"]
        self.assertEqual(root["nbytes"], 56850)
        self.assertEqual(len(root["children"]), 3)

        # Check root's first child.
        root_children = sorted(root["children"], key=lambda x: x["nbytes"],
                               reverse=True)
        first_root_child = root_children[0]
        self.assertEqual(first_root_child["nbytes"], 50456)
        self.assertEqual(first_root_child["details"], {
            "function": "DancingLinksSolver::build_cover_matrix(Sudoku&)",
            "address": "0x804BFC0",
            "file": "SudokuSolver.cpp",
            "line": 114
        })

    def test_find_peak_snapshot(self):
        peak_snapshot_index = self.data["peak_snapshot_index"]
        self.assertEqual(peak_snapshot_index, 3)

class MassifParserErrorDetectionTest(TestCase):
    def test_detect_partial_file(self):
        with self.assertRaises(msparser.ParseError):
            msparser.parse(MockFile("desc: --time-unit=ms\n"
                                    "c broken  md: ./a.out\n"
                                    "time_unit: ms\n"
                                    "#-----------\n"
                                    "snapshot=0\n"
                                    "#-----------\n"
                                    "time=0\n"
                                    "mem_heap_B=0"))

    def test_detect_broken_header(self):
        with self.assertRaises(msparser.ParseError):
            msparser.parse(MockFile("desc: --time-unit=ms\n"
                                    "c broken  md: ./a.out\n"  # Bad tag name.
                                    "time_unit: ms\n"
                                    "#-----------\n"
                                    "snapshot=0\n"
                                    "#-----------\n"
                                    "time=0\n"
                                    "mem_heap_B=0\n"
                                    "mem_heap_extra_B=0\n"
                                    "mem_stacks_B=0\n"
                                    "heap_tree=empty\n"))

    def test_detect_broken_snapshot(self):
        with self.assertRaises(msparser.ParseError):
            msparser.parse(MockFile("desc: --time-unit=ms\n"
                                    "cmd: ./a.out\n"
                                    "time_unit: ms\n"
                                    "#-----------\n"
                                    "snapshot=0\n"
                                    "#-----------\n"
                                    "time=0\n"
                                    "mem_heap_B=0\n"
                                    "mem_h e a p_extra_B=0\n"  # Bad tag name.
                                    "mem_stacks_B=0\n"
                                    "heap_tree=empty\n"))

    def test_detect_broken_heap_tree(self):
        fd = MockFile("desc: --time-unit=ms\n"
                      "cmd: ./a.out\n"
                      "time_unit: ms\n"
                      "#-----------\n"
                      "snapshot=0\n"
                      "#-----------\n"
                      "time=0\n"
                      "mem_heap_B=0\n"
                      "mem_heap_extra_B=0\n"
                      "mem_stacks_B=0\n"
                      "heap_tree=peak\n"
                      "n4: 32000 (heap allocation functions) malloc/new/new[], --alloc-fns, etc.\n"
                      " n1: 12000 0x8048404: h (prog.c:4)\n"
                      "  n2: 12000 0x804841D: g (prog.c:9)\n"
                      "   n1: 6000 0x8048436: f (prog.c:14)\n"
                      "    n0: 6000 0x8048472: main (prog.c:24)\n"
                      "   n0: 6000 0x8048477: main (prog.c:26)\n"
                      " n0: 10000 0x8048457: main (prog.c:22)\n"
                      " n2: 8000 0x8048418: g (prog.c:8)\n"
                      "  n1: 4000 0x8048436: f (prog.c:14)\n"
                      # "   n0: 4000 0x8048472: main (prog.c:24)\n"
                      "  n0: 4000 0x8048477: main (prog.c:26)\n"
                      " n1: 2000 0x8048431: f (prog.c:13)\n"
                      "  n0: 2000 0x8048472: main (prog.c:24)\n")

        with self.assertRaises(msparser.ParseError):
            msparser.parse(fd)

if __name__ == "__main__":
    main()
<|endoftext|>