<commit_before>import numpy as np
from pybug.align.base import MultipleAlignment
from pybug.align.rigid.base import RigidAlignment
from pybug.transform.affine import Rotation, Scale, Translation

class Procrustes(RigidAlignment):
    r"""
    Procrustes Alignment of a set of source landmarks to a target.

    Parameters
    -----------
    source : (N, D) ndarray
        The source points to perform Procrustes alignment from
    target : (N, D) ndarray
        The target points to perform Procrustes alignment to
    """

    def __init__(self, source, target):
        RigidAlignment.__init__(self, source, target)
        self._procrustes_step()

    @property
    def error(self):
        r"""
        The Frobenius Norm of the difference between the target and
        the aligned source.

        :type: float
        """
        return np.linalg.norm(self.target - self.aligned_source)

    @property
    def transform_chain(self):
        r"""
        List of transforms containing the chain of transformations to apply.

        :type: list of :class:`pybug.transform.base.Transform`
        """
        return [self.source_translation, self.scale, self.rotation,
                self.target_translation.inverse]

    @property
    def transform(self):
        r"""
        The single transformation from source to target.

        :type: :class:`pybug.transform.base.Transform`
        """
        return reduce(lambda x, y: x.compose(y), self.transform_chain)

    def _procrustes_step(self):
        # firstly, translate the target to the origin
        self.target_translation = Translation(-self.target.mean(axis=0))
        self.centred_target = self.target_translation.apply(self.target)
        # now translate the source to the origin
        self.source_translation = Translation(-self.source.mean(axis=0))
        # apply the translation to the source
        self.aligned_source = self.source_translation.apply(self.source)
        scale_source = np.linalg.norm(self.aligned_source)
        scale_target = np.linalg.norm(self.centred_target)
        self.scale = Scale(scale_target / scale_source, n_dims=self.n_dims)
        self.aligned_source = self.scale.apply(self.aligned_source)
        # calculate the correlation along each dimension + find the optimal
        # rotation to maximise it
        correlation = np.dot(self.centred_target.T, self.aligned_source)
        U, D, Vt = np.linalg.svd(correlation)
        self.rotation = Rotation(np.dot(U, Vt))
        self.aligned_source = self.rotation.apply(self.aligned_source)
        # finally, move the source back out to where the target is
        self.aligned_source = self.target_translation.inverse.apply(
            self.aligned_source)

    def __str__(self):
        msg = 'Alignment delta_target: %f\n' % self.error
        msg += 'Optimal alignment given by:\n'
        return msg + str(self.transform)

class GeneralizedProcrustesAnalysis(MultipleAlignment):
    r"""
    TODO: Write documentation
    """
    def __init__(self, sources, **kwargs):
        super(GeneralizedProcrustesAnalysis, self).__init__(sources, **kwargs)
        self.procrustes = [[Procrustes(s, self.target)] for s in self.sources]
        self.target_scale = np.linalg.norm(self.target)
        self.n_iterations = 1
        self.max_iterations = 100
        self._recursive_procrustes()

    def _recursive_procrustes(self):
        """
        Recursively calculates a Procrustes alignment
        """
        # find the average of the latest aligned sources:
        if self.n_iterations > self.max_iterations:
            print 'max number of iterations reached.'
            return False
        new_target = (sum(p[-1].aligned_source for p in self.procrustes) /
                      self.n_sources)
        rescale = Scale(self.target_scale / np.linalg.norm(new_target),
                        n_dims=self.n_dims)
        centre = Translation(-new_target.mean(axis=0))
        rescale_about_com = centre.compose(rescale).compose(centre.inverse)
        new_target = rescale_about_com.apply(new_target)
        self.delta_target = np.linalg.norm(self.target - new_target)
        print 'at iteration %d, the delta_target is %f' % (self.n_iterations,
                                                           self.delta_target)
        if self.delta_target < 1e-6:
            print 'delta_target sufficiently small, stopping.'
            return True
        else:
            self.n_iterations += 1
            self.target = new_target
            for p in self.procrustes:
                p.append(Procrustes(p[-1].aligned_source, new_target))
            return self._recursive_procrustes()

    @property
    def transforms(self):
        return [reduce(lambda a, b: a.compose(b), [x.transform for x in p])
                for p in self.procrustes]

    @property
    def mean_aligned_shape(self):
        return np.mean([p[-1].target for p in self.procrustes], axis=0)

    @property
    def errors(self):
        return [p[-1].error for p in self.procrustes]

    @property
    def average_error(self):
        return sum(self.errors)/self.n_sources

    def __str__(self):
        return ('Converged after %d iterations with av. error %f'
                % (self.n_iterations, self.average_error))
<commit_msg>GPA is now a little bit quieter...<commit_after>import numpy as np
from pybug.align.base import MultipleAlignment
from pybug.align.rigid.base import RigidAlignment
from pybug.transform.affine import Rotation, Scale, Translation

class Procrustes(RigidAlignment):
    r"""
    Procrustes Alignment of a set of source landmarks to a target.

    Parameters
    -----------
    source : (N, D) ndarray
        The source points to perform Procrustes alignment from
    target : (N, D) ndarray
        The target points to perform Procrustes alignment to
    """

    def __init__(self, source, target):
        RigidAlignment.__init__(self, source, target)
        self._procrustes_step()

    @property
    def error(self):
        r"""
        The Frobenius Norm of the difference between the target and
        the aligned source.

        :type: float
        """
        return np.linalg.norm(self.target - self.aligned_source)

    @property
    def transform_chain(self):
        r"""
        List of transforms containing the chain of transformations to apply.

        :type: list of :class:`pybug.transform.base.Transform`
        """
        return [self.source_translation, self.scale, self.rotation,
                self.target_translation.inverse]

    @property
    def transform(self):
        r"""
        The single transformation from source to target.

        :type: :class:`pybug.transform.base.Transform`
        """
        return reduce(lambda x, y: x.compose(y), self.transform_chain)

    def _procrustes_step(self):
        # firstly, translate the target to the origin
        self.target_translation = Translation(-self.target.mean(axis=0))
        self.centred_target = self.target_translation.apply(self.target)
        # now translate the source to the origin
        self.source_translation = Translation(-self.source.mean(axis=0))
        # apply the translation to the source
        self.aligned_source = self.source_translation.apply(self.source)
        scale_source = np.linalg.norm(self.aligned_source)
        scale_target = np.linalg.norm(self.centred_target)
        self.scale = Scale(scale_target / scale_source, n_dims=self.n_dims)
        self.aligned_source = self.scale.apply(self.aligned_source)
        # calculate the correlation along each dimension + find the optimal
        # rotation to maximise it
        correlation = np.dot(self.centred_target.T, self.aligned_source)
        U, D, Vt = np.linalg.svd(correlation)
        self.rotation = Rotation(np.dot(U, Vt))
        self.aligned_source = self.rotation.apply(self.aligned_source)
        # finally, move the source back out to where the target is
        self.aligned_source = self.target_translation.inverse.apply(
            self.aligned_source)

    def __str__(self):
        msg = 'Alignment delta_target: %f\n' % self.error
        msg += 'Optimal alignment given by:\n'
        return msg + str(self.transform)

class GeneralizedProcrustesAnalysis(MultipleAlignment):
    r"""
    TODO: Write documentation
    """
    def __init__(self, sources, **kwargs):
        super(GeneralizedProcrustesAnalysis, self).__init__(sources, **kwargs)
        self.procrustes = [[Procrustes(s, self.target)] for s in self.sources]
        self.target_scale = np.linalg.norm(self.target)
        self.n_iterations = 1
        self.max_iterations = 100
        self.converged = self._recursive_procrustes()

    def _recursive_procrustes(self):
        """
        Recursively calculates a Procrustes alignment
        """
        # find the average of the latest aligned sources:
        if self.n_iterations > self.max_iterations:
            print 'max number of iterations reached without convergence.'
            return False
        new_target = (sum(p[-1].aligned_source for p in self.procrustes) /
                      self.n_sources)
        rescale = Scale(self.target_scale / np.linalg.norm(new_target),
                        n_dims=self.n_dims)
        centre = Translation(-new_target.mean(axis=0))
        rescale_about_com = centre.compose(rescale).compose(centre.inverse)
        new_target = rescale_about_com.apply(new_target)
        self.delta_target = np.linalg.norm(self.target - new_target)
        #print 'at iteration %d, the delta_target is %f' % (self.n_iterations,
        #                                                   self.delta_target)
        if self.delta_target < 1e-6:
            print self
            return True
        else:
            self.n_iterations += 1
            self.target = new_target
            for p in self.procrustes:
                p.append(Procrustes(p[-1].aligned_source, new_target))
            return self._recursive_procrustes()

    @property
    def transforms(self):
        return [reduce(lambda a, b: a.compose(b), [x.transform for x in p])
                for p in self.procrustes]

    @property
    def mean_aligned_shape(self):
        return np.mean([p[-1].target for p in self.procrustes], axis=0)

    @property
    def errors(self):
        return [p[-1].error for p in self.procrustes]

    @property
    def average_error(self):
        return sum(self.errors)/self.n_sources

    def __str__(self):
        if self.converged:
            return ('Converged after %d iterations with av. error %f'
                    % (self.n_iterations, self.average_error))
        else:
            return ('Failed to converge after %d iterations with av. error '
                    '%f' % (self.n_iterations, self.average_error))
<|endoftext|>


<commit_before>"""
Tools for generating forms based on Peewee models
(cribbed from wtforms.ext.django)
"""
from wtforms import fields as f
from wtforms import Form
from wtforms import validators
from wtfpeewee.fields import ModelSelectField

from peewee import PrimaryKeyField, IntegerField, FloatField, DateTimeField,\
    BooleanField, CharField, TextField, ForeignKeyField

__all__ = (
    'model_fields', 'model_form',
)

class ModelConverter(object):
    defaults = {
        PrimaryKeyField: f.HiddenField,
        IntegerField: f.IntegerField,
        FloatField: f.DecimalField,
        DateTimeField: f.DateTimeField,
        BooleanField: f.BooleanField,
        CharField: f.TextField,
        TextField: f.TextAreaField,
    }
    required = (DateTimeField, CharField, TextField, ForeignKeyField,)

    def __init__(self, additional=None):
        self.converters = {
            ForeignKeyField: self.handle_foreign_key,
        }
        if additional:
            self.converters.update(additional)

    def handle_foreign_key(self, model, field, **kwargs):
        return field.descriptor, ModelSelectField(model=field.to, **kwargs)

    def convert(self, model, field, field_args):
        kwargs = dict(
            label=field.verbose_name,
            validators=[],
            filters=[],
            default=field.default,
            description=field.help_text,
        )
        if field_args:
            kwargs.update(field_args)

        if field.null:
            kwargs['validators'].append(validators.Optional())
        else:
            if isinstance(field, self.required):
                kwargs['validators'].append(validators.Required())

        field_class = type(field)

        if field_class in self.converters:
            return self.converters[field_class](model, field, **kwargs)
        elif field_class in self.defaults:
            return field.name, self.defaults[field_class](**kwargs)

def model_fields(model, allow_pk=False, only=None, exclude=None, field_args=None, converter=None):
    """
    Generate a dictionary of fields for a given Peewee model.

    See `model_form` docstring for description of parameters.
    """
    converter = converter or ModelConverter()
    field_args = field_args or {}

    model_fields = list(model._meta.get_sorted_fields())

    if not allow_pk:
        model_fields.pop(0)

    if only:
        model_fields = (x for x in model_fields if x[0] in only)
    elif exclude:
        model_fields = (x for x in model_fields if x[0] not in exclude)

    field_dict = {}
    for name, model_field in model_fields:
        field_info = converter.convert(model, model_field, field_args.get(name))
        if field_info is not None:
            field_name, field_obj = field_info
            field_dict[field_name] = field_obj

    return field_dict

def model_form(model, base_class=Form, allow_pk=False, only=None, exclude=None, field_args=None, converter=None):
    """
    Create a wtforms Form for a given Peewee model class::

        from wtfpeewee.orm import model_form
        from myproject.myapp.models import User
        UserForm = model_form(User)

    :param model:
        A Peewee model class
    :param base_class:
        Base form class to extend from. Must be a ``wtforms.Form`` subclass.
    :param only:
        An optional iterable with the property names that should be included in
        the form. Only these properties will have fields.
    :param exclude:
        An optional iterable with the property names that should be excluded
        from the form. All other properties will have fields.
    :param field_args:
        An optional dictionary of field names mapping to keyword arguments used
        to construct each field object.
    :param converter:
        A converter to generate the fields based on the model properties. If
        not set, ``ModelConverter`` is used.
    """
    field_dict = model_fields(model, allow_pk, only, exclude, field_args, converter)
    return type(model.__name__ + 'Form', (base_class, ), field_dict)
<commit_msg>Use a FloatField instead of a DecimalField<commit_after>"""
Tools for generating forms based on Peewee models
(cribbed from wtforms.ext.django)
"""
from wtforms import fields as f
from wtforms import Form
from wtforms import validators
from wtfpeewee.fields import ModelSelectField

from peewee import PrimaryKeyField, IntegerField, FloatField, DateTimeField,\
    BooleanField, CharField, TextField, ForeignKeyField

__all__ = (
    'model_fields', 'model_form',
)

class ModelConverter(object):
    defaults = {
        PrimaryKeyField: f.HiddenField,
        IntegerField: f.IntegerField,
        FloatField: f.FloatField,
        DateTimeField: f.DateTimeField,
        BooleanField: f.BooleanField,
        CharField: f.TextField,
        TextField: f.TextAreaField,
    }
    required = (DateTimeField, CharField, TextField, ForeignKeyField,)

    def __init__(self, additional=None):
        self.converters = {
            ForeignKeyField: self.handle_foreign_key,
        }
        if additional:
            self.converters.update(additional)

    def handle_foreign_key(self, model, field, **kwargs):
        return field.descriptor, ModelSelectField(model=field.to, **kwargs)

    def convert(self, model, field, field_args):
        kwargs = dict(
            label=field.verbose_name,
            validators=[],
            filters=[],
            default=field.default,
            description=field.help_text,
        )
        if field_args:
            kwargs.update(field_args)

        if field.null:
            kwargs['validators'].append(validators.Optional())
        else:
            if isinstance(field, self.required):
                kwargs['validators'].append(validators.Required())

        field_class = type(field)

        if field_class in self.converters:
            return self.converters[field_class](model, field, **kwargs)
        elif field_class in self.defaults:
            return field.name, self.defaults[field_class](**kwargs)

def model_fields(model, allow_pk=False, only=None, exclude=None, field_args=None, converter=None):
    """
    Generate a dictionary of fields for a given Peewee model.

    See `model_form` docstring for description of parameters.
    """
    converter = converter or ModelConverter()
    field_args = field_args or {}

    model_fields = list(model._meta.get_sorted_fields())

    if not allow_pk:
        model_fields.pop(0)

    if only:
        model_fields = (x for x in model_fields if x[0] in only)
    elif exclude:
        model_fields = (x for x in model_fields if x[0] not in exclude)

    field_dict = {}
    for name, model_field in model_fields:
        field_info = converter.convert(model, model_field, field_args.get(name))
        if field_info is not None:
            field_name, field_obj = field_info
            field_dict[field_name] = field_obj

    return field_dict

def model_form(model, base_class=Form, allow_pk=False, only=None, exclude=None, field_args=None, converter=None):
    """
    Create a wtforms Form for a given Peewee model class::

        from wtfpeewee.orm import model_form
        from myproject.myapp.models import User
        UserForm = model_form(User)

    :param model:
        A Peewee model class
    :param base_class:
        Base form class to extend from. Must be a ``wtforms.Form`` subclass.
    :param only:
        An optional iterable with the property names that should be included in
        the form. Only these properties will have fields.
    :param exclude:
        An optional iterable with the property names that should be excluded
        from the form. All other properties will have fields.
    :param field_args:
        An optional dictionary of field names mapping to keyword arguments used
        to construct each field object.
    :param converter:
        A converter to generate the fields based on the model properties. If
        not set, ``ModelConverter`` is used.
    """
    field_dict = model_fields(model, allow_pk, only, exclude, field_args, converter)
    return type(model.__name__ + 'Form', (base_class, ), field_dict)
<|endoftext|>


<commit_before>"""
Module for handling HDF5 data within qcodes.
Based on hdf5 datawrapper from qtlab originally by Reinier Heeres and
Wolfgang Pfaff.

Contains:
- a data class (HDF5Data) which is essentially a wrapper of a h5py data
  object, adapted for usage with qcodes
- name generators in the style of qtlab Data objects
- functions to create standard data sets
"""

import os
import time
import h5py
import numpy as np
import logging
log = logging.getLogger(__name__)

class DateTimeGenerator:
    """
    Class to generate filenames / directories based on the date and time.
    """

    def __init__(self):
        pass

    def create_data_dir(self, datadir: str, name: str=None, ts=None,
                        datesubdir: bool=True, timesubdir: bool=True):
        """
        Create and return a new data directory.

        Input:
            datadir (string): base directory
            name (string): optional name of measurement
            ts (time.localtime()): timestamp which will be used
                if timesubdir=True
            datesubdir (bool): whether to create a subdirectory for the date
            timesubdir (bool): whether to create a subdirectory for the time

        Output:
            The directory to place the new file in
        """

        path = datadir
        if ts is None:
            ts = time.localtime()
        if datesubdir:
            path = os.path.join(path, time.strftime('%Y%m%d', ts))
        if timesubdir:
            tsd = time.strftime('%H%M%S', ts)
            timestamp_verified = False
            counter = 0
            # Verify if timestamp is unique by seeing if the folder exists
            while not timestamp_verified:
                counter += 1
                try:
                    measdirs = [d for d in os.listdir(path)
                                if d[:6] == tsd]
                    if len(measdirs) == 0:
                        timestamp_verified = True
                    else:
                        # if timestamp not unique, add one second
                        # This is quite a hack
                        ts = time.localtime((time.mktime(ts)+1))
                        tsd = time.strftime('%H%M%S', ts)
                    if counter >= 3600:
                        raise Exception()
                except OSError as err:
                    if 'cannot find the path specified' in str(err):
                        timestamp_verified = True
                    elif 'No such file or directory' in str(err):
                        timestamp_verified = True
                    else:
                        raise err
            if name is not None:
                path = os.path.join(path, tsd+'_'+name)
            else:
                path = os.path.join(path, tsd)

        return path, tsd

    def new_filename(self, data_obj, folder):
        """Return a new filename, based on name and timestamp."""
        path, tstr = self.create_data_dir(folder,
                                          name=data_obj._name,
                                          ts=data_obj._localtime)
        filename = '%s_%s.hdf5' % (tstr, data_obj._name)
        return os.path.join(path, filename)

class Data(h5py.File):

    def __init__(self, name: str, datadir: str):
        """
        Creates an empty data set including the file, for which the currently
        set file name generator is used.

        kwargs:
            name (string) : base name of the file
            datadir (string) : A folder will be created within the datadir
                using the standard timestamp structure
        """
        self._name = name

        self._localtime = time.localtime()
        self._timestamp = time.asctime(self._localtime)
        self._timemark = time.strftime('%H%M%S', self._localtime)
        self._datemark = time.strftime('%Y%m%d', self._localtime)

        self.filepath = DateTimeGenerator().new_filename(
            self, folder=datadir)

        self.filepath = self.filepath.replace("%timemark", self._timemark)

        self.folder, self._filename = os.path.split(self.filepath)
        if not os.path.isdir(self.folder):
            os.makedirs(self.folder)
        super(Data, self).__init__(self.filepath, 'a')
        self.flush()

def encode_to_utf8(s):
    """
    Required because h5py does not support python3 strings
    """
    # converts byte type to string because of h5py datasaving
    if isinstance(s, str):
        s = s.encode('utf-8')
    # If it is an array of value decodes individual entries
    elif isinstance(s, (np.ndarray, list, tuple)):
        s = [s.encode('utf-8') for s in s]
    return s

def write_dict_to_hdf5(data_dict: dict, entry_point, overwrite=False):
    """
    Args:
        data_dict (dict): dictionary to write to hdf5 file
        entry_point (hdf5 group.file) : location in the nested hdf5 structure
            where to write to.
    """
    for key, item in data_dict.items():
        # Basic types
        if isinstance(item, (str, float, int, bool, np.number,
                             np.float_, np.int_, np.bool_)):
            if not hasattr(key, "encode"):
                key = repr(key)
            try:
                entry_point.attrs[key] = item
            except Exception as e:
                log.error(e)
                log.error('Exception occurred while writing'
                      ' {}:{} of type {}'.format(key, item, type(item)))
        elif isinstance(item, np.ndarray):
            try:
                entry_point.create_dataset(key, data=item)
            except RuntimeError:
                if overwrite:
                    del entry_point[key]
                    entry_point.create_dataset(key, data=item)
                else:
                    raise

        elif item is None:
            # as h5py does not support saving None as attribute
            # I create special string, note that this can create
            # unexpected behaviour if someone saves a string with this name
            entry_point.attrs[key] = 'NoneType:__None__'

        elif isinstance(item, dict):
            try:
                entry_point.create_group(key)
            except RuntimeError:
                if overwrite:
                    del entry_point[key]
                    entry_point.create_group(key)
                else:
                    raise
            write_dict_to_hdf5(data_dict=item,
                               entry_point=entry_point[key],
                               overwrite=overwrite)
        elif isinstance(item, (list, tuple)):
            if len(item) > 0:
                elt_type = type(item[0])
                # Lists of a single type, are stored as an hdf5 dset
                if (all(isinstance(x, elt_type) for x in item) and
                        not isinstance(item[0], dict) and
                        not isinstance(item, tuple)):
                    if isinstance(item[0], (int, float,
                                            np.int32, np.int64)):
                        try:
                            entry_point.create_dataset(key,
                                                       data=np.array(item))
                        except RuntimeError:
                            if overwrite:
                                del entry_point[key]
                                entry_point.create_dataset(key,
                                                           data=np.array(item))
                            else:
                                raise
                        entry_point[key].attrs['list_type'] = 'array'

                    # strings are saved as a special dtype hdf5 dataset
                    elif isinstance(item[0], str):
                        dt = h5py.special_dtype(vlen=str)
                        data = np.array(item)
                        data = data.reshape((-1, 1))
                        try:
                            ds = entry_point.create_dataset(
                                key, (len(data), 1), dtype=dt)
                        except RuntimeError:
                            if overwrite:
                                del entry_point[key]
                                entry_point.create_dataset(
                                    key, (len(data), 1), dtype=dt)
                            else:
                                raise
                        ds.attrs['list_type'] = 'str'
                        ds[:] = data
                    else:
                        log.warning(
                            'List of type "{}" for "{}":"{}" not '
                            'supported, storing as string'.format(
                                elt_type, key, item))
                        entry_point.attrs[key] = str(item)
                # Storing of generic lists/tuples
                else:
                    try:
                        entry_point.create_group(key)
                    except RuntimeError:
                        if overwrite:
                            del entry_point[key]
                            entry_point.create_group(key)
                        else:
                            raise
                    # N.B. item is of type list
                    list_dct = {'list_idx_{}'.format(idx): entry for
                                idx, entry in enumerate(item)}
                    group_attrs = entry_point[key].attrs
                    if isinstance(item, tuple):
                        group_attrs['list_type'] = 'generic_tuple'
                    else:
                        group_attrs['list_type'] = 'generic_list'
                    group_attrs['list_length'] = len(item)
                    write_dict_to_hdf5(
                        data_dict=list_dct,
                        entry_point=entry_point[key],
                        overwrite=overwrite)
            else:
                # as h5py does not support saving None as attribute
                entry_point.attrs[key] = 'NoneType:__emptylist__'

        else:
            log.warning(
                'Type "{}" for "{}" (key): "{}" (item) at location {} '
                'not supported, '
                'storing as string'.format(type(item), key, item,
                                           entry_point))
            entry_point.attrs[key] = str(item)

def read_dict_from_hdf5(data_dict: dict, h5_group):
    """
    Reads a dictionary from an hdf5 file or group that was written using the
    corresponding "write_dict_to_hdf5" function defined above.

    Args:
        data_dict (dict):
                dictionary to which to add entries being read out.
                This argument exists because it allows nested calls of this
                function to add the data to an existing data_dict.
        h5_group  (hdf5 group):
                hdf5 file or group from which to read.
    """
    # if 'list_type' not in h5_group.attrs:
    for key, item in h5_group.items():
        if isinstance(item, h5py.Group):
            data_dict[key] = {}
            data_dict[key] = read_dict_from_hdf5(data_dict[key],
                                                 item)
        else:  # item either a group or a dataset
            if 'list_type' not in item.attrs:
                data_dict[key] = item[()]
            elif item.attrs['list_type'] == 'str':
                # lists of strings needs some special care, see also
                # the writing part in the writing function above.
                list_of_str = [x[0] for x in item[()]]
                data_dict[key] = list_of_str

            else:
                data_dict[key] = list(item[()])
    for key, item in h5_group.attrs.items():
        if isinstance(item, str):
            # Extracts "None" as an exception as h5py does not support
            # storing None, nested if statement to avoid elementwise
            # comparison warning
            if item == 'NoneType:__None__':
                item = None
            elif item == 'NoneType:__emptylist__':
                item = []
        data_dict[key] = item

    if 'list_type' in h5_group.attrs:
        if (h5_group.attrs['list_type'] == 'generic_list' or
                h5_group.attrs['list_type'] == 'generic_tuple'):
            list_dict = data_dict
            data_list = []
            for i in range(list_dict['list_length']):
                data_list.append(list_dict['list_idx_{}'.format(i)])

            if h5_group.attrs['list_type'] == 'generic_tuple':
                return tuple(data_list)
            else:
                return data_list
        else:
            raise NotImplementedError('cannot read "list_type":"{}"'.format(
                h5_group.attrs['list_type']))
    return data_dict
<commit_msg>Adds proper handling of qutip quantum object in write_dict_to_hdf5.<commit_after>"""
Module for handling HDF5 data within qcodes.
Based on hdf5 datawrapper from qtlab originally by Reinier Heeres and
Wolfgang Pfaff.

Contains:
- a data class (HDF5Data) which is essentially a wrapper of a h5py data
  object, adapted for usage with qcodes
- name generators in the style of qtlab Data objects
- functions to create standard data sets
"""

import os
import time
import h5py
import qutip
import numpy as np
import logging
log = logging.getLogger(__name__)

class DateTimeGenerator:
    """
    Class to generate filenames / directories based on the date and time.
    """

    def __init__(self):
        pass

    def create_data_dir(self, datadir: str, name: str=None, ts=None,
                        datesubdir: bool=True, timesubdir: bool=True):
        """
        Create and return a new data directory.

        Input:
            datadir (string): base directory
            name (string): optional name of measurement
            ts (time.localtime()): timestamp which will be used
                if timesubdir=True
            datesubdir (bool): whether to create a subdirectory for the date
            timesubdir (bool): whether to create a subdirectory for the time

        Output:
            The directory to place the new file in
        """

        path = datadir
        if ts is None:
            ts = time.localtime()
        if datesubdir:
            path = os.path.join(path, time.strftime('%Y%m%d', ts))
        if timesubdir:
            tsd = time.strftime('%H%M%S', ts)
            timestamp_verified = False
            counter = 0
            # Verify if timestamp is unique by seeing if the folder exists
            while not timestamp_verified:
                counter += 1
                try:
                    measdirs = [d for d in os.listdir(path)
                                if d[:6] == tsd]
                    if len(measdirs) == 0:
                        timestamp_verified = True
                    else:
                        # if timestamp not unique, add one second
                        # This is quite a hack
                        ts = time.localtime((time.mktime(ts)+1))
                        tsd = time.strftime('%H%M%S', ts)
                    if counter >= 3600:
                        raise Exception()
                except OSError as err:
                    if 'cannot find the path specified' in str(err):
                        timestamp_verified = True
                    elif 'No such file or directory' in str(err):
                        timestamp_verified = True
                    else:
                        raise err
            if name is not None:
                path = os.path.join(path, tsd+'_'+name)
            else:
                path = os.path.join(path, tsd)

        return path, tsd

    def new_filename(self, data_obj, folder):
        """Return a new filename, based on name and timestamp."""
        path, tstr = self.create_data_dir(folder,
                                          name=data_obj._name,
                                          ts=data_obj._localtime)
        filename = '%s_%s.hdf5' % (tstr, data_obj._name)
        return os.path.join(path, filename)

class Data(h5py.File):

    def __init__(self, name: str, datadir: str):
        """
        Creates an empty data set including the file, for which the currently
        set file name generator is used.

        kwargs:
            name (string) : base name of the file
            datadir (string) : A folder will be created within the datadir
                using the standard timestamp structure
        """
        self._name = name

        self._localtime = time.localtime()
        self._timestamp = time.asctime(self._localtime)
        self._timemark = time.strftime('%H%M%S', self._localtime)
        self._datemark = time.strftime('%Y%m%d', self._localtime)

        self.filepath = DateTimeGenerator().new_filename(
            self, folder=datadir)

        self.filepath = self.filepath.replace("%timemark", self._timemark)

        self.folder, self._filename = os.path.split(self.filepath)
        if not os.path.isdir(self.folder):
            os.makedirs(self.folder)
        super(Data, self).__init__(self.filepath, 'a')
        self.flush()

def encode_to_utf8(s):
    """
    Required because h5py does not support python3 strings
    """
    # converts byte type to string because of h5py datasaving
    if isinstance(s, str):
        s = s.encode('utf-8')
    # If it is an array of value decodes individual entries
    elif isinstance(s, (np.ndarray, list, tuple)):
        s = [s.encode('utf-8') for s in s]
    return s

def write_dict_to_hdf5(data_dict: dict, entry_point, overwrite=False):
    """
    Args:
        data_dict (dict): dictionary to write to hdf5 file
        entry_point (hdf5 group.file) : location in the nested hdf5 structure
            where to write to.
    """
    for key, item in data_dict.items():
        # Basic types
        if isinstance(item, (str, float, int, bool, np.number,
                             np.float_, np.int_, np.bool_)):
            if not hasattr(key, "encode"):
                key = repr(key)
            try:
                entry_point.attrs[key] = item
            except Exception as e:
                log.error(e)
                log.error('Exception occurred while writing'
                      ' {}:{} of type {}'.format(key, item, type(item)))
        elif isinstance(item, (np.ndarray, qutip.qobj.Qobj)):
            if isinstance(item, qutip.qobj.Qobj):
                item = item.full()
            try:
                print()
                print(key)
                entry_point.create_dataset(key, data=item)
            except RuntimeError:
                if overwrite:
                    del entry_point[key]
                    entry_point.create_dataset(key, data=item)
                else:
                    raise
        elif item is None:
            # as h5py does not support saving None as attribute
            # I create special string, note that this can create
            # unexpected behaviour if someone saves a string with this name
            entry_point.attrs[key] = 'NoneType:__None__'

        elif isinstance(item, dict):
            try:
                entry_point.create_group(key)
            except RuntimeError:
                if overwrite:
                    del entry_point[key]
                    entry_point.create_group(key)
                else:
                    raise
            write_dict_to_hdf5(data_dict=item,
                               entry_point=entry_point[key],
                               overwrite=overwrite)
        elif isinstance(item, (list, tuple)):
            if len(item) > 0:
                elt_type = type(item[0])
                # Lists of a single type, are stored as an hdf5 dset
                if (all(isinstance(x, elt_type) for x in item) and
                        not isinstance(item[0], dict) and
                        not isinstance(item, tuple)):
                    if isinstance(item[0], (int, float,
                                            np.int32, np.int64)):
                        try:
                            entry_point.create_dataset(key,
                                                       data=np.array(item))
                        except RuntimeError:
                            if overwrite:
                                del entry_point[key]
                                entry_point.create_dataset(key,
                                                           data=np.array(item))
                            else:
                                raise
                        entry_point[key].attrs['list_type'] = 'array'

                    # strings are saved as a special dtype hdf5 dataset
                    elif isinstance(item[0], str):
                        dt = h5py.special_dtype(vlen=str)
                        data = np.array(item)
                        data = data.reshape((-1, 1))
                        try:
                            ds = entry_point.create_dataset(
                                key, (len(data), 1), dtype=dt)
                        except RuntimeError:
                            if overwrite:
                                del entry_point[key]
                                entry_point.create_dataset(
                                    key, (len(data), 1), dtype=dt)
                            else:
                                raise
                        ds.attrs['list_type'] = 'str'
                        ds[:] = data
                    else:
                        log.warning(
                            'List of type "{}" for "{}":"{}" not '
                            'supported, storing as string'.format(
                                elt_type, key, item))
                        entry_point.attrs[key] = str(item)
                # Storing of generic lists/tuples
                else:
                    try:
                        entry_point.create_group(key)
                    except RuntimeError:
                        if overwrite:
                            del entry_point[key]
                            entry_point.create_group(key)
                        else:
                            raise
                    # N.B. item is of type list
                    list_dct = {'list_idx_{}'.format(idx): entry for
                                idx, entry in enumerate(item)}
                    group_attrs = entry_point[key].attrs
                    if isinstance(item, tuple):
                        group_attrs['list_type'] = 'generic_tuple'
                    else:
                        group_attrs['list_type'] = 'generic_list'
                    group_attrs['list_length'] = len(item)
                    write_dict_to_hdf5(
                        data_dict=list_dct,
                        entry_point=entry_point[key],
                        overwrite=overwrite)
            else:
                # as h5py does not support saving None as attribute
                entry_point.attrs[key] = 'NoneType:__emptylist__'

        else:
            log.warning(
                'Type "{}" for "{}" (key): "{}" (item) at location {} '
                'not supported, '
                'storing as string'.format(type(item), key, item,
                                           entry_point))
            entry_point.attrs[key] = str(item)

def read_dict_from_hdf5(data_dict: dict, h5_group):
    """
    Reads a dictionary from an hdf5 file or group that was written using the
    corresponding "write_dict_to_hdf5" function defined above.

    Args:
        data_dict (dict):
                dictionary to which to add entries being read out.
                This argument exists because it allows nested calls of this
                function to add the data to an existing data_dict.
        h5_group  (hdf5 group):
                hdf5 file or group from which to read.
    """
    # if 'list_type' not in h5_group.attrs:
    for key, item in h5_group.items():
        if isinstance(item, h5py.Group):
            data_dict[key] = {}
            data_dict[key] = read_dict_from_hdf5(data_dict[key],
                                                 item)
        else:  # item either a group or a dataset
            if 'list_type' not in item.attrs:
                data_dict[key] = item[()]
            elif item.attrs['list_type'] == 'str':
                # lists of strings needs some special care, see also
                # the writing part in the writing function above.
                list_of_str = [x[0] for x in item[()]]
                data_dict[key] = list_of_str

            else:
                data_dict[key] = list(item[()])
    for key, item in h5_group.attrs.items():
        if isinstance(item, str):
            # Extracts "None" as an exception as h5py does not support
            # storing None, nested if statement to avoid elementwise
            # comparison warning
            if item == 'NoneType:__None__':
                item = None
            elif item == 'NoneType:__emptylist__':
                item = []
        data_dict[key] = item

    if 'list_type' in h5_group.attrs:
        if (h5_group.attrs['list_type'] == 'generic_list' or
                h5_group.attrs['list_type'] == 'generic_tuple'):
            list_dict = data_dict
            data_list = []
            for i in range(list_dict['list_length']):
                data_list.append(list_dict['list_idx_{}'.format(i)])

            if h5_group.attrs['list_type'] == 'generic_tuple':
                return tuple(data_list)
            else:
                return data_list
        else:
            raise NotImplementedError('cannot read "list_type":"{}"'.format(
                h5_group.attrs['list_type']))
    return data_dict
<|endoftext|>


<commit_before>"""
nnblocks.py
=========

Individual NN blocks

by Maxim Ziatdinov (email: ziatdinovmax@gmail.com)
"""
from typing import Union, Tuple, Type

import torch
import torch.nn as nn
import torch.nn.functional as F

from warnings import warn

class ConvBlock(nn.Module):
    """
    Creates a block of layers each consisting of convolution operation,
    nonlinear activation and (optionally) batch normalization

    Args:
        ndim:
            Data dimensionality (1, 2, or 3)
        nb_layers:
            Number of layers in the block
        input_channels:
            Number of input channels for the block
        output_channels:
            Number of the output channels for the block
        kernel_size:
            Size of convolutional filter in pixels (Default: 3)
        stride:
            Stride of convolutional filter (Default: 1)
        padding:
            Value for edge padding (Default: 1)
        batch_norm:
            Add batch normalization to each layer in the block (Default: False)
        activation:
            Non-linear activation ("relu", "lrelu", "tanh", "softplus", or None)
        pool:
            Applies max-pooling operation at the end of the block

        Example:

        Get convolutional block with three 1D convolutions and a batch normalization

        >>> convblock1d = ConvBlock(
        >>>     ndim=1, nlayers=3, input_channels=1,
        >>>     output_channels=32, batch_norm=True)

        Get convolutional block with two 2D convolutions and max-pooling at the end

        >>> convblock1d = ConvBlock(
        >>>     ndim=2, nlayers=2, input_channels=1,
        >>>     output_channels=32, pool=True)
    """
    def __init__(self,
                 ndim: int,
                 nlayers: int,
                 input_channels: int,
                 output_channels: int,
                 kernel_size: Union[Tuple[int], int] = 3,
                 stride: Union[Tuple[int], int] = 1,
                 padding: Union[Tuple[int], int] = 1,
                 batchnorm: bool = False,
                 activation: str = "lrelu",
                 pool: bool = False,
                 **kwargs: float,
                 ) -> None:
        """
        Initializes module parameters
        """
        super(ConvBlock, self).__init__()
        if not 0 < ndim < 4:
            raise AssertionError("ndim must be equal to 1, 2 or 3")
        activation = get_activation(activation)
        block = []
        for i in range(nlayers):
            input_channels = output_channels if i > 0 else input_channels
            block.append(get_conv(ndim)(input_channels, output_channels,
                         kernel_size=kernel_size, stride=stride, padding=padding))
            if activation is not None:
                block.append(activation())
            if batchnorm:
                block.append(get_bnorm(ndim)(output_channels))
        if pool:
            block.append(get_maxpool(ndim)(2, 2))
        self.block = nn.Sequential(*block)

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        """
        Defines a forward pass
        """
        output = self.block(x)
        return output

class UpsampleBlock(nn.Module):
    """
    Upsampling performed using bilinear or nearest-neigbor interpolation
    followed by 1-by-1 convolution, which an be used to reduce a number of
    feature channels

    Args:
        ndim:
            Data dimensionality (1, 2, or 3)
        input_channels:
            Number of input channels for the block
        output_channels:
            Number of the output channels for the block
        scale_factor:
            Scale factor for upsampling
        mode:
            Upsampling mode. Select between "bilinear" and "nearest"
            (Default: bilinear for 2D, nearest for 1D and 3D)

        Example:

        >>> # Get a 2-dimenisonal upsampling module with 2x channel reduction
        >>> upsample2d = UpsampleBlock(ndim=1, input_channels=64, output_channels=32,
        >>>                            scale_factor=2, mode="bilinear")
    """
    def __init__(self,
                 ndim: int,
                 input_channels: int,
                 output_channels: int,
                 scale_factor: int = 2,
                 mode: str = "bilinear") -> None:
        """
        Initializes module parameters
        """
        super(UpsampleBlock, self).__init__()
        warn_msg = ("'bilinear' mode is not supported for 1D and 3D;" +
                    " switching to 'nearest' mode")
        if mode not in ("bilinear", "nearest"):
            raise NotImplementedError(
                "Use 'bilinear' or 'nearest' for upsampling mode")
        if not 0 < ndim < 4:
            raise AssertionError("ndim must be equal to 1, 2 or 3")
        if mode == "bilinear" and ndim in (3, 1):
            warn(warn_msg, category=UserWarning)
            mode = "nearest"
        self.mode = mode
        self.scale_factor = scale_factor
        self.conv = get_conv(ndim)(
            input_channels, output_channels,
            kernel_size=1, stride=1, padding=0)

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        """
        Defines a forward pass
        """
        x = F.interpolate(
            x, scale_factor=self.scale_factor, mode=self.mode)
        return self.conv(x)

def get_bnorm(dim: int) -> Type[nn.Module]:
    bn_dict = {1: nn.BatchNorm1d, 2: nn.BatchNorm2d, 3: nn.BatchNorm3d}
    return bn_dict[dim]

def get_conv(dim: int) -> Type[nn.Module]:
    conv_dict = {1: nn.Conv1d, 2: nn.Conv2d, 3: nn.Conv3d}
    return conv_dict[dim]

def get_maxpool(dim: int) -> Type[nn.Module]:
    conv_dict = {1: nn.MaxPool1d, 2: nn.MaxPool2d, 3: nn.MaxPool3d}
    return conv_dict[dim]

def get_activation(activation: int) -> Type[nn.Module]:
    if activation is None:
        return
    activations = {"lrelu": nn.LeakyReLU, "tanh": nn.Tanh,
                   "softplus": nn.Softplus, "relu": nn.ReLU}
    return activations[activation]
<commit_msg>Add features2latent and latent2features blocks<commit_after>"""
nnblocks.py
=========

Individual NN blocks

by Maxim Ziatdinov (email: ziatdinovmax@gmail.com)
"""
from typing import Union, Tuple, Type

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.tensor as tt

from warnings import warn

class ConvBlock(nn.Module):
    """
    Creates a block of layers each consisting of convolution operation,
    nonlinear activation and (optionally) batch normalization

    Args:
        ndim:
            Data dimensionality (1, 2, or 3)
        nb_layers:
            Number of layers in the block
        input_channels:
            Number of input channels for the block
        output_channels:
            Number of the output channels for the block
        kernel_size:
            Size of convolutional filter in pixels (Default: 3)
        stride:
            Stride of convolutional filter (Default: 1)
        padding:
            Value for edge padding (Default: 1)
        batch_norm:
            Add batch normalization to each layer in the block (Default: False)
        activation:
            Non-linear activation ("relu", "lrelu", "tanh", "softplus", or None)
        pool:
            Applies max-pooling operation at the end of the block

        Example:

        Get convolutional block with three 1D convolutions and a batch normalization

        >>> convblock1d = ConvBlock(
        >>>     ndim=1, nlayers=3, input_channels=1,
        >>>     output_channels=32, batch_norm=True)

        Get convolutional block with two 2D convolutions and max-pooling at the end

        >>> convblock1d = ConvBlock(
        >>>     ndim=2, nlayers=2, input_channels=1,
        >>>     output_channels=32, pool=True)
    """
    def __init__(self,
                 ndim: int,
                 nlayers: int,
                 input_channels: int,
                 output_channels: int,
                 kernel_size: Union[Tuple[int], int] = 3,
                 stride: Union[Tuple[int], int] = 1,
                 padding: Union[Tuple[int], int] = 1,
                 batchnorm: bool = False,
                 activation: str = "lrelu",
                 pool: bool = False,
                 ) -> None:
        """
        Initializes module parameters
        """
        super(ConvBlock, self).__init__()
        if not 0 < ndim < 4:
            raise AssertionError("ndim must be equal to 1, 2 or 3")
        activation = get_activation(activation)
        block = []
        for i in range(nlayers):
            input_channels = output_channels if i > 0 else input_channels
            block.append(get_conv(ndim)(input_channels, output_channels,
                         kernel_size=kernel_size, stride=stride, padding=padding))
            if activation is not None:
                block.append(activation())
            if batchnorm:
                block.append(get_bnorm(ndim)(output_channels))
        if pool:
            block.append(get_maxpool(ndim)(2, 2))
        self.block = nn.Sequential(*block)

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        """
        Defines a forward pass
        """
        output = self.block(x)
        return output

class UpsampleBlock(nn.Module):
    """
    Upsampling performed using bilinear or nearest-neigbor interpolation
    followed by 1-by-1 convolution, which an be used to reduce a number of
    feature channels

    Args:
        ndim:
            Data dimensionality (1, 2, or 3)
        input_channels:
            Number of input channels for the block
        output_channels:
            Number of the output channels for the block
        scale_factor:
            Scale factor for upsampling
        mode:
            Upsampling mode. Select between "bilinear" and "nearest"
            (Default: bilinear for 2D, nearest for 1D and 3D)

        Example:

        >>> # Get a 2-dimenisonal upsampling module with 2x channel reduction
        >>> upsample2d = UpsampleBlock(ndim=1, input_channels=64, output_channels=32,
        >>>                            scale_factor=2, mode="bilinear")
    """
    def __init__(self,
                 ndim: int,
                 input_channels: int,
                 output_channels: int,
                 scale_factor: int = 2,
                 mode: str = "bilinear") -> None:
        """
        Initializes module parameters
        """
        super(UpsampleBlock, self).__init__()
        warn_msg = ("'bilinear' mode is not supported for 1D and 3D;" +
                    " switching to 'nearest' mode")
        if mode not in ("bilinear", "nearest"):
            raise NotImplementedError(
                "Use 'bilinear' or 'nearest' for upsampling mode")
        if not 0 < ndim < 4:
            raise AssertionError("ndim must be equal to 1, 2 or 3")
        if mode == "bilinear" and ndim in (3, 1):
            warn(warn_msg, category=UserWarning)
            mode = "nearest"
        self.mode = mode
        self.scale_factor = scale_factor
        self.conv = get_conv(ndim)(
            input_channels, output_channels,
            kernel_size=1, stride=1, padding=0)

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        """
        Defines a forward pass
        """
        x = F.interpolate(
            x, scale_factor=self.scale_factor, mode=self.mode)
        return self.conv(x)

class features_to_latent(nn.Module):
    """
    Maps features (usually, from a convolutional net/layer) to latent space
    """
    def __init__(self, input_dim: Tuple[int], latent_dim: int = 2) -> None:
        super(features_to_latent, self).__init__()
        print(input_dim)
        self.reshape_ = torch.prod(tt(input_dim))
        self.fc_latent = nn.Linear(self.reshape_, latent_dim)

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        x = x.view(-1, self.reshape_)
        return self.fc_latent(x)

class latent_to_features(nn.Module):
    """
    Maps latent vector to feature space
    """
    def __init__(self, latent_dim: int, out_dim: Tuple[int]) -> None:
        super(latent_to_features, self).__init__()
        self.reshape_ = out_dim
        self.fc = nn.Linear(latent_dim, torch.prod(tt(out_dim)).item())

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        x = self.fc(x)
        return x.view(-1, *self.reshape_)

def get_bnorm(dim: int) -> Type[nn.Module]:
    bn_dict = {1: nn.BatchNorm1d, 2: nn.BatchNorm2d, 3: nn.BatchNorm3d}
    return bn_dict[dim]

def get_conv(dim: int) -> Type[nn.Module]:
    conv_dict = {1: nn.Conv1d, 2: nn.Conv2d, 3: nn.Conv3d}
    return conv_dict[dim]

def get_maxpool(dim: int) -> Type[nn.Module]:
    conv_dict = {1: nn.MaxPool1d, 2: nn.MaxPool2d, 3: nn.MaxPool3d}
    return conv_dict[dim]

def get_activation(activation: int) -> Type[nn.Module]:
    if activation is None:
        return
    activations = {"lrelu": nn.LeakyReLU, "tanh": nn.Tanh,
                   "softplus": nn.Softplus, "relu": nn.ReLU}
    return activations[activation]
<|endoftext|>


<commit_before># $Id$

import sys
import os
import urllib, urlparse
from gtk import *
from gnome.ui import *
from gtkhtml import *
import cStringIO, threading, Queue, time
import pycurl

# URL history
history = []
# Links for 'forward'
forward = []
# Number of concurrent connections to the web-server
NUM_THREADS = 4

# About
def about(button):
    GnomeAbout('GtkHTML Test with PycURL', '',
                'License GPL2',
                ['Original code by Matt Wilson, modified by Kjetil Jacobsen'],
                ('This is a useless application demonstrating the '
                'GtkHTML widget with Python and PycURL.')).show()

# HTML template for reporting internal errors
internal_error = """

<title>Error</title>

Error

%s

"""

# Worker threads downloads objects and passes them to the renderer
class WorkerThread(threading.Thread):

    def __init__(self, queue, render):
        threading.Thread.__init__(self)
        self.queue = queue      # Download request queue
        self.render = render    # Render output queue

    def run(self):
        curl = pycurl.Curl()
        curl.setopt(pycurl.FOLLOWLOCATION, 1)
        curl.setopt(pycurl.MAXREDIRS, 5)
        curl.setopt(pycurl.HTTPHEADER, ["User-Agent: GtkHTML/PycURL demo browser"])
        while 1:
            url, handle = self.queue.get()
            if url == None:
                curl.close()
                raise SystemExit
            b = cStringIO.StringIO()
            curl.setopt(pycurl.WRITEFUNCTION, b.write)
            curl.setopt(pycurl.URL, url)
            try:
                curl.perform()
            except pycurl.error, msg:
                b.write(internal_error % msg[1])
            except:
                msg = "Error retrieving URL: %s" % url
                b.write(internal_error % msg)
            # Flag empty documents to the renderer
            if b.tell() == 0:
                b.close()
                b = None
            # Enqueue the document on the rendering pipeline
            self.render.append((b, handle))

# Main rendering window, handles gtk events and sends requests to worker threads
class HtmlWindow(GtkHTML):

    def __init__(self):
        GtkHTML.__init__(self)
        self.queue = Queue.Queue()
        self.render = []
        self.threads = []
        for num_threads in range(NUM_THREADS):
            t = WorkerThread(self.queue, self.render)
            t.start()
            self.threads.append(t)

    def mainquit(self, *args):
        # Send a 'terminate' message to the worker threads
        for t in self.threads:
            t.queue.put((None, None))
        mainquit()

    def load_url(self, html, url):
        t1 = time.time()
        self.num_obj = 0
        if history: url = urllib.basejoin(history[-1], url)
        history.append(url)
        html.load_empty()
        handle = html.begin()
        url = url.strip()
        self.request_url(html, url, handle)
        self.urlentry.set_text(url)
        # Render incoming objects
        while self.num_obj > 0:
            if len(self.render) == 0:
                mainiteration(0)
                continue
            self.num_obj -= 1
            buf, handle = self.render.pop(0)
            if buf != None:
                html.write(handle, buf.getvalue())
                buf.close()
            html.end(handle, HTML_STREAM_OK)
        t2 = time.time()
        self.statusbar.set_text("Done (%.3f seconds)" % (t2-t1))

    def submit(self, html, method, path, params):
        print 'Submit is not supported yet'
        print method, path, params

    def request_url(self, html, url, handle):
        url = urllib.basejoin(history[-1], url)
        self.statusbar.set_text("Requesting URL: %s" % url)
        self.queue.put((url, handle))
        self.num_obj += 1

    def entry_activate(self, entry, html):
        url = entry.get_text()
        self.load_url(html, url)
        del forward[:]

    def do_back(self, _b):
        if len(history) > 1:
            forward.append(history[-1])
            del history[-1]
            url = history[-1]
            del history[-1]
            self.load_url(html, url)

    def do_forward(self, _b):
        if len(forward) == 0: return
        self.load_url(html, forward[-1])
        del forward[-1]

    def do_reload(self, _b):
        if len(history) == 0: return
        url = history[-1]
        del history[-1]
        self.load_url(html, url)

# Setup windows and menus
html = HtmlWindow()

file_menu = [
    UIINFO_ITEM_STOCK('Quit', None, html.mainquit, STOCK_MENU_QUIT),
]
help_menu = [
    UIINFO_ITEM_STOCK('About...', None, about, STOCK_MENU_ABOUT),
]
menus = [
    UIINFO_SUBTREE('File', file_menu),
    UIINFO_SUBTREE('Help', help_menu)
]

toolbar = [
    UIINFO_ITEM_STOCK('Back', 'Previous page', html.do_back, STOCK_MENU_BACK),
    UIINFO_ITEM_STOCK('Forward', 'Next page', html.do_forward, STOCK_MENU_FORWARD),
    UIINFO_ITEM_STOCK('Reload', 'Reload current page', html.do_reload, STOCK_MENU_REFRESH)
]

win = GnomeApp("html_demo", "Python GtkHTML Test")
win.set_wmclass("gtk_html_test", "GtkHTMLTest")
win.connect('delete_event', html.mainquit)

vbox = GtkVBox(spacing=3)
vbox.set_border_width(2)
vbox.show()
win.set_contents(vbox)

entry = GtkEntry()
html.connect('url_requested', html.request_url)
html.connect('link_clicked', html.load_url)
html.connect('submit', html.submit)
entry.connect('activate', html.entry_activate, html)
vbox.pack_start(entry, expand=FALSE)
entry.show()
html.urlentry = entry
html.set_usize(800, 600)

sw = GtkScrolledWindow()
sw.set_policy(POLICY_AUTOMATIC, POLICY_AUTOMATIC)
sw.add(html)
vbox.pack_start(sw)

sep = GtkHSeparator()
vbox.pack_start(sep, expand=FALSE)

status = GtkLabel('')
status.set_justify(JUSTIFY_LEFT)
status.set_alignment(0.0, 0.5)
win.set_statusbar(status)
win.create_menus(menus)
win.create_toolbar(toolbar)

html.statusbar = status
html.load_empty()

win.show_all()

threads_enter()
mainloop()
threads_leave()
<commit_msg>cleanups<commit_after># $Id$

import sys, os, urllib, cStringIO, threading, Queue, time
from gtk import *
from gnome.ui import *
from gtkhtml import *
import pycurl

# URL history
history = []
# Links for 'forward'
forward = []
# Number of concurrent connections to the web-server
NUM_THREADS = 4

# About
def about(button):
    GnomeAbout('GtkHTML Test with PycURL', '',
                'License GPL2',
                ['Original code by Matt Wilson, modified by Kjetil Jacobsen'],
                ('This is a useless application demonstrating the '
                'GtkHTML widget with Python and PycURL.')).show()

# HTML template for reporting internal errors
internal_error = """

<title>Error</title>

Error

%s

"""

# Worker threads downloads objects and passes them to the renderer
class WorkerThread(threading.Thread):

    def __init__(self, queue, render):
        threading.Thread.__init__(self)
        self.queue = queue      # Download request queue
        self.render = render    # Render output queue

    def run(self):
        curl = pycurl.Curl()
        curl.setopt(pycurl.FOLLOWLOCATION, 1)
        curl.setopt(pycurl.MAXREDIRS, 5)
        curl.setopt(pycurl.HTTPHEADER, ["User-Agent: GtkHTML/PycURL demo browser"])
        while 1:
            url, handle = self.queue.get()
            if url == None:
                curl.close()
                raise SystemExit
            b = cStringIO.StringIO()
            curl.setopt(pycurl.WRITEFUNCTION, b.write)
            curl.setopt(pycurl.URL, url)
            try:
                curl.perform()
            except pycurl.error, msg:
                b.write(internal_error % msg[1])
            except:
                msg = "Error retrieving URL: %s" % url
                b.write(internal_error % msg)
            # Flag empty documents to the renderer
            if b.tell() == 0:
                b.close()
                b = None
            # Enqueue the document on the rendering pipeline
            self.render.append((b, handle))

# Main rendering window, handles gtk events and sends requests to worker threads
class HtmlWindow(GtkHTML):

    def __init__(self):
        GtkHTML.__init__(self)
        self.queue = Queue.Queue()
        self.render = []
        self.threads = []
        for num_threads in range(NUM_THREADS):
            t = WorkerThread(self.queue, self.render)
            t.start()
            self.threads.append(t)

    def mainquit(self, *args):
        # Send a 'terminate' message to the worker threads
        for t in self.threads:
            t.queue.put((None, None))
        mainquit()

    def load_url(self, html, url):
        t1 = time.time()
        self.num_obj = 0
        if history: url = urllib.basejoin(history[-1], url)
        history.append(url)
        html.load_empty()
        handle = html.begin()
        url = url.strip()
        self.request_url(html, url, handle)
        self.urlentry.set_text(url)
        # Render incoming objects
        while self.num_obj > 0:
            if len(self.render) == 0:
                mainiteration(0)
                continue
            self.num_obj -= 1
            buf, handle = self.render.pop(0)
            if buf != None:
                html.write(handle, buf.getvalue())
                buf.close()
            html.end(handle, HTML_STREAM_OK)
        t2 = time.time()
        self.statusbar.set_text("Done (%.3f seconds)" % (t2-t1))

    def submit(self, html, method, path, params):
        print 'Submit is not supported yet'
        print method, path, params

    def request_url(self, html, url, handle):
        url = urllib.basejoin(history[-1], url)
        self.statusbar.set_text("Requesting URL: %s" % url)
        self.queue.put((url, handle))
        self.num_obj += 1

    def entry_activate(self, entry, html):
        url = entry.get_text()
        self.load_url(html, url)
        del forward[:]

    def do_back(self, _b):
        if len(history) > 1:
            forward.append(history[-1])
            del history[-1]
            url = history[-1]
            del history[-1]
            self.load_url(html, url)

    def do_forward(self, _b):
        if len(forward) == 0: return
        self.load_url(html, forward[-1])
        del forward[-1]

    def do_reload(self, _b):
        if len(history) == 0: return
        url = history[-1]
        del history[-1]
        self.load_url(html, url)

# Setup windows and menus
html = HtmlWindow()

file_menu = [
    UIINFO_ITEM_STOCK('Quit', None, html.mainquit, STOCK_MENU_QUIT),
]
help_menu = [
    UIINFO_ITEM_STOCK('About...', None, about, STOCK_MENU_ABOUT),
]
menus = [
    UIINFO_SUBTREE('File', file_menu),
    UIINFO_SUBTREE('Help', help_menu)
]

toolbar = [
    UIINFO_ITEM_STOCK('Back', 'Previous page', html.do_back, STOCK_MENU_BACK),
    UIINFO_ITEM_STOCK('Forward', 'Next page', html.do_forward, STOCK_MENU_FORWARD),
    UIINFO_ITEM_STOCK('Reload', 'Reload current page', html.do_reload, STOCK_MENU_REFRESH)
]

win = GnomeApp("html_demo", "Python GtkHTML Test")
win.set_wmclass("gtk_html_test", "GtkHTMLTest")
win.connect('delete_event', html.mainquit)

vbox = GtkVBox(spacing=3)
vbox.set_border_width(2)
vbox.show()
win.set_contents(vbox)

entry = GtkEntry()
html.connect('url_requested', html.request_url)
html.connect('link_clicked', html.load_url)
html.connect('submit', html.submit)
entry.connect('activate', html.entry_activate, html)
vbox.pack_start(entry, expand=FALSE)
entry.show()
html.urlentry = entry
html.set_usize(800, 600)

sw = GtkScrolledWindow()
sw.set_policy(POLICY_AUTOMATIC, POLICY_AUTOMATIC)
sw.add(html)
vbox.pack_start(sw)

sep = GtkHSeparator()
vbox.pack_start(sep, expand=FALSE)

status = GtkLabel('')
status.set_justify(JUSTIFY_LEFT)
status.set_alignment(0.0, 0.5)
win.set_statusbar(status)
win.create_menus(menus)
win.create_toolbar(toolbar)

html.statusbar = status
html.load_empty()

win.show_all()

threads_enter()
mainloop()
threads_leave()
<|endoftext|>


<commit_before>import unittest

from code_parser import CodeParser
from file_browser import FileBrowser

class TestFileBrowser(unittest.TestCase):
    def test_get_files(self):
        files = FileBrowser.get_files(extension='*.java')
        self.assertEqual(3, len(files))

class TestCodeParser(unittest.TestCase):
    def test_get_requirements_and_test_cases(self):
        files = FileBrowser().get_files(extension='file1.java')
        self.assertTrue(1, len(files))
        parser = CodeParser()
        parser.get_requirements_and_test_cases(files[0])
        self.assertEqual(3, len(parser.req_test_mapping))

    def test_get_requirements_from_requirements_line(self):
        req = CodeParser().get_requirements_from_requirements_line('foo')
        self.assertEqual(0, len(req))

        req = CodeParser().get_requirements_from_requirements_line('// REQ:')
        self.assertEqual(1, len(req))

        req = CodeParser().get_requirements_from_requirements_line('   // REQ:   ')
        self.assertEqual(1, len(req))

        req = CodeParser().get_requirements_from_requirements_line('   // REQ: 1,2,3  ')
        self.assertEqual(3, len(req))
        self.assertEqual('1', req[0])
        self.assertEqual('2', req[1])
        self.assertEqual('3', req[2])

        req = CodeParser().get_requirements_from_requirements_line('// REQ: 1 ,2, 3456  ')
        self.assertEqual(3, len(req))
        self.assertEqual('1', req[0])
        self.assertEqual('2', req[1])
        self.assertEqual('3456', req[2])

    def test_is_test(self):
        self.assertTrue(CodeParser().is_test('@Test'))
        self.assertTrue(CodeParser().is_test(' @Test '))
        self.assertTrue(CodeParser().is_test(' @Test //some comment '))

    def test_get_test_method_name(self):
        self.assertFalse(CodeParser().get_method_name(' no method'))
        self.assertEqual('testTwo', CodeParser().get_method_name('    public void testTwo(){'))
        self.assertEqual('testOne', CodeParser().get_method_name('public void testOne(param){  //comment'))

class MatrixWriter:
    def __init__(self, req_tests_mapping):
        self.req_tests_mapping = req_tests_mapping
        self.requirements = self.get_requirements_set()

    def get_requirements_set(self):
        self.requirements = set()

        for key in self.req_tests_mapping:
            for req in self.req_tests_mapping[key]:
                self.requirements.add(req)

        return sorted(self.requirements)

    def get_tests_for_requirement(self, requirement):
        tests = list()
        for key in self.req_tests_mapping:
            if requirement in self.req_tests_mapping[key]:
                tests.append(key)

        return sorted(tests)

    def get_result_for_test(self, test):
        pass

    def write_table_header(self, filename, columns):
        # begin longtable definition
        filename.write('\\begin{longtable}{')
        for c in range(0, columns):
            filename.write('|c')
        filename.write('|}\n')

        #write caption
        filename.write('\\caption{Requirements traceability matrix}\\\\ \n')
        filename.write('\\hline \n')

    def write_table_header_requirements(self, filename, columns, range_start):
        filename.write('\\textbf{Req. ID} & ')

        for c in range(range_start, columns - 1):
            filename.write('\\textbf{')
            if len(self.requirements) > c:
                filename.write(self.requirements[c])
            filename.write('}')
            if c is not (columns - 2):
                filename.write(' & ')
            else:
                filename.write("\\\\ \n")

        filename.write('\\hline \n')
        filename.write('\\endfirsthead \n')
        filename.write('\\hline \n')

    def write_table_test_cases(self, filename, columns, range_start):
        filename.write('\\textbf{TestCase} & ')

        for c in range(range_start, columns - 1):
            filename.write('\\textbf{')
            filename.write('}')
            if c is not (columns - 2):
                filename.write(' & ')
            else:
                filename.write("\\\\ \n")

        filename.write('\\hline \n')

    def write_table_footer(self, filename):
        filename.write('\\end{longtable} \n')

    def write(self):
        # filename = os.path.dirname(os.path.abspath(__file__)) + "matrix.tex"
        filename = "matrix.tex"
        f = open(filename, 'w')

        columns = 10
        rangeCnt = 0
        reqCnt = 1;
        print self.requirements[10:20]
        for req in self.requirements[rangeCnt * columns:(rangeCnt * columns) + columns]:
            print rangeCnt * columns
            print (rangeCnt * columns) + columns
            print req
            if reqCnt % columns is 0:
                rangeCnt += 1
            reqCnt += 1

        self.write_table_header(f, columns)
        self.write_table_header_requirements(f, columns, 0)
        self.write_table_test_cases(f, columns, 0)
        self.write_table_footer(f)

        # write a fixed number of requirements, but the testcases relevant to those
        # cases can be over multiple pages
        for req in self.requirements:
            f.write(req + '\r\n')

        f.close()

class TestMatrixWriter(unittest.TestCase):
    def test_get_requirements_set(self):
        mapping = dict()
        mapping['a'] = ['req1', 'req2', 'req3']
        mapping['b'] = ['req2', 'req3', 'req4']
        writer = MatrixWriter(req_tests_mapping=mapping)
        self.assertEqual(4, len(writer.requirements))
        self.assertEqual(['req1', 'req2', 'req3', 'req4'], writer.requirements)

    def test_get_tests_for_requirement(self):
        mapping = dict()
        mapping['a'] = ['req1', 'req2', 'req3']
        mapping['b'] = ['req2', 'req3']
        mapping['c'] = ['req3', 'req4']
        writer = MatrixWriter(req_tests_mapping=mapping)
        self.assertEqual(['a'], writer.get_tests_for_requirement('req1'))
        self.assertEqual(['a', 'b'], writer.get_tests_for_requirement('req2'))
        self.assertEqual(['a', 'b', 'c'], writer.get_tests_for_requirement('req3'))
        self.assertEqual(['c'], writer.get_tests_for_requirement('req4'))

    def test_write_matrix_file(self):
        mapping = dict()
        mapping['a'] = ['req1', 'req2', 'req3']
        mapping['b'] = ['req2', 'req3', 'req4']
        writer = MatrixWriter(req_tests_mapping=mapping)
        writer.write()

    def test_get_result_for_test(self):
        self.assertFalse(False)

if __name__ == '__main__':
    unittest.main()<commit_msg>- generation of a traceability matrix implemented. Next step will be the test result in the matrix<commit_after>import unittest
from math import ceil

from code_parser import CodeParser
from file_browser import FileBrowser

class TestFileBrowser(unittest.TestCase):
    def test_get_files(self):
        files = FileBrowser.get_files(extension='*.java')
        self.assertEqual(3, len(files))

class TestCodeParser(unittest.TestCase):
    def test_get_requirements_and_test_cases(self):
        files = FileBrowser().get_files(extension='file1.java')
        self.assertTrue(1, len(files))
        parser = CodeParser()
        parser.get_requirements_and_test_cases(files[0])
        self.assertEqual(3, len(parser.req_test_mapping))

    def test_get_requirements_from_requirements_line(self):
        req = CodeParser().get_requirements_from_requirements_line('foo')
        self.assertEqual(0, len(req))

        req = CodeParser().get_requirements_from_requirements_line('// REQ:')
        self.assertEqual(1, len(req))

        req = CodeParser().get_requirements_from_requirements_line('   // REQ:   ')
        self.assertEqual(1, len(req))

        req = CodeParser().get_requirements_from_requirements_line('   // REQ: 1,2,3  ')
        self.assertEqual(3, len(req))
        self.assertEqual('1', req[0])
        self.assertEqual('2', req[1])
        self.assertEqual('3', req[2])

        req = CodeParser().get_requirements_from_requirements_line('// REQ: 1 ,2, 3456  ')
        self.assertEqual(3, len(req))
        self.assertEqual('1', req[0])
        self.assertEqual('2', req[1])
        self.assertEqual('3456', req[2])

    def test_is_test(self):
        self.assertTrue(CodeParser().is_test('@Test'))
        self.assertTrue(CodeParser().is_test(' @Test '))
        self.assertTrue(CodeParser().is_test(' @Test //some comment '))

    def test_get_test_method_name(self):
        self.assertFalse(CodeParser().get_method_name(' no method'))
        self.assertEqual('testTwo', CodeParser().get_method_name('    public void testTwo(){'))
        self.assertEqual('testOne', CodeParser().get_method_name('public void testOne(param){  //comment'))

class MatrixWriter:
    def __init__(self, req_tests_mapping):
        self.req_tests_mapping = req_tests_mapping
        self.requirements = self.get_requirements_set()

    def get_requirements_set(self):
        self.requirements = set()

        for key in self.req_tests_mapping:
            for req in self.req_tests_mapping[key]:
                self.requirements.add(req)

        return sorted(self.requirements)

    def get_tests_for_requirement(self, requirement):
        tests = list()
        for key in self.req_tests_mapping:
            if requirement in self.req_tests_mapping[key]:
                tests.append(key)

        return sorted(tests)

    def get_result_for_test(self, test):
        pass

    def write_table_header(self, filename, columns):
        # begin longtable definition
        filename.write('\\begin{longtable}{')
        for c in range(0, columns + 1):
            filename.write('|c')
        filename.write('|}\n')

        #write caption
        filename.write('\\caption{Requirements traceability matrix}\\\\ \n')
        filename.write('\\hline \n')

    def write_table_header_requirements(self, filename, columns, range_start):
        filename.write('\\textbf{Req. ID} & ')
        cnt = 0;
        for req in self.requirements[range_start:(range_start + columns)]:
            cnt += 1
            filename.write('\\textbf{')
            filename.write(req)
            filename.write('}')
            if cnt < len(self.requirements[range_start:(range_start + columns)]):
                filename.write(' & ')
        filename.write("\\\\ \n")
        filename.write('\\hline \n')
        filename.write('\\endfirsthead \n')
        filename.write('\\hline \n')

    def write_table_test_cases(self, filename, columns, range_start):
        filename.write('\\textbf{TestCase} & ')

        test_cases_for_requirement = list()
        cnt = 0
        for req in self.requirements[range_start:(range_start + columns)]:
            cnt += 1
            filename.write('\\textbf{')
            filename.write(str(len(self.get_tests_for_requirement(req))))
            test_cases_for_requirement.append(self.get_tests_for_requirement(req))
            filename.write('}')
            if cnt < len(self.requirements[range_start:(range_start + columns)]):
                filename.write(' & ')
        filename.write("\\\\ \n")
        filename.write('\\hline \n')

        test_cases_set = set()
        for test_cases in test_cases_for_requirement:
            for test_case in test_cases:
                test_cases_set.add(test_case)

        for test_case in test_cases_set:
            filename.write('\\textbf{')
            filename.write(test_case)
            filename.write('} & ')
            cnt = 0
            for req in self.requirements[range_start:(range_start + columns)]:
                cnt += 1
                filename.write('\\textbf{')
                #TODO check if test cases is passed failed when applicable
                filename.write('}')
                if cnt < len(self.requirements[range_start:(range_start + columns)]):
                    filename.write(' & ')
            filename.write("\\\\ \n")
            filename.write('\\hline \n')

    def write_table_footer(self, filename):
        filename.write('\\end{longtable} \n')

    def write(self):
        # filename = os.path.dirname(os.path.abspath(__file__)) + "matrix.tex"
        filename = "matrix.tex"
        f = open(filename, 'w')

        columns = 8
        iterations = 0
        while iterations < ceil(float(len(self.requirements)) / float(columns)):
            self.write_table_header(f, columns)
            self.write_table_header_requirements(f, columns, iterations * columns)
            self.write_table_test_cases(f, columns, iterations * columns)
            self.write_table_footer(f)
            iterations += 1

        f.close()

class TestMatrixWriter(unittest.TestCase):
    def test_get_requirements_set(self):
        mapping = dict()
        mapping['a'] = ['req1', 'req2', 'req3']
        mapping['b'] = ['req2', 'req3', 'req4']
        writer = MatrixWriter(req_tests_mapping=mapping)
        self.assertEqual(4, len(writer.requirements))
        self.assertEqual(['req1', 'req2', 'req3', 'req4'], writer.requirements)

    def test_get_tests_for_requirement(self):
        mapping = dict()
        mapping['a'] = ['req1', 'req2', 'req3']
        mapping['b'] = ['req2', 'req3']
        mapping['c'] = ['req3', 'req4']
        writer = MatrixWriter(req_tests_mapping=mapping)
        self.assertEqual(['a'], writer.get_tests_for_requirement('req1'))
        self.assertEqual(['a', 'b'], writer.get_tests_for_requirement('req2'))
        self.assertEqual(['a', 'b', 'c'], writer.get_tests_for_requirement('req3'))
        self.assertEqual(['c'], writer.get_tests_for_requirement('req4'))

    def test_write_matrix_file(self):
        mapping = dict()
        mapping['a'] = ['req1', 'req2', 'req3', 'req4', 'req5', 'req6', 'req7', 'req8', 'req9']
        mapping['b'] = ['req7', 'req8', 'req9', 'req10', 'req11', 'req12', 'req13', 'req14', 'req15']
        mapping['c'] = ['req7']
        mapping['d'] = ['req8']
        mapping['e'] = ['req9']
        mapping['f'] = ['req9']
        mapping['g'] = ['req9']
        mapping['h'] = ['req10']
        mapping['i'] = ['req11']
        mapping['j'] = ['req12']
        mapping['k'] = ['req13', 'req14', 'req15', 'req17','req18', 'req19', 'req20', 'req21']
        writer = MatrixWriter(req_tests_mapping=mapping)
        writer.write()

    @unittest.skip("not implemented yet")
    def test_get_result_for_test(self):
        self.assertFalse(True)

if __name__ == '__main__':
    unittest.main()<|endoftext|>