<commit_before>from future import standard_library
standard_library.install_aliases()
from builtins import object

from pickle import dumps
import simplejson
import os
import getpass
from socket import getfqdn
from uuid import uuid1

import numpy as np
try:
    from collections import OrderedDict
except ImportError:
    from ordereddict import OrderedDict

import prov.model as pm
from ..external.six import string_types

from .. import get_info
from .filemanip import (md5, hashlib, hash_infile)
from .. import logging
iflogger = logging.getLogger('interface')

foaf = pm.Namespace("foaf", "http://xmlns.com/foaf/0.1/")
dcterms = pm.Namespace("dcterms", "http://purl.org/dc/terms/")
nipype_ns = pm.Namespace("nipype", "http://nipy.org/nipype/terms/")
niiri = pm.Namespace("niiri", "http://iri.nidash.org/")
crypto = pm.Namespace("crypto",
                      ("http://id.loc.gov/vocabulary/preservation/"
                       "cryptographicHashFunctions/"))
get_id = lambda: niiri[uuid1().hex]

def get_attr_id(attr, skip=None):
    dictwithhash, hashval = get_hashval(attr, skip=skip)
    return niiri[hashval]

max_text_len = 1024000

def get_hashval(inputdict, skip=None):
    """Return a dictionary of our items with hashes for each file.

    Searches through dictionary items and if an item is a file, it
    calculates the md5 hash of the file contents and stores the
    file name and hash value as the new key value.

    However, the overall bunch hash is calculated only on the hash
    value of a file. The path and name of the file are not used in
    the overall hash calculation.

    Returns
    -------
    dict_withhash : dict
        Copy of our dictionary with the new file hashes included
        with each file.
    hashvalue : str
        The md5 hash value of the traited spec

    """

    dict_withhash = {}
    dict_nofilename = OrderedDict()
    keys = {}
    for key in inputdict:
        if skip is not None and key in skip:
            continue
        keys[key.uri] = key
    for key in sorted(keys):
        val = inputdict[keys[key]]
        outname = key
        try:
            if isinstance(val, pm.URIRef):
                val = val.decode()
        except AttributeError:
            pass
        if isinstance(val, pm.QualifiedName):
            val = val.uri
        if isinstance(val, pm.Literal):
            val = val.value
        dict_nofilename[outname] = _get_sorteddict(val)
        dict_withhash[outname] = _get_sorteddict(val, True)
    return (dict_withhash, md5(str(dict_nofilename)).hexdigest())

def _get_sorteddict(object, dictwithhash=False):
    if isinstance(object, dict):
        out = OrderedDict()
        for key, val in sorted(object.items()):
            if val:
                out[key] = _get_sorteddict(val, dictwithhash)
    elif isinstance(object, (list, tuple)):
        out = []
        for val in object:
            if val:
                out.append(_get_sorteddict(val, dictwithhash))
        if isinstance(object, tuple):
            out = tuple(out)
    else:
        if isinstance(object, string_types) and os.path.isfile(object):
            hash = hash_infile(object)
            if dictwithhash:
                out = (object, hash)
            else:
                out = hash
        elif isinstance(object, float):
            out = '%.10f' % object
        else:
            out = object
    return out

def safe_encode(x, as_literal=True):
    """Encodes a python value for prov
    """
    if x is None:
        value = "Unknown"
        if as_literal:
            return pm.Literal(value, pm.XSD['string'])
        else:
            return value
    try:
        if isinstance(x, (str, string_types)):
            if os.path.exists(x):
                value = 'file://%s%s' % (getfqdn(), x)
                if not as_literal:
                    return value
                try:
                    return pm.URIRef(value)
                except AttributeError:
                    return pm.Literal(value, pm.XSD['anyURI'])
            else:
                if len(x) > max_text_len:
                    value = x[:max_text_len - 13] + ['...Clipped...']
                else:
                    value = x
                if not as_literal:
                    return value
                return pm.Literal(value, pm.XSD['string'])
        if isinstance(x, int):
            if not as_literal:
                return x
            return pm.Literal(int(x), pm.XSD['integer'])
        if isinstance(x, float):
            if not as_literal:
                return x
            return pm.Literal(x, pm.XSD['float'])
        if isinstance(x, dict):
            outdict = {}
            for key, value in list(x.items()):
                encoded_value = safe_encode(value, as_literal=False)
                if isinstance(encoded_value, pm.Literal):
                    outdict[key] = encoded_value.json_representation()
                else:
                    outdict[key] = encoded_value
            if not as_literal:
                return simplejson.dumps(outdict)
            return pm.Literal(simplejson.dumps(outdict), pm.XSD['string'])
        if isinstance(x, list):
            try:
                nptype = np.array(x).dtype
                if nptype == np.dtype(object):
                    raise ValueError('dtype object')
            except ValueError as e:
                outlist = []
                for value in x:
                    encoded_value = safe_encode(value, as_literal=False)
                    if isinstance(encoded_value, pm.Literal):
                        outlist.append(encoded_value.json_representation())
                    else:
                        outlist.append(encoded_value)
            else:
                outlist = x
            if not as_literal:
                return simplejson.dumps(outlist)
            return pm.Literal(simplejson.dumps(outlist), pm.XSD['string'])
        if not as_literal:
            return dumps(x)
        return pm.Literal(dumps(x), nipype_ns['pickle'])
    except TypeError as e:
        iflogger.info(e)
        value = "Could not encode: " + str(e)
        if not as_literal:
            return value
        return pm.Literal(value, pm.XSD['string'])

def prov_encode(graph, value, create_container=True):
    if isinstance(value, list) and create_container:
        if len(value) == 0:
            encoded_literal = safe_encode(value)
            attr = {pm.PROV['value']: encoded_literal}
            id = get_attr_id(attr)
            entity = graph.entity(id, attr)
        elif len(value) > 1:
            try:
                entities = []
                for item in value:
                    item_entity = prov_encode(graph, item)
                    entities.append(item_entity)
                    if isinstance(item, list):
                        continue
                    if not isinstance(item_entity.value[0], string_types):
                        raise ValueError('Not a string literal')
                    if 'file://' not in item_entity.value[0]:
                        raise ValueError('No file found')
                id = get_id()
                entity = graph.collection(identifier=id)
                for item_entity in entities:
                    graph.hadMember(id, item_entity)
            except ValueError as e:
                iflogger.debug(e)
                entity = prov_encode(graph, value, create_container=False)
        else:
            entity = prov_encode(graph, value[0])
    else:
        encoded_literal = safe_encode(value)
        attr = {pm.PROV['value']: encoded_literal}
        if isinstance(value, string_types) and os.path.exists(value):
            attr.update({pm.PROV['location']: encoded_literal})
            if not os.path.isdir(value):
                sha512 = hash_infile(value, crypto=hashlib.sha512)
                attr.update({crypto['sha512']: pm.Literal(sha512,
                                                          pm.XSD['string'])})
                id = get_attr_id(attr, skip=[pm.PROV['location'],
                                             pm.PROV['value']])
            else:
                id = get_attr_id(attr, skip=[pm.PROV['location']])
        else:
            id = get_attr_id(attr)
        entity = graph.entity(id, attr)
    return entity

def write_provenance(results, filename='provenance', format='turtle'):
    ps = ProvStore()
    ps.add_results(results)
    return ps.write_provenance(filename=filename, format=format)

class ProvStore(object):

    def __init__(self):
        self.g = pm.ProvDocument() #ProvBundle(identifier=get_id())
        self.g.add_namespace(foaf)
        self.g.add_namespace(dcterms)
        self.g.add_namespace(nipype_ns)
        self.g.add_namespace(niiri)

    def add_results(self, results):
        if results.provenance:
            try:
                self.g.add_bundle(results.provenance)
            except pm.ProvException:
                self.g.add_bundle(results.provenance, get_id())
            return self.g
        runtime = results.runtime
        interface = results.interface
        inputs = results.inputs
        outputs = results.outputs
        classname = interface.__name__
        modulepath = "{0}.{1}".format(interface.__module__, interface.__name__)
        activitytype = ''.join([i.capitalize() for i in modulepath.split('.')])

        a0_attrs = {nipype_ns['module']: interface.__module__,
                    nipype_ns["interface"]: classname,
                    pm.PROV["type"]: nipype_ns[activitytype],
                    pm.PROV["label"]: classname,
                    nipype_ns['duration']: safe_encode(runtime.duration),
                    nipype_ns['workingDirectory']: safe_encode(runtime.cwd),
                    nipype_ns['returnCode']: safe_encode(runtime.returncode),
                    nipype_ns['platform']: safe_encode(runtime.platform),
                    nipype_ns['version']: safe_encode(runtime.version),
                    }
        a0_attrs[foaf["host"]] = pm.Literal(runtime.hostname,
                                            pm.XSD['anyURI'])

        try:
            a0_attrs.update({nipype_ns['command']: safe_encode(runtime.cmdline)})
            a0_attrs.update({nipype_ns['commandPath']:
                             safe_encode(runtime.command_path)})
            a0_attrs.update({nipype_ns['dependencies']:
                             safe_encode(runtime.dependencies)})
        except AttributeError:
            pass
        a0 = self.g.activity(get_id(), runtime.startTime, runtime.endTime,
                             a0_attrs)
        # environment
        id = get_id()
        env_collection = self.g.collection(id)
        env_collection.add_attributes({pm.PROV['type']:
                                           nipype_ns['Environment'],
                                       pm.PROV['label']: "Environment"})
        self.g.used(a0, id)
        # write environment entities
        for idx, (key, val) in enumerate(sorted(runtime.environ.items())):
            if key not in ['PATH', 'FSLDIR', 'FREESURFER_HOME', 'ANTSPATH',
                           'CAMINOPATH', 'CLASSPATH', 'LD_LIBRARY_PATH',
                           'DYLD_LIBRARY_PATH', 'FIX_VERTEX_AREA',
                           'FSF_OUTPUT_FORMAT', 'FSLCONFDIR', 'FSLOUTPUTTYPE',
                           'LOGNAME', 'USER',
                           'MKL_NUM_THREADS', 'OMP_NUM_THREADS']:
                continue
            in_attr = {pm.PROV["label"]: key,
                       nipype_ns["environmentVariable"]: key,
                       pm.PROV["value"]: safe_encode(val)}
            id = get_attr_id(in_attr)
            self.g.entity(id, in_attr)
            self.g.hadMember(env_collection, id)
        # write input entities
        if inputs:
            id = get_id()
            input_collection = self.g.collection(id)
            input_collection.add_attributes({pm.PROV['type']:
                                                 nipype_ns['Inputs'],
                                             pm.PROV['label']: "Inputs"})
            # write input entities
            for idx, (key, val) in enumerate(sorted(inputs.items())):
                in_entity = prov_encode(self.g, val).identifier
                self.g.hadMember(input_collection, in_entity)
                used_attr = {pm.PROV["label"]: key,
                             nipype_ns["inPort"]: key}
                self.g.used(activity=a0, entity=in_entity,
                            other_attributes=used_attr)
        # write output entities
        if outputs:
            id = get_id()
            output_collection = self.g.collection(id)
            if not isinstance(outputs, dict):
                outputs = outputs.get_traitsfree()
            output_collection.add_attributes({pm.PROV['type']:
                                                  nipype_ns['Outputs'],
                                              pm.PROV['label']:
                                                  "Outputs"})
            self.g.wasGeneratedBy(output_collection, a0)
            # write output entities
            for idx, (key, val) in enumerate(sorted(outputs.items())):
                out_entity = prov_encode(self.g, val).identifier
                self.g.hadMember(output_collection, out_entity)
                gen_attr = {pm.PROV["label"]: key,
                            nipype_ns["outPort"]: key}
                self.g.generation(out_entity, activity=a0,
                                  other_attributes=gen_attr)
        # write runtime entities
        id = get_id()
        runtime_collection = self.g.collection(id)
        runtime_collection.add_attributes({pm.PROV['type']:
                                               nipype_ns['Runtime'],
                                           pm.PROV['label']:
                                               "RuntimeInfo"})
        self.g.wasGeneratedBy(runtime_collection, a0)
        for key, value in sorted(runtime.items()):
            if not value:
                continue
            if key not in ['stdout', 'stderr', 'merged']:
                continue
            attr = {pm.PROV["label"]: key,
                    nipype_ns[key]: safe_encode(value)}
            id = get_id()
            self.g.entity(get_id(), attr)
            self.g.hadMember(runtime_collection, id)

        # create agents
        user_attr = {pm.PROV["type"]: pm.PROV["Person"],
                     pm.PROV["label"]: getpass.getuser(),
                     foaf["name"]: safe_encode(getpass.getuser())}
        user_agent = self.g.agent(get_attr_id(user_attr), user_attr)
        agent_attr = {pm.PROV["type"]: pm.PROV["SoftwareAgent"],
                      pm.PROV["label"]: "Nipype",
                      foaf["name"]: safe_encode("Nipype")}
        for key, value in list(get_info().items()):
            agent_attr.update({nipype_ns[key]: safe_encode(value)})
        software_agent = self.g.agent(get_attr_id(agent_attr), agent_attr)
        self.g.wasAssociatedWith(a0, user_agent, None, None,
                                 {pm.PROV["hadRole"]: nipype_ns["LoggedInUser"]})
        self.g.wasAssociatedWith(a0, software_agent)
        return self.g

    def write_provenance(self, filename='provenance', format='all'):
        if format in ['provn', 'all']:
            with open(filename + '.provn', 'wt') as fp:
                fp.writelines(self.g.get_provn())
        if format in ['json', 'all']:
            g.serialize(filename + '.json', format='json')
        return self.g
<commit_msg>fix: account for sets<commit_after>from future import standard_library
standard_library.install_aliases()
from builtins import object

from pickle import dumps
import simplejson
import os
import getpass
from socket import getfqdn
from uuid import uuid1

import numpy as np
try:
    from collections import OrderedDict
except ImportError:
    from ordereddict import OrderedDict

import prov.model as pm
from ..external.six import string_types

from .. import get_info
from .filemanip import (md5, hashlib, hash_infile)
from .. import logging
iflogger = logging.getLogger('interface')

foaf = pm.Namespace("foaf", "http://xmlns.com/foaf/0.1/")
dcterms = pm.Namespace("dcterms", "http://purl.org/dc/terms/")
nipype_ns = pm.Namespace("nipype", "http://nipy.org/nipype/terms/")
niiri = pm.Namespace("niiri", "http://iri.nidash.org/")
crypto = pm.Namespace("crypto",
                      ("http://id.loc.gov/vocabulary/preservation/"
                       "cryptographicHashFunctions/"))
get_id = lambda: niiri[uuid1().hex]

def get_attr_id(attr, skip=None):
    dictwithhash, hashval = get_hashval(attr, skip=skip)
    return niiri[hashval]

max_text_len = 1024000

def get_hashval(inputdict, skip=None):
    """Return a dictionary of our items with hashes for each file.

    Searches through dictionary items and if an item is a file, it
    calculates the md5 hash of the file contents and stores the
    file name and hash value as the new key value.

    However, the overall bunch hash is calculated only on the hash
    value of a file. The path and name of the file are not used in
    the overall hash calculation.

    Returns
    -------
    dict_withhash : dict
        Copy of our dictionary with the new file hashes included
        with each file.
    hashvalue : str
        The md5 hash value of the traited spec

    """

    dict_withhash = {}
    dict_nofilename = OrderedDict()
    keys = {}
    for key in inputdict:
        if skip is not None and key in skip:
            continue
        keys[key.uri] = key
    for key in sorted(keys):
        val = inputdict[keys[key]]
        outname = key
        try:
            if isinstance(val, pm.URIRef):
                val = val.decode()
        except AttributeError:
            pass
        if isinstance(val, pm.QualifiedName):
            val = val.uri
        if isinstance(val, pm.Literal):
            val = val.value
        dict_nofilename[outname] = _get_sorteddict(val)
        dict_withhash[outname] = _get_sorteddict(val, True)
    return (dict_withhash, md5(str(dict_nofilename)).hexdigest())

def _get_sorteddict(object, dictwithhash=False):
    if isinstance(object, dict):
        out = OrderedDict()
        for key, val in sorted(object.items()):
            if val:
                out[key] = _get_sorteddict(val, dictwithhash)
    elif isinstance(object, (list, tuple)):
        out = []
        for val in object:
            if val:
                out.append(_get_sorteddict(val, dictwithhash))
        if isinstance(object, tuple):
            out = tuple(out)
    else:
        if isinstance(object, string_types) and os.path.isfile(object):
            hash = hash_infile(object)
            if dictwithhash:
                out = (object, hash)
            else:
                out = hash
        elif isinstance(object, float):
            out = '%.10f' % object
        else:
            out = object
    return out

def safe_encode(x, as_literal=True):
    """Encodes a python value for prov
    """
    if x is None:
        value = "Unknown"
        if as_literal:
            return pm.Literal(value, pm.XSD['string'])
        else:
            return value
    try:
        if isinstance(x, (str, string_types)):
            if os.path.exists(x):
                value = 'file://%s%s' % (getfqdn(), x)
                if not as_literal:
                    return value
                try:
                    return pm.URIRef(value)
                except AttributeError:
                    return pm.Literal(value, pm.XSD['anyURI'])
            else:
                if len(x) > max_text_len:
                    value = x[:max_text_len - 13] + ['...Clipped...']
                else:
                    value = x
                if not as_literal:
                    return value
                return pm.Literal(value, pm.XSD['string'])
        if isinstance(x, int):
            if not as_literal:
                return x
            return pm.Literal(int(x), pm.XSD['integer'])
        if isinstance(x, float):
            if not as_literal:
                return x
            return pm.Literal(x, pm.XSD['float'])
        if isinstance(x, dict):
            outdict = {}
            for key, value in list(x.items()):
                encoded_value = safe_encode(value, as_literal=False)
                if isinstance(encoded_value, pm.Literal):
                    outdict[key] = encoded_value.json_representation()
                else:
                    outdict[key] = encoded_value
            if not as_literal:
                return simplejson.dumps(outdict)
            return pm.Literal(simplejson.dumps(outdict), pm.XSD['string'])
        if isinstance(x, list):
            try:
                nptype = np.array(x).dtype
                if nptype == np.dtype(object):
                    raise ValueError('dtype object')
            except ValueError as e:
                outlist = []
                for value in x:
                    encoded_value = safe_encode(value, as_literal=False)
                    if isinstance(encoded_value, pm.Literal):
                        outlist.append(encoded_value.json_representation())
                    else:
                        outlist.append(encoded_value)
            else:
                outlist = x
            if not as_literal:
                return simplejson.dumps(outlist)
            return pm.Literal(simplejson.dumps(outlist), pm.XSD['string'])
        if not as_literal:
            return dumps(x)
        return pm.Literal(dumps(x), nipype_ns['pickle'])
    except TypeError as e:
        iflogger.info(e)
        value = "Could not encode: " + str(e)
        if not as_literal:
            return value
        return pm.Literal(value, pm.XSD['string'])

def prov_encode(graph, value, create_container=True):
    if isinstance(value, list) and create_container:
        if len(value) == 0:
            encoded_literal = safe_encode(value)
            attr = {pm.PROV['value']: encoded_literal}
            id = get_attr_id(attr)
            entity = graph.entity(id, attr)
        elif len(value) > 1:
            try:
                entities = []
                for item in value:
                    item_entity = prov_encode(graph, item)
                    entities.append(item_entity)
                    if isinstance(item, list):
                        continue
                    if not isinstance(list(item_entity.value)[0], string_types):
                        raise ValueError('Not a string literal')
                    if 'file://' not in list(item_entity.value)[0]:
                        raise ValueError('No file found')
                id = get_id()
                entity = graph.collection(identifier=id)
                for item_entity in entities:
                    graph.hadMember(id, item_entity)
            except ValueError as e:
                iflogger.debug(e)
                entity = prov_encode(graph, value, create_container=False)
        else:
            entity = prov_encode(graph, value[0])
    else:
        encoded_literal = safe_encode(value)
        attr = {pm.PROV['value']: encoded_literal}
        if isinstance(value, string_types) and os.path.exists(value):
            attr.update({pm.PROV['location']: encoded_literal})
            if not os.path.isdir(value):
                sha512 = hash_infile(value, crypto=hashlib.sha512)
                attr.update({crypto['sha512']: pm.Literal(sha512,
                                                          pm.XSD['string'])})
                id = get_attr_id(attr, skip=[pm.PROV['location'],
                                             pm.PROV['value']])
            else:
                id = get_attr_id(attr, skip=[pm.PROV['location']])
        else:
            id = get_attr_id(attr)
        entity = graph.entity(id, attr)
    return entity

def write_provenance(results, filename='provenance', format='turtle'):
    ps = ProvStore()
    ps.add_results(results)
    return ps.write_provenance(filename=filename, format=format)

class ProvStore(object):

    def __init__(self):
        self.g = pm.ProvDocument() #ProvBundle(identifier=get_id())
        self.g.add_namespace(foaf)
        self.g.add_namespace(dcterms)
        self.g.add_namespace(nipype_ns)
        self.g.add_namespace(niiri)

    def add_results(self, results):
        if results.provenance:
            try:
                self.g.add_bundle(results.provenance)
            except pm.ProvException:
                self.g.add_bundle(results.provenance, get_id())
            return self.g
        runtime = results.runtime
        interface = results.interface
        inputs = results.inputs
        outputs = results.outputs
        classname = interface.__name__
        modulepath = "{0}.{1}".format(interface.__module__, interface.__name__)
        activitytype = ''.join([i.capitalize() for i in modulepath.split('.')])

        a0_attrs = {nipype_ns['module']: interface.__module__,
                    nipype_ns["interface"]: classname,
                    pm.PROV["type"]: nipype_ns[activitytype],
                    pm.PROV["label"]: classname,
                    nipype_ns['duration']: safe_encode(runtime.duration),
                    nipype_ns['workingDirectory']: safe_encode(runtime.cwd),
                    nipype_ns['returnCode']: safe_encode(runtime.returncode),
                    nipype_ns['platform']: safe_encode(runtime.platform),
                    nipype_ns['version']: safe_encode(runtime.version),
                    }
        a0_attrs[foaf["host"]] = pm.Literal(runtime.hostname,
                                            pm.XSD['anyURI'])

        try:
            a0_attrs.update({nipype_ns['command']: safe_encode(runtime.cmdline)})
            a0_attrs.update({nipype_ns['commandPath']:
                             safe_encode(runtime.command_path)})
            a0_attrs.update({nipype_ns['dependencies']:
                             safe_encode(runtime.dependencies)})
        except AttributeError:
            pass
        a0 = self.g.activity(get_id(), runtime.startTime, runtime.endTime,
                             a0_attrs)
        # environment
        id = get_id()
        env_collection = self.g.collection(id)
        env_collection.add_attributes({pm.PROV['type']:
                                           nipype_ns['Environment'],
                                       pm.PROV['label']: "Environment"})
        self.g.used(a0, id)
        # write environment entities
        for idx, (key, val) in enumerate(sorted(runtime.environ.items())):
            if key not in ['PATH', 'FSLDIR', 'FREESURFER_HOME', 'ANTSPATH',
                           'CAMINOPATH', 'CLASSPATH', 'LD_LIBRARY_PATH',
                           'DYLD_LIBRARY_PATH', 'FIX_VERTEX_AREA',
                           'FSF_OUTPUT_FORMAT', 'FSLCONFDIR', 'FSLOUTPUTTYPE',
                           'LOGNAME', 'USER',
                           'MKL_NUM_THREADS', 'OMP_NUM_THREADS']:
                continue
            in_attr = {pm.PROV["label"]: key,
                       nipype_ns["environmentVariable"]: key,
                       pm.PROV["value"]: safe_encode(val)}
            id = get_attr_id(in_attr)
            self.g.entity(id, in_attr)
            self.g.hadMember(env_collection, id)
        # write input entities
        if inputs:
            id = get_id()
            input_collection = self.g.collection(id)
            input_collection.add_attributes({pm.PROV['type']:
                                                 nipype_ns['Inputs'],
                                             pm.PROV['label']: "Inputs"})
            # write input entities
            for idx, (key, val) in enumerate(sorted(inputs.items())):
                in_entity = prov_encode(self.g, val).identifier
                self.g.hadMember(input_collection, in_entity)
                used_attr = {pm.PROV["label"]: key,
                             nipype_ns["inPort"]: key}
                self.g.used(activity=a0, entity=in_entity,
                            other_attributes=used_attr)
        # write output entities
        if outputs:
            id = get_id()
            output_collection = self.g.collection(id)
            if not isinstance(outputs, dict):
                outputs = outputs.get_traitsfree()
            output_collection.add_attributes({pm.PROV['type']:
                                                  nipype_ns['Outputs'],
                                              pm.PROV['label']:
                                                  "Outputs"})
            self.g.wasGeneratedBy(output_collection, a0)
            # write output entities
            for idx, (key, val) in enumerate(sorted(outputs.items())):
                out_entity = prov_encode(self.g, val).identifier
                self.g.hadMember(output_collection, out_entity)
                gen_attr = {pm.PROV["label"]: key,
                            nipype_ns["outPort"]: key}
                self.g.generation(out_entity, activity=a0,
                                  other_attributes=gen_attr)
        # write runtime entities
        id = get_id()
        runtime_collection = self.g.collection(id)
        runtime_collection.add_attributes({pm.PROV['type']:
                                               nipype_ns['Runtime'],
                                           pm.PROV['label']:
                                               "RuntimeInfo"})
        self.g.wasGeneratedBy(runtime_collection, a0)
        for key, value in sorted(runtime.items()):
            if not value:
                continue
            if key not in ['stdout', 'stderr', 'merged']:
                continue
            attr = {pm.PROV["label"]: key,
                    nipype_ns[key]: safe_encode(value)}
            id = get_id()
            self.g.entity(get_id(), attr)
            self.g.hadMember(runtime_collection, id)

        # create agents
        user_attr = {pm.PROV["type"]: pm.PROV["Person"],
                     pm.PROV["label"]: getpass.getuser(),
                     foaf["name"]: safe_encode(getpass.getuser())}
        user_agent = self.g.agent(get_attr_id(user_attr), user_attr)
        agent_attr = {pm.PROV["type"]: pm.PROV["SoftwareAgent"],
                      pm.PROV["label"]: "Nipype",
                      foaf["name"]: safe_encode("Nipype")}
        for key, value in list(get_info().items()):
            agent_attr.update({nipype_ns[key]: safe_encode(value)})
        software_agent = self.g.agent(get_attr_id(agent_attr), agent_attr)
        self.g.wasAssociatedWith(a0, user_agent, None, None,
                                 {pm.PROV["hadRole"]: nipype_ns["LoggedInUser"]})
        self.g.wasAssociatedWith(a0, software_agent)
        return self.g

    def write_provenance(self, filename='provenance', format='all'):
        if format in ['provn', 'all']:
            with open(filename + '.provn', 'wt') as fp:
                fp.writelines(self.g.get_provn())
        if format in ['json', 'all']:
            g.serialize(filename + '.json', format='json')
        return self.g
<|endoftext|>


<commit_before># Copyright 2013 Novo Nordisk Foundation Center for Biosustainability, DTU.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import, print_function

import six
from six.moves import range

import re
from collections import OrderedDict
from uuid import uuid1
from time import time
from datetime import datetime
import colorsys
from functools import wraps

import pip
import platform
from itertools import islice
from functools import partial

from numpy.random import RandomState

import logging

logger = logging.getLogger(__name__)

class ProblemCache(object):
    def __init__(self, model):
        self.time_machine = None
        self._model = model
        self.variables = {}
        self.constraints = {}
        self.original_objective = model.objective.expression
        self.time_machine = TimeMachine()
        self.transaction_id = None

    def begin_transaction(self):
        self.transaction_id = uuid1()
        self.time_machine(do=int, undo=int, bookmark=self.transaction_id)

    @property
    def model(self):
        return self._model

    def _rebuild_constraint(self, constraint):
        (expression, lb, ub, name) = constraint.expression, constraint.lb, constraint.ub, constraint.name

        def rebuild():
            self.model.solver._remove_constraint(constraint)
            new_constraint = self.model.solver.interface.Constraint(expression, lb=lb, ub=ub, name=name)
            self.constraints[name] = new_constraint
            self.model.solver._add_constraint(new_constraint, sloppy=True)

        return rebuild

    def _append_constraint(self, constraint_id, create, *args, **kwargs):
        self.constraints[constraint_id] = create(self.model, constraint_id, *args, **kwargs)
        self.model.solver._add_constraint(self.constraints[constraint_id])

    def _remove_constraint(self, constraint_id):
        constraint = self.constraints.pop(constraint_id)
        self.model.solver._remove_constraint(constraint)

    def _append_variable(self, variable_id, create, *args, **kwargs):
        self.variables[variable_id] = create(self.model, variable_id, *args, **kwargs)
        self.model.solver._add_variable(self.variables[variable_id])

    def _remove_variable(self, variable_id):
        variable = self.variables.pop(variable_id)
        self.model.solver._remove_variable(variable)

    def _rebuild_variable(self, variable):
        (type, lb, ub, name) = variable.type, variable.lb, variable.ub, variable.name

        def rebuild():
            self.model.solver._remove_variable(variable)
            new_variable = self.model.solver.interface.Variable(name, lb=lb, ub=ub, type=type)
            self.variables[name] = variable
            self.model.solver._add_variable(new_variable, sloppy=True)

        return rebuild

    def add_constraint(self, constraint_id, create, update, *args, **kwargs):
        if constraint_id in self.constraints:
            if update is not None:
                self.time_machine(
                    do=partial(update, self.model, self.constraints[constraint_id], *args, **kwargs),
                    undo=self._rebuild_constraint(self.constraints[constraint_id]))
        else:
            self.time_machine(
                do=partial(self._append_constraint, constraint_id, create, *args, **kwargs),
                undo=partial(self._remove_constraint, constraint_id)
            )

    def add_variable(self, variable_id, create, update, *args, **kwargs):
        if variable_id in self.variables:
            if update is not None:
                self.time_machine(
                    do=partial(update, self.model, self.variables[variable_id], *args, **kwargs),
                    undo=self._rebuild_variable(self.variables[variable_id]))
        else:
            self.time_machine(
                do=partial(self._append_variable, variable_id, create, *args, **kwargs),
                undo=partial(self._remove_variable, variable_id)
            )

    def reset(self):
        self.model.solver._remove_constraints(self.constraints.values())
        self.model.solver._remove_variables(self.variables.values())
        self.model.objective = self.original_objective
        self.variables = {}
        self.constraints = {}
        self.transaction_id = None
        self.time_machine.history.clear()

    def rollback(self):
        if self.transaction_id is None:
            raise RuntimeError("Start transaction must be called before rollback")
        self.time_machine.undo(self.transaction_id)
        self.transaction_id = None

class RandomGenerator(object):
    def __init__(self, seed=None):
        self._random = RandomState(seed=seed)

    def random(self):
        return self._random.rand()

    def randint(self, a, b=None):
        if b is None:
            b = a
            a = 0
        r = self._random.randint(a, high=b, size=1)
        return r[0]

    def sample(self, population, k):
        if k == 0:
            return []
        return self._random.choice(population, size=k, replace=False)

    def __getattr__(self, attr):
        return getattr(self._random, attr)

    def __getstate__(self):
        return {'_random': self._random}

    def __setstate__(self, d):
        self._random = d['_random']

    def uniform(self, low=0.0, high=1.0, size=None):
        return self._random.uniform(low, high, size)

class Singleton(object):
    """
    Singleton class to be extended
    """
    _instance = None

    def __new__(cls, *args, **kwargs):
        if not cls._instance:
            cls._instance = super(Singleton, cls).__new__(cls, *args, **kwargs)
        return cls._instance

class AutoVivification(dict):
    """Implementation of perl's autovivification feature. Checkout http://stackoverflow.com/a/652284/280182"""

    def __getitem__(self, item):
        try:
            return dict.__getitem__(self, item)
        except KeyError:
            value = self[item] = type(self)()
            return value

class TimeMachine(object):
    """Travel back and forth in time."""

    def __init__(self):
        super(TimeMachine, self).__init__()
        self.history = OrderedDict()

    def __call__(self, do=None, undo=None, bookmark=None):
        output = do()
        current_time = time()
        if bookmark is None:
            entry_id = uuid1()
        else:
            entry_id = bookmark
        # make sure that entry is added to the end of history
        self.history.pop(entry_id, None)
        self.history[entry_id] = {'unix_epoch': current_time, 'undo': undo, 'redo': do}
        return entry_id, output

    def __str__(self):
        info = '\n'
        for item in six.iteritems(self.history):
            info += self._history_item_to_str(item)
        return info

    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        self.reset()

    @staticmethod
    def _history_item_to_str(item):
        info = ''
        uuid, entry = item
        info += datetime.fromtimestamp(entry['unix_epoch']).strftime('%Y-%m-%d %H:%M:%S') + '\n'
        undo_entry = entry['undo']
        try:
            elements = undo_entry.func, undo_entry.args, undo_entry.keywords  # partial
            info += 'undo: ' + ' '.join([str(elem) for elem in elements]) + '\n'
        except AttributeError:  # normal python function
            info += 'undo: ' + undo_entry.__name__ + '\n'

        redo_entry = entry['redo']
        try:
            elements = redo_entry.func, redo_entry.args, redo_entry.keywords  # partial
            info += 'redo: ' + ' '.join([str(elem) for elem in elements]) + '\n'
        except AttributeError:
            info += 'redo: ' + redo_entry.__name__ + '\n'
        return info

    def undo(self, bookmark=None):
        if bookmark is None:
            try:
                (uuid, entry) = self.history.popitem()
                entry['undo']()
            except KeyError:  # history is empty
                pass
        elif bookmark in list(self.history.keys()):
            uuid = False
            while uuid is not bookmark:
                (uuid, entry) = self.history.popitem()
                entry['undo']()
        else:
            raise Exception(
                'Provided bookmark %s cannot be found in the time machine.')

    def redo(self):
        raise NotImplementedError

    def reset(self):
        if self.history:  # history is not empty
            self.undo(bookmark=list(self.history.keys())[0])

class Timer(object):
    """Taken from http://stackoverflow.com/a/5849861/280182"""

    def __init__(self, name=None):
        self.name = name

    def __enter__(self):
        self.tstart = time()

    def __exit__(self, type, value, traceback):
        if self.name:
            print('[%s]' % self.name, end=' ')
        print('Elapsed: %s' % (time() - self.tstart))

class IntelliContainer(object):
    def __init__(self, **kwargs):
        self._dict = dict(**kwargs)

    def __getattr__(self, value):
        return self._dict.get(value)

    def __setitem__(self, key, value):
        self._dict[key] = value

    def __iter__(self):
        return six.itervalues(self._dict)

    def __dir__(self):
        return list(self._dict.keys())

class DocInherit(object):
    """
    Adapted from http://code.activestate.com/recipes/576862/ (licensed under MIT)
    Docstring inheriting method descriptor

    The class itself is also used as a decorator
    """

    def __init__(self, mthd):
        self.mthd = mthd
        self.name = mthd.__name__

    def __get__(self, obj, cls):
        if obj:
            return self.get_with_inst(obj, cls)
        else:
            return self.get_no_inst(cls)

    def get_with_inst(self, obj, cls):

        overridden = getattr(super(cls, obj), self.name, None)

        @wraps(self.mthd, assigned=('__name__','__module__'))
        def f(*args, **kwargs):
            return self.mthd(obj, *args, **kwargs)

        return self.use_parent_doc(f, overridden)

    def get_no_inst(self, cls):

        for parent in cls.__mro__[1:]:
            overridden = getattr(parent, self.name, None)
            if overridden: break

        @wraps(self.mthd, assigned=('__name__','__module__'))
        def f(*args, **kwargs):
            return self.mthd(*args, **kwargs)

        return self.use_parent_doc(f, overridden)

    def use_parent_doc(self, func, source):
        if source is None:
            raise NameError("Can't find '%s' in parents" % self.name)
        func.__doc__ = source.__doc__
        return func

doc_inherit = DocInherit

# class DisplayItemsWidget(progressbar.widgets.Widget):
#     """Display an items[pbar.currval]
#
#     Examples
#     --------
#     import time
#     from progressbar import Progressbar, widges
#     pbar = ProgressBar(widgets=[DisplayItemsWidget(["asdf"+str(i) for i in range(10)]), widgets.Bar()])
#     pbar.maxval = 10
#     pbar.start()
#     for i in range(10):
#         time.sleep(.2)
#         pbar.update(i)
#     pbar.finish()
#     """
#
#     def __init__(self, items):
#         self.items = items
#
#     def update(self, pbar):
#         try:
#             return "%s" % self.items[pbar.currval]
#         except IndexError:
#             return ""

def partition_(lst, n):
    """Partition a list into n bite size chunks."""
    division = len(lst) / float(n)
    return [lst[int(round(division * i)): int(round(division * (i + 1)))] for i in range(n)]

def partition(ite, n):
    """Partition an iterable into n bite size chunks."""
    try:
        length = len(ite)
    except TypeError:
        ite = list(ite)
        length = len(ite)
    division = length / float(n)
    iterator = iter(ite)
    return [list(islice(iterator, 0, round(division * (i + 1)) - round(division * i))) for i in range(n)]

def flatten(l):
    return [item for sublist in l for item in sublist]

def generate_colors(n):
    hsv_tuples = [(v * 1.0 / n, 0.5, 0.5) for v in range(n)]
    color_map = {}
    for i in range(n):
        rgb = colorsys.hsv_to_rgb(*hsv_tuples[i])
        color = tuple([rgb[0] * 256, rgb[1] * 256, rgb[2] * 256])
        color_map[i] = '#%02x%02x%02x' % color
    return color_map

def memoize(function, memo={}):
    def wrapper(*args):
        if args in memo:
            return memo[args]
        else:
            rv = function(*args)
            memo[args] = rv
            return rv

    return wrapper

def get_system_info():
    # pip freeze (adapted from http://stackoverflow.com/a/24322465/280182)
    package_info = list()
    for dist in pip.get_installed_distributions():
        req = str(dist.as_requirement())
        package_info.append(req)
    return dict(package_info=package_info,
                platform=platform.platform(),
                machine=platform.machine(),
                system=platform.system())

def in_ipnb():
    """
    Check if it is running inside an IPython Notebook (updated for new notebooks)
    """
    try:
        import IPython

        ip = IPython.get_ipython()

        front_end = None
        if "IPKernelApp" in ip.config:
            front_end = ip.config.get('IPKernelApp').get("parent_appname")
        elif "KernelApp" in ip.config:
            front_end = ip.config.get('KernelApp').get("parent_appname")

        if isinstance(front_end, IPython.config.loader.LazyConfigValue) or front_end is None:
            if isinstance(ip, IPython.kernel.zmq.zmqshell.ZMQInteractiveShell):
                return True
            else:
                return False
        elif isinstance(front_end, six.string_types):
            if 'ipython-notebook' in front_end.lower():
                return True
            elif 'notebook' in front_end.lower():
                return True
    except Exception as e:
        logger.debug("Cannot determine if running a notebook because of %s" % e)
        return False
    return False

def str_to_valid_variable_name(s):
    """Adapted from http://stackoverflow.com/a/3303361/280182"""

    # Remove invalid characters
    s = re.sub('[^0-9a-zA-Z_]', '_', s)

    # Remove leading characters until we find a letter or underscore
    s = re.sub('^[^a-zA-Z_]+', '', s)

    return s
<commit_msg>Piggyback on pandas by using pandas.core.common.in_ipython_frontend to determine if running inside Jupyter/IPython notebook<commit_after># Copyright 2013 Novo Nordisk Foundation Center for Biosustainability, DTU.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import, print_function

import six
from six.moves import range

import re
from collections import OrderedDict
from uuid import uuid1
from time import time
from datetime import datetime
import colorsys
from functools import wraps

import pip
import platform
from itertools import islice
from functools import partial

import pandas
from numpy.random import RandomState

import logging

logger = logging.getLogger(__name__)

class ProblemCache(object):
    def __init__(self, model):
        self.time_machine = None
        self._model = model
        self.variables = {}
        self.constraints = {}
        self.original_objective = model.objective.expression
        self.time_machine = TimeMachine()
        self.transaction_id = None

    def begin_transaction(self):
        self.transaction_id = uuid1()
        self.time_machine(do=int, undo=int, bookmark=self.transaction_id)

    @property
    def model(self):
        return self._model

    def _rebuild_constraint(self, constraint):
        (expression, lb, ub, name) = constraint.expression, constraint.lb, constraint.ub, constraint.name

        def rebuild():
            self.model.solver._remove_constraint(constraint)
            new_constraint = self.model.solver.interface.Constraint(expression, lb=lb, ub=ub, name=name)
            self.constraints[name] = new_constraint
            self.model.solver._add_constraint(new_constraint, sloppy=True)

        return rebuild

    def _append_constraint(self, constraint_id, create, *args, **kwargs):
        self.constraints[constraint_id] = create(self.model, constraint_id, *args, **kwargs)
        self.model.solver._add_constraint(self.constraints[constraint_id])

    def _remove_constraint(self, constraint_id):
        constraint = self.constraints.pop(constraint_id)
        self.model.solver._remove_constraint(constraint)

    def _append_variable(self, variable_id, create, *args, **kwargs):
        self.variables[variable_id] = create(self.model, variable_id, *args, **kwargs)
        self.model.solver._add_variable(self.variables[variable_id])

    def _remove_variable(self, variable_id):
        variable = self.variables.pop(variable_id)
        self.model.solver._remove_variable(variable)

    def _rebuild_variable(self, variable):
        (type, lb, ub, name) = variable.type, variable.lb, variable.ub, variable.name

        def rebuild():
            self.model.solver._remove_variable(variable)
            new_variable = self.model.solver.interface.Variable(name, lb=lb, ub=ub, type=type)
            self.variables[name] = variable
            self.model.solver._add_variable(new_variable, sloppy=True)

        return rebuild

    def add_constraint(self, constraint_id, create, update, *args, **kwargs):
        if constraint_id in self.constraints:
            if update is not None:
                self.time_machine(
                    do=partial(update, self.model, self.constraints[constraint_id], *args, **kwargs),
                    undo=self._rebuild_constraint(self.constraints[constraint_id]))
        else:
            self.time_machine(
                do=partial(self._append_constraint, constraint_id, create, *args, **kwargs),
                undo=partial(self._remove_constraint, constraint_id)
            )

    def add_variable(self, variable_id, create, update, *args, **kwargs):
        if variable_id in self.variables:
            if update is not None:
                self.time_machine(
                    do=partial(update, self.model, self.variables[variable_id], *args, **kwargs),
                    undo=self._rebuild_variable(self.variables[variable_id]))
        else:
            self.time_machine(
                do=partial(self._append_variable, variable_id, create, *args, **kwargs),
                undo=partial(self._remove_variable, variable_id)
            )

    def reset(self):
        self.model.solver._remove_constraints(self.constraints.values())
        self.model.solver._remove_variables(self.variables.values())
        self.model.objective = self.original_objective
        self.variables = {}
        self.constraints = {}
        self.transaction_id = None
        self.time_machine.history.clear()

    def rollback(self):
        if self.transaction_id is None:
            raise RuntimeError("Start transaction must be called before rollback")
        self.time_machine.undo(self.transaction_id)
        self.transaction_id = None

class RandomGenerator(object):
    def __init__(self, seed=None):
        self._random = RandomState(seed=seed)

    def random(self):
        return self._random.rand()

    def randint(self, a, b=None):
        if b is None:
            b = a
            a = 0
        r = self._random.randint(a, high=b, size=1)
        return r[0]

    def sample(self, population, k):
        if k == 0:
            return []
        return self._random.choice(population, size=k, replace=False)

    def __getattr__(self, attr):
        return getattr(self._random, attr)

    def __getstate__(self):
        return {'_random': self._random}

    def __setstate__(self, d):
        self._random = d['_random']

    def uniform(self, low=0.0, high=1.0, size=None):
        return self._random.uniform(low, high, size)

class Singleton(object):
    """
    Singleton class to be extended
    """
    _instance = None

    def __new__(cls, *args, **kwargs):
        if not cls._instance:
            cls._instance = super(Singleton, cls).__new__(cls, *args, **kwargs)
        return cls._instance

class AutoVivification(dict):
    """Implementation of perl's autovivification feature. Checkout http://stackoverflow.com/a/652284/280182"""

    def __getitem__(self, item):
        try:
            return dict.__getitem__(self, item)
        except KeyError:
            value = self[item] = type(self)()
            return value

class TimeMachine(object):
    """Travel back and forth in time."""

    def __init__(self):
        super(TimeMachine, self).__init__()
        self.history = OrderedDict()

    def __call__(self, do=None, undo=None, bookmark=None):
        output = do()
        current_time = time()
        if bookmark is None:
            entry_id = uuid1()
        else:
            entry_id = bookmark
        # make sure that entry is added to the end of history
        self.history.pop(entry_id, None)
        self.history[entry_id] = {'unix_epoch': current_time, 'undo': undo, 'redo': do}
        return entry_id, output

    def __str__(self):
        info = '\n'
        for item in six.iteritems(self.history):
            info += self._history_item_to_str(item)
        return info

    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        self.reset()

    @staticmethod
    def _history_item_to_str(item):
        info = ''
        uuid, entry = item
        info += datetime.fromtimestamp(entry['unix_epoch']).strftime('%Y-%m-%d %H:%M:%S') + '\n'
        undo_entry = entry['undo']
        try:
            elements = undo_entry.func, undo_entry.args, undo_entry.keywords  # partial
            info += 'undo: ' + ' '.join([str(elem) for elem in elements]) + '\n'
        except AttributeError:  # normal python function
            info += 'undo: ' + undo_entry.__name__ + '\n'

        redo_entry = entry['redo']
        try:
            elements = redo_entry.func, redo_entry.args, redo_entry.keywords  # partial
            info += 'redo: ' + ' '.join([str(elem) for elem in elements]) + '\n'
        except AttributeError:
            info += 'redo: ' + redo_entry.__name__ + '\n'
        return info

    def undo(self, bookmark=None):
        if bookmark is None:
            try:
                (uuid, entry) = self.history.popitem()
                entry['undo']()
            except KeyError:  # history is empty
                pass
        elif bookmark in list(self.history.keys()):
            uuid = False
            while uuid is not bookmark:
                (uuid, entry) = self.history.popitem()
                entry['undo']()
        else:
            raise Exception(
                'Provided bookmark %s cannot be found in the time machine.')

    def redo(self):
        raise NotImplementedError

    def reset(self):
        if self.history:  # history is not empty
            self.undo(bookmark=list(self.history.keys())[0])

class Timer(object):
    """Taken from http://stackoverflow.com/a/5849861/280182"""

    def __init__(self, name=None):
        self.name = name

    def __enter__(self):
        self.tstart = time()

    def __exit__(self, type, value, traceback):
        if self.name:
            print('[%s]' % self.name, end=' ')
        print('Elapsed: %s' % (time() - self.tstart))

class IntelliContainer(object):
    def __init__(self, **kwargs):
        self._dict = dict(**kwargs)

    def __getattr__(self, value):
        return self._dict.get(value)

    def __setitem__(self, key, value):
        self._dict[key] = value

    def __iter__(self):
        return six.itervalues(self._dict)

    def __dir__(self):
        return list(self._dict.keys())

class DocInherit(object):
    """
    Adapted from http://code.activestate.com/recipes/576862/ (licensed under MIT)
    Docstring inheriting method descriptor

    The class itself is also used as a decorator
    """

    def __init__(self, mthd):
        self.mthd = mthd
        self.name = mthd.__name__

    def __get__(self, obj, cls):
        if obj:
            return self.get_with_inst(obj, cls)
        else:
            return self.get_no_inst(cls)

    def get_with_inst(self, obj, cls):

        overridden = getattr(super(cls, obj), self.name, None)

        @wraps(self.mthd, assigned=('__name__','__module__'))
        def f(*args, **kwargs):
            return self.mthd(obj, *args, **kwargs)

        return self.use_parent_doc(f, overridden)

    def get_no_inst(self, cls):

        for parent in cls.__mro__[1:]:
            overridden = getattr(parent, self.name, None)
            if overridden: break

        @wraps(self.mthd, assigned=('__name__','__module__'))
        def f(*args, **kwargs):
            return self.mthd(*args, **kwargs)

        return self.use_parent_doc(f, overridden)

    def use_parent_doc(self, func, source):
        if source is None:
            raise NameError("Can't find '%s' in parents" % self.name)
        func.__doc__ = source.__doc__
        return func

doc_inherit = DocInherit

# class DisplayItemsWidget(progressbar.widgets.Widget):
#     """Display an items[pbar.currval]
#
#     Examples
#     --------
#     import time
#     from progressbar import Progressbar, widges
#     pbar = ProgressBar(widgets=[DisplayItemsWidget(["asdf"+str(i) for i in range(10)]), widgets.Bar()])
#     pbar.maxval = 10
#     pbar.start()
#     for i in range(10):
#         time.sleep(.2)
#         pbar.update(i)
#     pbar.finish()
#     """
#
#     def __init__(self, items):
#         self.items = items
#
#     def update(self, pbar):
#         try:
#             return "%s" % self.items[pbar.currval]
#         except IndexError:
#             return ""

def partition_(lst, n):
    """Partition a list into n bite size chunks."""
    division = len(lst) / float(n)
    return [lst[int(round(division * i)): int(round(division * (i + 1)))] for i in range(n)]

def partition(ite, n):
    """Partition an iterable into n bite size chunks."""
    try:
        length = len(ite)
    except TypeError:
        ite = list(ite)
        length = len(ite)
    division = length / float(n)
    iterator = iter(ite)
    return [list(islice(iterator, 0, round(division * (i + 1)) - round(division * i))) for i in range(n)]

def flatten(l):
    return [item for sublist in l for item in sublist]

def generate_colors(n):
    hsv_tuples = [(v * 1.0 / n, 0.5, 0.5) for v in range(n)]
    color_map = {}
    for i in range(n):
        rgb = colorsys.hsv_to_rgb(*hsv_tuples[i])
        color = tuple([rgb[0] * 256, rgb[1] * 256, rgb[2] * 256])
        color_map[i] = '#%02x%02x%02x' % color
    return color_map

def memoize(function, memo={}):
    def wrapper(*args):
        if args in memo:
            return memo[args]
        else:
            rv = function(*args)
            memo[args] = rv
            return rv

    return wrapper

def get_system_info():
    # pip freeze (adapted from http://stackoverflow.com/a/24322465/280182)
    package_info = list()
    for dist in pip.get_installed_distributions():
        req = str(dist.as_requirement())
        package_info.append(req)
    return dict(package_info=package_info,
                platform=platform.platform(),
                machine=platform.machine(),
                system=platform.system())

def in_ipnb():
    """
    Check if it is running inside an IPython Notebook (updated for new notebooks)
    """
    return pandas.core.common.in_ipython_frontend()

def str_to_valid_variable_name(s):
    """Adapted from http://stackoverflow.com/a/3303361/280182"""

    # Remove invalid characters
    s = re.sub('[^0-9a-zA-Z_]', '_', s)

    # Remove leading characters until we find a letter or underscore
    s = re.sub('^[^a-zA-Z_]+', '', s)

    return s
<|endoftext|>


<commit_before>import json
from urlparse import parse_qs

from twisted.cred.portal import IRealm, Portal
from twisted.cred.checkers import FilePasswordDB
from twisted.internet import reactor
from twisted.web import server
from twisted.web.guard import HTTPAuthSessionWrapper, DigestCredentialFactory
from twisted.web.resource import Resource, IResource
from twisted.web.rewrite import RewriterResource
from zope.interface import implements

class Hashioki(Resource):
    def getChild(self, name, request):
        if name == '':
            return self
        return Resource.getChild(self, name, request)

    def render_GET(self, request):
        return """
If it existed, this would display the irc client for '{0}'.
""".format(request.irc_nick)

class API(Resource):
    """RESTful API

    Example URL: /freenode/#hashi/topic"""
    def getChild(self, name, request):
        if name == '':
            return self
        elif name == 'session':
            return Resource.getChild(self, name, request)
        else:
            return IRCServer(name)

    def render_GET(self, request):
        return """
Should document the API here!
"""

class APISession(Resource):
    isLeaf = True

    def render_GET(self, request):
        """Return session uid."""
        return json.dumps(request.getSession().uid)

    def render_POST(self, request):
        """Log out."""
        request.getSession().expire()
        return json.dumps(True)

class IRCServer(Resource):
    def __init__(self, name):
        Resource.__init__(self)
        self.name = name

class IRCChannel(Resource):
    isLeaf = True

    def render_GET(self, request):
        return json.dumps(request.irc_client.channels)

    def render_POST(self, request):
        chan_cmd = parse_qs(request.content.getvalue())
        for cmd, arg in chan_cmd.items():
            if cmd == "join":
                request.irc_client.join(arg[0])
            if cmd == "leave":
                request.irc_client.leave(arg[0])
        return ''

class History(Resource):
    isLeaf = True

    def render_GET(self, request):
        print(request.irc_client.history)
        print(request.args["source"])
        return str(request.irc_client.history[request.args["source"][0]])

def irc_rewriter(avatarId, client):
    def func(request):
        request.irc_nick = avatarId
        request.irc_client = client
    return func

class HashiUserRealm(object):
    implements(IRealm)

    def __init__(self, irc_clients, rest_api):
        self.irc_clients = irc_clients
        self.rest_api = rest_api

    def requestAvatar(self, avatarId, mind, *interfaces):
        if IResource in interfaces:
            client = self.irc_clients[avatarId]
            rewrite = RewriterResource(self.rest_api,
                                       irc_rewriter(avatarId, client))
            return (IResource,
                    rewrite,
                    lambda: None)
        raise NotImplementedError()

def start(irc_clients):
    root = Hashioki()
    rest_api = API()
    rest_api.putChild('session', APISession())
    root.putChild('api', rest_api)

    portal = Portal(HashiUserRealm(irc_clients, root),
                    [FilePasswordDB('httpd.password')])
    credentialFactory = DigestCredentialFactory("md5", "localhost:8080")
    resource = HTTPAuthSessionWrapper(portal, [credentialFactory])

    site = server.Site(resource)

    reactor.listenTCP(8080, site)

    return site
<commit_msg>Create an /api/networks/... path to prevent namespace collision between networks and other top level API objects.<commit_after>import json
from urlparse import parse_qs

from twisted.cred.portal import IRealm, Portal
from twisted.cred.checkers import FilePasswordDB
from twisted.internet import reactor
from twisted.web import server
from twisted.web.guard import HTTPAuthSessionWrapper, DigestCredentialFactory
from twisted.web.resource import Resource, IResource
from twisted.web.rewrite import RewriterResource
from zope.interface import implements

class Hashioki(Resource):
    def getChild(self, name, request):
        if name == '':
            return self
        return Resource.getChild(self, name, request)

    def render_GET(self, request):
        return """
If it existed, this would display the irc client for '{0}'.
""".format(request.irc_nick)

class API(Resource):
    """RESTful API

    Example URL: /freenode/#hashi/topic"""
    def getChild(self, name, request):
        if name == '':
            return self
        else:
            return Resource.getChild(self, name, request)

    def render_GET(self, request):
        return """
Should document the API here!
"""

class APISession(Resource):
    isLeaf = True

    def render_GET(self, request):
        """Return session uid."""
        return json.dumps(request.getSession().uid)

    def render_POST(self, request):
        """Log out."""
        request.getSession().expire()
        return json.dumps(True)

class IRCNetwork(Resource):
    def render_GET(self, request):
        return """
List of networks!
"""

class IRCChannel(Resource):
    isLeaf = True

    def render_GET(self, request):
        return json.dumps(request.irc_client.channels)

    def render_POST(self, request):
        chan_cmd = parse_qs(request.content.getvalue())
        for cmd, arg in chan_cmd.items():
            if cmd == "join":
                request.irc_client.join(arg[0])
            if cmd == "leave":
                request.irc_client.leave(arg[0])
        return ''

class History(Resource):
    isLeaf = True

    def render_GET(self, request):
        print(request.irc_client.history)
        print(request.args["source"])
        return str(request.irc_client.history[request.args["source"][0]])

def irc_rewriter(avatarId, client):
    def func(request):
        request.irc_nick = avatarId
        request.irc_client = client
    return func

class HashiUserRealm(object):
    implements(IRealm)

    def __init__(self, irc_clients, rest_api):
        self.irc_clients = irc_clients
        self.rest_api = rest_api

    def requestAvatar(self, avatarId, mind, *interfaces):
        if IResource in interfaces:
            client = self.irc_clients[avatarId]
            rewrite = RewriterResource(self.rest_api,
                                       irc_rewriter(avatarId, client))
            return (IResource,
                    rewrite,
                    lambda: None)
        raise NotImplementedError()

def start(irc_clients):
    root = Hashioki()
    rest_api = API()
    root.putChild('api', rest_api)
    rest_api.putChild('session', APISession())
    irc_network = IRCNetwork()
    rest_api.putChild('networks', irc_network)

    portal = Portal(HashiUserRealm(irc_clients, root),
                    [FilePasswordDB('httpd.password')])
    credentialFactory = DigestCredentialFactory("md5", "localhost:8080")
    resource = HTTPAuthSessionWrapper(portal, [credentialFactory])

    site = server.Site(resource)

    reactor.listenTCP(8080, site)

    return site
<|endoftext|>


<commit_before>import simuvex

######################################
# Returns an unconstrained byte
######################################

class ReturnUnconstrained(simuvex.SimProcedure):
    def run(self, resolves=None): #pylint:disable=arguments-differ
        #pylint:disable=attribute-defined-outside-init
        self.resolves = resolves

        o = self.state.se.Unconstrained("unconstrained_ret_%s" % self.resolves, self.state.arch.bits)
        #if 'unconstrained_ret_9_64' in o.variables:
        #   __import__('ipdb').set_trace()
        return o

    def __repr__(self):
        return '<ReturnUnconstrained %s>' % self.resolves
<commit_msg>ReturnUnconstrained: write "resolves" to successors.artifacts. Make repr better.<commit_after>import simuvex

######################################
# Returns an unconstrained byte
######################################

class ReturnUnconstrained(simuvex.SimProcedure):
    def run(self, resolves=None): #pylint:disable=arguments-differ
        #pylint:disable=attribute-defined-outside-init
        self.resolves = resolves

        self.successors.artifacts['resolves'] = resolves

        o = self.state.se.Unconstrained("unconstrained_ret_%s" % self.resolves, self.state.arch.bits)
        #if 'unconstrained_ret_9_64' in o.variables:
        #   __import__('ipdb').set_trace()
        return o

    def __repr__(self):
        if 'resolves' in self.kwargs:
            return '<ReturnUnconstrained %s>' % self.kwargs['resolves']
        else:
            return '<ReturnUnconstrained>'
<|endoftext|>


<commit_before>from __future__ import print_function
from __future__ import division

import numpy as np
from numpy.testing import *
from numpy.fft import ifftshift, ifftn
import itertools
from skimage.transform import *

def rescale(x):
    x = x.astype(float)
    x -= x.min()
    x /= x.max()
    return x

def check_radon_center(shape, circle):
    # Create a test image with only a single non-zero pixel at the origin
    image = np.zeros(shape, dtype=np.float)
    image[(shape[0] // 2, shape[1] // 2)] = 1.
    # Calculate the sinogram
    theta = np.linspace(0., 180., max(shape), endpoint=False)
    sinogram = radon(image, theta=theta, circle=circle)
    # The sinogram should be a straight, horizontal line
    sinogram_max = np.argmax(sinogram, axis=0)
    print(sinogram_max)
    assert np.std(sinogram_max) < 1e-6

def test_radon_center():
    shapes = [(16, 16), (17, 17)]
    circles = [False, True]
    for shape, circle in itertools.product(shapes, circles):
        yield check_radon_center, shape, circle
    rectangular_shapes = [(32, 16), (33, 17)]
    for shape in rectangular_shapes:
        yield check_radon_center, shape, False

def check_iradon_center(size, theta, circle):
    debug = False
    # Create a test sinogram corresponding to a single projection
    # with a single non-zero pixel at the rotation center
    if circle:
        sinogram = np.zeros((size, 1), dtype=np.float)
        sinogram[size // 2, 0] = 1.
    else:
        diagonal = int(np.ceil(np.sqrt(2) * size))
        sinogram = np.zeros((diagonal, 1), dtype=np.float)
        sinogram[sinogram.shape[0] // 2, 0] = 1.
    maxpoint = np.unravel_index(np.argmax(sinogram), sinogram.shape)
    print('shape of generated sinogram', sinogram.shape)
    print('maximum in generated sinogram', maxpoint)
    # Compare reconstructions for theta=angle and theta=angle + 180;
    # these should be exactly equal
    reconstruction = iradon(sinogram, theta=[theta], circle=circle)
    reconstruction_opposite = iradon(sinogram, theta=[theta + 180],
                                     circle=circle)
    print('rms deviance:',
          np.sqrt(np.mean((reconstruction_opposite - reconstruction)**2)))
    if debug:
        import matplotlib.pyplot as plt
        imkwargs = dict(cmap='gray', interpolation='nearest')
        plt.figure()
        plt.subplot(221)
        plt.imshow(sinogram, **imkwargs)
        plt.subplot(222)
        plt.imshow(reconstruction_opposite - reconstruction, **imkwargs)
        plt.subplot(223)
        plt.imshow(reconstruction, **imkwargs)
        plt.subplot(224)
        plt.imshow(reconstruction_opposite, **imkwargs)
        plt.show()

    assert np.allclose(reconstruction, reconstruction_opposite)

def test_iradon_center():
    sizes = [16, 17]
    thetas = [0, 90]
    circles = [False, True]
    for size, theta, circle in itertools.product(sizes, thetas, circles):
        yield check_iradon_center, size, theta, circle

def test_radon_iradon():
    size = 100
    debug = False
    image = np.tri(size) + np.tri(size)[::-1]
    for filter_type in ["ramp", "shepp-logan", "cosine", "hamming", "hann"]:
        reconstructed = iradon(radon(image), filter=filter_type)

        image = rescale(image)
        reconstructed = rescale(reconstructed)
        delta = np.mean(np.abs(image - reconstructed))

        if debug:
            print(delta)
            import matplotlib.pyplot as plt
            f, (ax1, ax2) = plt.subplots(1, 2)
            ax1.imshow(image, cmap=plt.cm.gray)
            ax2.imshow(reconstructed, cmap=plt.cm.gray)
            plt.show()

        assert delta < 0.05

    reconstructed = iradon(radon(image), filter="ramp",
                           interpolation="nearest")
    delta = np.mean(abs(image - reconstructed))
    assert delta < 0.05
    size = 20
    image = np.tri(size) + np.tri(size)[::-1]
    reconstructed = iradon(radon(image), filter="ramp",
                           interpolation="nearest")

def test_iradon_angles():
    """
    Test with different number of projections
    """
    size = 100
    # Synthetic data
    image = np.tri(size) + np.tri(size)[::-1]
    # Large number of projections: a good quality is expected
    nb_angles = 200
    radon_image_200 = radon(image, theta=np.linspace(0, 180, nb_angles,
                                                     endpoint=False))
    reconstructed = iradon(radon_image_200)
    delta_200 = np.mean(abs(rescale(image) - rescale(reconstructed)))
    assert delta_200 < 0.03
    # Lower number of projections
    nb_angles = 80
    radon_image_80 = radon(image, theta=np.linspace(0, 180, nb_angles,
                                                    endpoint=False))
    # Test whether the sum of all projections is approximately the same
    s = radon_image_80.sum(axis=0)
    assert np.allclose(s, s[0], rtol=0.01)
    reconstructed = iradon(radon_image_80)
    delta_80 = np.mean(abs(image / np.max(image) -
                           reconstructed / np.max(reconstructed)))
    # Loss of quality when the number of projections is reduced
    assert delta_80 > delta_200

def test_radon_minimal():
    """
    Test for small images for various angles
    """
    thetas = [np.arange(180)]
    for theta in thetas:
        a = np.zeros((3, 3))
        a[1, 1] = 1
        p = radon(a, theta)
        reconstructed = iradon(p, theta)
        reconstructed /= np.max(reconstructed)
        assert np.all(abs(a - reconstructed) < 0.4)

        b = np.zeros((4, 4))
        b[1:3, 1:3] = 1
        p = radon(b, theta)
        reconstructed = iradon(p, theta)
        reconstructed /= np.max(reconstructed)
        assert np.all(abs(b - reconstructed) < 0.4)

        c = np.zeros((5, 5))
        c[1:3, 1:3] = 1
        p = radon(c, theta)
        reconstructed = iradon(p, theta)
        reconstructed /= np.max(reconstructed)
        assert np.all(abs(c - reconstructed) < 0.4)

def test_reconstruct_with_wrong_angles():
    a = np.zeros((3, 3))
    p = radon(a, theta=[0, 1, 2])
    iradon(p, theta=[0, 1, 2])
    assert_raises(ValueError, iradon, p, theta=[0, 1, 2, 3])

def _random_circle(shape):
    # Synthetic random data, zero outside reconstruction circle
    np.random.seed(98312871)
    image = np.random.rand(*shape)
    c0, c1 = np.ogrid[0:shape[0], 0:shape[1]]
    r = np.sqrt((c0 - shape[0] // 2)**2 + (c1 - shape[1] // 2)**2)
    radius = min(shape) // 2
    image[r >= radius] = 0.
    return image

def test_radon_circle():
    a = np.ones((10, 10))
    assert_raises(ValueError, radon, a, circle=True)

    # Synthetic data, circular symmetry
    shape = (61, 79)
    c0, c1 = np.ogrid[0:shape[0], 0:shape[1]]
    r = np.sqrt((c0 - shape[0] // 2)**2 + (c1 - shape[1] // 2)**2)
    radius = min(shape) // 2
    image = np.clip(radius - r, 0, np.inf)
    image = rescale(image)
    angles = np.linspace(0, 180, min(shape), endpoint=False)
    sinogram = radon(image, theta=angles, circle=True)
    assert np.all(sinogram.std(axis=1) < 1e-2)

    # Synthetic data, random
    image = _random_circle(shape)
    sinogram = radon(image, theta=angles, circle=True)
    mass = sinogram.sum(axis=0)
    average_mass = mass.mean()
    relative_error = np.abs(mass - average_mass) / average_mass
    print(relative_error.max(), relative_error.mean())
    assert np.all(relative_error < 3e-3)

def check_radon_iradon_circle(interpolation, shape, output_size):
    # Forward and inverse radon on synthetic data
    image = _random_circle(shape)
    radius = min(shape) // 2
    sinogram_rectangle = radon(image, circle=False)
    reconstruction_rectangle = iradon(sinogram_rectangle,
                                        output_size=output_size,
                                        interpolation=interpolation,
                                        circle=False)
    sinogram_circle = radon(image, circle=True)
    reconstruction_circle = iradon(sinogram_circle,
                                    output_size=output_size,
                                    interpolation=interpolation,
                                    circle=True)
    # Crop rectangular reconstruction to match circle=True reconstruction
    width = reconstruction_circle.shape[0]
    excess = int(np.ceil((reconstruction_rectangle.shape[0] - width) / 2))
    s = np.s_[excess:width + excess, excess:width + excess]
    reconstruction_rectangle = reconstruction_rectangle[s]
    # Find the reconstruction circle, set reconstruction to zero outside
    c0, c1 = np.ogrid[0:width, 0:width]
    r = np.sqrt((c0 - width // 2)**2 + (c1 - width // 2)**2)
    reconstruction_rectangle[r >= radius] = 0.
    print(reconstruction_circle.shape)
    print(reconstruction_rectangle.shape)
    np.allclose(reconstruction_rectangle, reconstruction_circle)

def test_radon_iradon_circle():
    shape = (61, 79)
    interpolations = ('nearest', 'linear')
    output_sizes = (None, min(shape), max(shape), 97)
    for interpolation, output_size in itertools.product(interpolations,
                                                        output_sizes):
        yield check_radon_iradon_circle, interpolation, shape, output_size

if __name__ == "__main__":
    from numpy.testing import run_module_suite
    run_module_suite()
<commit_msg>test_radon_transform: Add helper functions.<commit_after>from __future__ import print_function
from __future__ import division

import numpy as np
from numpy.testing import *
from numpy.fft import ifftshift, ifftn
import itertools
from skimage.transform import *
from skimage.io import imread
from skimage import data_dir

__PHANTOM = imread(data_dir + "/phantom.png", as_grey=True)[::2, ::2]

def _get_phantom():
    return __PHANTOM

def _debug_plot(original, result, sinogram=None):
    from matplotlib import pyplot as plt
    imkwargs = dict(cmap='gray', interpolation='nearest')
    if sinogram is None:
        plt.figure(figsize=(15, 6))
        sp = 130
    else:
        plt.figure(figsize=(11, 11))
        sp = 221
        plt.subplot(sp + 0)
        plt.imshow(sinogram, aspect='auto', **imkwargs)
    plt.subplot(sp + 1)
    plt.imshow(original, **imkwargs)
    plt.subplot(sp + 2)
    plt.imshow(result, vmin=original.min(), vmax=original.max(), **imkwargs)
    plt.subplot(sp + 3)
    plt.imshow(result - original, **imkwargs)
    plt.colorbar()
    plt.show()

def rescale(x):
    x = x.astype(float)
    x -= x.min()
    x /= x.max()
    return x

def check_radon_center(shape, circle):
    # Create a test image with only a single non-zero pixel at the origin
    image = np.zeros(shape, dtype=np.float)
    image[(shape[0] // 2, shape[1] // 2)] = 1.
    # Calculate the sinogram
    theta = np.linspace(0., 180., max(shape), endpoint=False)
    sinogram = radon(image, theta=theta, circle=circle)
    # The sinogram should be a straight, horizontal line
    sinogram_max = np.argmax(sinogram, axis=0)
    print(sinogram_max)
    assert np.std(sinogram_max) < 1e-6

def test_radon_center():
    shapes = [(16, 16), (17, 17)]
    circles = [False, True]
    for shape, circle in itertools.product(shapes, circles):
        yield check_radon_center, shape, circle
    rectangular_shapes = [(32, 16), (33, 17)]
    for shape in rectangular_shapes:
        yield check_radon_center, shape, False

def check_iradon_center(size, theta, circle):
    debug = False
    # Create a test sinogram corresponding to a single projection
    # with a single non-zero pixel at the rotation center
    if circle:
        sinogram = np.zeros((size, 1), dtype=np.float)
        sinogram[size // 2, 0] = 1.
    else:
        diagonal = int(np.ceil(np.sqrt(2) * size))
        sinogram = np.zeros((diagonal, 1), dtype=np.float)
        sinogram[sinogram.shape[0] // 2, 0] = 1.
    maxpoint = np.unravel_index(np.argmax(sinogram), sinogram.shape)
    print('shape of generated sinogram', sinogram.shape)
    print('maximum in generated sinogram', maxpoint)
    # Compare reconstructions for theta=angle and theta=angle + 180;
    # these should be exactly equal
    reconstruction = iradon(sinogram, theta=[theta], circle=circle)
    reconstruction_opposite = iradon(sinogram, theta=[theta + 180],
                                     circle=circle)
    print('rms deviance:',
          np.sqrt(np.mean((reconstruction_opposite - reconstruction)**2)))
    if debug:
        import matplotlib.pyplot as plt
        imkwargs = dict(cmap='gray', interpolation='nearest')
        plt.figure()
        plt.subplot(221)
        plt.imshow(sinogram, **imkwargs)
        plt.subplot(222)
        plt.imshow(reconstruction_opposite - reconstruction, **imkwargs)
        plt.subplot(223)
        plt.imshow(reconstruction, **imkwargs)
        plt.subplot(224)
        plt.imshow(reconstruction_opposite, **imkwargs)
        plt.show()

    assert np.allclose(reconstruction, reconstruction_opposite)

def test_iradon_center():
    sizes = [16, 17]
    thetas = [0, 90]
    circles = [False, True]
    for size, theta, circle in itertools.product(sizes, thetas, circles):
        yield check_iradon_center, size, theta, circle

def test_radon_iradon():
    size = 100
    debug = False
    image = np.tri(size) + np.tri(size)[::-1]
    for filter_type in ["ramp", "shepp-logan", "cosine", "hamming", "hann"]:
        reconstructed = iradon(radon(image), filter=filter_type)

        image = rescale(image)
        reconstructed = rescale(reconstructed)
        delta = np.mean(np.abs(image - reconstructed))

        if debug:
            print(delta)
            import matplotlib.pyplot as plt
            f, (ax1, ax2) = plt.subplots(1, 2)
            ax1.imshow(image, cmap=plt.cm.gray)
            ax2.imshow(reconstructed, cmap=plt.cm.gray)
            plt.show()

        assert delta < 0.05

    reconstructed = iradon(radon(image), filter="ramp",
                           interpolation="nearest")
    delta = np.mean(abs(image - reconstructed))
    assert delta < 0.05
    size = 20
    image = np.tri(size) + np.tri(size)[::-1]
    reconstructed = iradon(radon(image), filter="ramp",
                           interpolation="nearest")

def test_iradon_angles():
    """
    Test with different number of projections
    """
    size = 100
    # Synthetic data
    image = np.tri(size) + np.tri(size)[::-1]
    # Large number of projections: a good quality is expected
    nb_angles = 200
    radon_image_200 = radon(image, theta=np.linspace(0, 180, nb_angles,
                                                     endpoint=False))
    reconstructed = iradon(radon_image_200)
    delta_200 = np.mean(abs(rescale(image) - rescale(reconstructed)))
    assert delta_200 < 0.03
    # Lower number of projections
    nb_angles = 80
    radon_image_80 = radon(image, theta=np.linspace(0, 180, nb_angles,
                                                    endpoint=False))
    # Test whether the sum of all projections is approximately the same
    s = radon_image_80.sum(axis=0)
    assert np.allclose(s, s[0], rtol=0.01)
    reconstructed = iradon(radon_image_80)
    delta_80 = np.mean(abs(image / np.max(image) -
                           reconstructed / np.max(reconstructed)))
    # Loss of quality when the number of projections is reduced
    assert delta_80 > delta_200

def test_radon_minimal():
    """
    Test for small images for various angles
    """
    thetas = [np.arange(180)]
    for theta in thetas:
        a = np.zeros((3, 3))
        a[1, 1] = 1
        p = radon(a, theta)
        reconstructed = iradon(p, theta)
        reconstructed /= np.max(reconstructed)
        assert np.all(abs(a - reconstructed) < 0.4)

        b = np.zeros((4, 4))
        b[1:3, 1:3] = 1
        p = radon(b, theta)
        reconstructed = iradon(p, theta)
        reconstructed /= np.max(reconstructed)
        assert np.all(abs(b - reconstructed) < 0.4)

        c = np.zeros((5, 5))
        c[1:3, 1:3] = 1
        p = radon(c, theta)
        reconstructed = iradon(p, theta)
        reconstructed /= np.max(reconstructed)
        assert np.all(abs(c - reconstructed) < 0.4)

def test_reconstruct_with_wrong_angles():
    a = np.zeros((3, 3))
    p = radon(a, theta=[0, 1, 2])
    iradon(p, theta=[0, 1, 2])
    assert_raises(ValueError, iradon, p, theta=[0, 1, 2, 3])

def _random_circle(shape):
    # Synthetic random data, zero outside reconstruction circle
    np.random.seed(98312871)
    image = np.random.rand(*shape)
    c0, c1 = np.ogrid[0:shape[0], 0:shape[1]]
    r = np.sqrt((c0 - shape[0] // 2)**2 + (c1 - shape[1] // 2)**2)
    radius = min(shape) // 2
    image[r >= radius] = 0.
    return image

def test_radon_circle():
    a = np.ones((10, 10))
    assert_raises(ValueError, radon, a, circle=True)

    # Synthetic data, circular symmetry
    shape = (61, 79)
    c0, c1 = np.ogrid[0:shape[0], 0:shape[1]]
    r = np.sqrt((c0 - shape[0] // 2)**2 + (c1 - shape[1] // 2)**2)
    radius = min(shape) // 2
    image = np.clip(radius - r, 0, np.inf)
    image = rescale(image)
    angles = np.linspace(0, 180, min(shape), endpoint=False)
    sinogram = radon(image, theta=angles, circle=True)
    assert np.all(sinogram.std(axis=1) < 1e-2)

    # Synthetic data, random
    image = _random_circle(shape)
    sinogram = radon(image, theta=angles, circle=True)
    mass = sinogram.sum(axis=0)
    average_mass = mass.mean()
    relative_error = np.abs(mass - average_mass) / average_mass
    print(relative_error.max(), relative_error.mean())
    assert np.all(relative_error < 3e-3)

def check_radon_iradon_circle(interpolation, shape, output_size):
    # Forward and inverse radon on synthetic data
    image = _random_circle(shape)
    radius = min(shape) // 2
    sinogram_rectangle = radon(image, circle=False)
    reconstruction_rectangle = iradon(sinogram_rectangle,
                                        output_size=output_size,
                                        interpolation=interpolation,
                                        circle=False)
    sinogram_circle = radon(image, circle=True)
    reconstruction_circle = iradon(sinogram_circle,
                                    output_size=output_size,
                                    interpolation=interpolation,
                                    circle=True)
    # Crop rectangular reconstruction to match circle=True reconstruction
    width = reconstruction_circle.shape[0]
    excess = int(np.ceil((reconstruction_rectangle.shape[0] - width) / 2))
    s = np.s_[excess:width + excess, excess:width + excess]
    reconstruction_rectangle = reconstruction_rectangle[s]
    # Find the reconstruction circle, set reconstruction to zero outside
    c0, c1 = np.ogrid[0:width, 0:width]
    r = np.sqrt((c0 - width // 2)**2 + (c1 - width // 2)**2)
    reconstruction_rectangle[r >= radius] = 0.
    print(reconstruction_circle.shape)
    print(reconstruction_rectangle.shape)
    np.allclose(reconstruction_rectangle, reconstruction_circle)

def test_radon_iradon_circle():
    shape = (61, 79)
    interpolations = ('nearest', 'linear')
    output_sizes = (None, min(shape), max(shape), 97)
    for interpolation, output_size in itertools.product(interpolations,
                                                        output_sizes):
        yield check_radon_iradon_circle, interpolation, shape, output_size

if __name__ == "__main__":
    from numpy.testing import run_module_suite
    run_module_suite()
<|endoftext|>


<commit_before># Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Read CIFAR-10 data from pickled numpy arrays and write TFExamples.

Generates TFRecord files from the python version of the CIFAR-10 dataset
downloaded from https://www.cs.toronto.edu/~kriz/cifar.html.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import cPickle
import os
import tensorflow as tf

FLAGS = tf.flags.FLAGS

tf.flags.DEFINE_string('input_dir', '',
                       'Directory where CIFAR10 data is located.')

tf.flags.DEFINE_string('output_dir', '',
                       'Directory where TFRecords will be saved.'
                       'The TFRecords will have the same name as'
                       ' the CIFAR10 inputs + .tfrecords.')

def _int64_feature(value):
  return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))

def _bytes_feature(value):
  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[str(value)]))

def _get_file_names():
  """Returns the file names expected to exist in the input_dir."""
  file_names = ['data_batch_%d' % i for i in xrange(1, 6)]
  file_names.append('test_batch')
  return file_names

def read_pickle_from_file(filename):
  with open(filename, 'r') as f:
    data_dict = cPickle.load(f)
  return data_dict

def main(argv):
  del argv  # Unused.

  file_names = _get_file_names()
  for file_name in file_names:
    input_file = os.path.join(FLAGS.input_dir, file_name)
    output_file = os.path.join(FLAGS.output_dir, file_name + '.tfrecords')

    print('Generating %s' % output_file)

    record_writer = tf.python_io.TFRecordWriter(output_file)

    data_dict = read_pickle_from_file(input_file)
    data = data_dict['data']
    labels = data_dict['labels']

    num_entries_in_batch = len(labels)
    for i in range(num_entries_in_batch):
      example = tf.train.Example(
          features=tf.train.Features(feature={
              'image': _bytes_feature(data[i].tobytes()),
              'label': _int64_feature(labels[i])
          }))
      record_writer.write(example.SerializeToString())
    record_writer.close()

  print('Done!')

if __name__ == '__main__':
  tf.app.run(main)
<commit_msg>Creating convert_to_tfrecord function<commit_after># Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Read CIFAR-10 data from pickled numpy arrays and write TFExamples.

Generates TFRecord files from the python version of the CIFAR-10 dataset
downloaded from https://www.cs.toronto.edu/~kriz/cifar.html.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import cPickle
import os
import tensorflow as tf

FLAGS = tf.flags.FLAGS

tf.flags.DEFINE_string('input_dir', '',
                       'Directory where CIFAR10 data is located.')

tf.flags.DEFINE_string('output_dir', '',
                       'Directory where TFRecords will be saved.'
                       'The TFRecords will have the same name as'
                       ' the CIFAR10 inputs + .tfrecords.')

def _int64_feature(value):
  return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))

def _bytes_feature(value):
  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[str(value)]))

def _get_file_names():
  """Returns the file names expected to exist in the input_dir."""
  file_names = ['data_batch_%d' % i for i in xrange(1, 6)]
  file_names.append('test_batch')
  return file_names

def read_pickle_from_file(filename):
  with open(filename, 'r') as f:
    data_dict = cPickle.load(f)
  return data_dict

def convert_to_tfrecord(input_file, name):
  """Converts a file to tfrecords."""
  print('Generating %s' % output_file)
  record_writer = tf.python_io.TFRecordWriter(output_file)

  data_dict = read_pickle_from_file(input_file)
  data = data_dict['data']
  labels = data_dict['labels']

  num_entries_in_batch = len(labels)
  for i in range(num_entries_in_batch):
    example = tf.train.Example(
        features=tf.train.Features(feature={
            'image': _bytes_feature(data[i].tobytes()),
            'label': _int64_feature(labels[i])
        }))
    record_writer.write(example.SerializeToString())
  record_writer.close()

def main(argv):
  del argv  # Unused.

  file_names = _get_file_names()
  for file_name in file_names:
    input_file = os.path.join(FLAGS.input_dir, file_name)
    output_file = os.path.join(FLAGS.output_dir, file_name + '.tfrecords')
    # Convert to Examples and write the result to TFRecords.
    convert_to_tfrecord(input_file, output_file)

  print('Done!')

if __name__ == '__main__':
  tf.app.run(main)
<|endoftext|>