<commit_before>import numpy as np
from collections import Iterable
import logging

from .base import BatchAttack
from .base import generator_decorator
from ..utils import softmax

class VirtualAdversarialAttack(BatchAttack):
    """Calculate an untargeted adversarial perturbation by performing a
    approximated second order optimization step on the KL divergence between
    the unperturbed predictions and the predictions for the adversarial
    perturbation. This attack was introduced in [1]_.

    References
    ----------
    .. [1] Takeru Miyato, Shin-ichi Maeda, Masanori Koyama, Ken Nakae,
           Shin Ishii,
           "Distributional Smoothing with Virtual Adversarial Training",
           https://arxiv.org/abs/1507.00677
    """

    def _clip_perturbation(self, a, perturbation, epsilon):
        # using mean to make range of epsilons comparable to Linf
        norm = np.sqrt(np.mean(np.square(perturbation)))
        norm = max(1e-12, norm)  # avoid divsion by zero
        min_, max_ = a.bounds()
        s = max_ - min_
        # clipping, i.e. only decreasing norm
        factor = min(1, epsilon * s / norm)
        return perturbation * factor

    @generator_decorator
    def as_generator(self, a, xi=1e-5, iterations=1, epsilons=1000, max_epsilon=0.3):
        """

        Parameters
        ----------
        inputs : `numpy.ndarray`
            Batch of inputs with shape as expected by the underlying model.
        labels : `numpy.ndarray`
            Class labels of the inputs as a vector of integers in [0, number of classes).
        unpack : bool
            If true, returns the adversarial inputs as an array, otherwise returns Adversarial objects.
        xi : float
            The finite difference size for performing the power method.
        iterations : int
            Number of iterations to perform power method to search for second
            order perturbation of KL divergence.
        epsilons : int or Iterable[float]
            Either Iterable of step sizes in the direction of the sign of
            the gradient or number of step sizes between 0 and max_epsilon
            that should be tried.
        max_epsilon : float
            Largest step size if epsilons is not an iterable.
        """

        assert a.target_class is None, (
            "Virtual Adversarial is an untargeted adversarial attack."
        )

        yield from self._run(
            a, xi=xi, iterations=iterations, epsilons=epsilons, max_epsilon=max_epsilon
        )

    def _run(self, a, xi, iterations, epsilons, max_epsilon):
        if not a.has_gradient():
            return

        x = a.unperturbed
        min_, max_ = a.bounds()

        logits, _ = yield from a.forward_one(x)

        if not isinstance(epsilons, Iterable):
            epsilons = np.linspace(0, max_epsilon, num=epsilons + 1)[1:]
            decrease_if_first = True
        else:
            decrease_if_first = False

        for _ in range(2):  # to repeat with decreased epsilons if necessary
            for i, epsilon in enumerate(epsilons):
                # start with random vector as search vector
                d = np.random.normal(0.0, 1.0, size=x.shape).astype(x.dtype)
                for it in range(iterations):
                    # normalize proposal to be unit vector
                    d = xi * d / np.sqrt((d ** 2).sum())

                    logits_d, _ = yield from a.forward_one(x + d, strict=False)

                    dl_dp = softmax(logits) - softmax(logits_d)

                    # d = dl_dd = dl_dp * dp_dd
                    # use gradient of KL divergence as new search vector
                    d = yield from a.backward_one(gradient=dl_dp, x=x + d,
                                                  strict=False)
                    d = (max_ - min_) * d

                    if np.allclose(np.sqrt((d ** 2).sum()), 0, atol=1e-16):
                        raise RuntimeError(
                            "Gradient vanished; this can happen if xi is too small."
                        )

                delta = d / np.sqrt((d ** 2).mean())

                perturbed = x + self._clip_perturbation(a, delta, epsilon)
                perturbed = np.clip(perturbed, min_, max_)

                _, is_adversarial = yield from a.forward_one(perturbed)
                if is_adversarial:
                    if decrease_if_first and i < 20:
                        logging.info("repeating attack with smaller epsilons")
                        break
                    return

            max_epsilon = epsilons[i]
            epsilons = np.linspace(0, max_epsilon, num=20 + 1)[1:]

VirtualAdversarialAttack.__call__.__doc__ = (
    VirtualAdversarialAttack.as_generator.__doc__
)
<commit_msg>Fix formatting<commit_after>import numpy as np
from collections import Iterable
import logging

from .base import BatchAttack
from .base import generator_decorator
from ..utils import softmax

class VirtualAdversarialAttack(BatchAttack):
    """Calculate an untargeted adversarial perturbation by performing a
    approximated second order optimization step on the KL divergence between
    the unperturbed predictions and the predictions for the adversarial
    perturbation. This attack was introduced in [1]_.

    References
    ----------
    .. [1] Takeru Miyato, Shin-ichi Maeda, Masanori Koyama, Ken Nakae,
           Shin Ishii,
           "Distributional Smoothing with Virtual Adversarial Training",
           https://arxiv.org/abs/1507.00677
    """

    def _clip_perturbation(self, a, perturbation, epsilon):
        # using mean to make range of epsilons comparable to Linf
        norm = np.sqrt(np.mean(np.square(perturbation)))
        norm = max(1e-12, norm)  # avoid divsion by zero
        min_, max_ = a.bounds()
        s = max_ - min_
        # clipping, i.e. only decreasing norm
        factor = min(1, epsilon * s / norm)
        return perturbation * factor

    @generator_decorator
    def as_generator(self, a, xi=1e-5, iterations=1, epsilons=1000, max_epsilon=0.3):
        """

        Parameters
        ----------
        inputs : `numpy.ndarray`
            Batch of inputs with shape as expected by the underlying model.
        labels : `numpy.ndarray`
            Class labels of the inputs as a vector of integers in [0, number of classes).
        unpack : bool
            If true, returns the adversarial inputs as an array, otherwise returns Adversarial objects.
        xi : float
            The finite difference size for performing the power method.
        iterations : int
            Number of iterations to perform power method to search for second
            order perturbation of KL divergence.
        epsilons : int or Iterable[float]
            Either Iterable of step sizes in the direction of the sign of
            the gradient or number of step sizes between 0 and max_epsilon
            that should be tried.
        max_epsilon : float
            Largest step size if epsilons is not an iterable.
        """

        assert (
            a.target_class is None
        ), "Virtual Adversarial is an untargeted adversarial attack."

        yield from self._run(
            a, xi=xi, iterations=iterations, epsilons=epsilons, max_epsilon=max_epsilon
        )

    def _run(self, a, xi, iterations, epsilons, max_epsilon):
        if not a.has_gradient():
            return

        x = a.unperturbed
        min_, max_ = a.bounds()

        logits, _ = yield from a.forward_one(x)

        if not isinstance(epsilons, Iterable):
            epsilons = np.linspace(0, max_epsilon, num=epsilons + 1)[1:]
            decrease_if_first = True
        else:
            decrease_if_first = False

        for _ in range(2):  # to repeat with decreased epsilons if necessary
            for i, epsilon in enumerate(epsilons):
                # start with random vector as search vector
                d = np.random.normal(0.0, 1.0, size=x.shape).astype(x.dtype)
                for it in range(iterations):
                    # normalize proposal to be unit vector
                    d = xi * d / np.sqrt((d ** 2).sum())

                    logits_d, _ = yield from a.forward_one(x + d, strict=False)

                    dl_dp = softmax(logits) - softmax(logits_d)

                    # d = dl_dd = dl_dp * dp_dd
                    # use gradient of KL divergence as new search vector
                    d = yield from a.backward_one(gradient=dl_dp, x=x + d, strict=False)
                    d = (max_ - min_) * d

                    if np.allclose(np.sqrt((d ** 2).sum()), 0, atol=1e-16):
                        raise RuntimeError(
                            "Gradient vanished; this can happen if xi is too small."
                        )

                delta = d / np.sqrt((d ** 2).mean())

                perturbed = x + self._clip_perturbation(a, delta, epsilon)
                perturbed = np.clip(perturbed, min_, max_)

                _, is_adversarial = yield from a.forward_one(perturbed)
                if is_adversarial:
                    if decrease_if_first and i < 20:
                        logging.info("repeating attack with smaller epsilons")
                        break
                    return

            max_epsilon = epsilons[i]
            epsilons = np.linspace(0, max_epsilon, num=20 + 1)[1:]

VirtualAdversarialAttack.__call__.__doc__ = (
    VirtualAdversarialAttack.as_generator.__doc__
)
<|endoftext|>


<commit_before>import UserDict

from pygr import annotation, sequence, sqlgraph, worldbase
from pygr.dbfile import ReadOnlyError

class UCSCStrandDescr(object):

    def __get__(self, obj, objtype):
        if obj.strand == '+':
            return 1
        else:
            return -1

class UCSCSeqIntervalRow(sqlgraph.TupleO):
    orientation = UCSCStrandDescr()

class UCSCEnsemblInterface(object):

    def __init__(self, ucsc_genome_name, ens_species=None, ucsc_serverInfo=None,
                ens_serverInfo=None):
        '''Set up everything needed to produce UCSC/Ensembl
        annotation databases. ucsc_genome_name should follow the worldbase
        naming convention. If ens_species is not specified, we will try
        to autodetect it.'''
        # Connect to both servers and prepare database names.
        if ucsc_serverInfo is not None:
            self.ucsc_server = ucsc_serverInfo
        else:
            self.ucsc_server = sqlgraph.DBServerInfo(
                host='genome-mysql.cse.ucsc.edu', user='genome')
        if ens_serverInfo is not None:
            self.ens_server = ens_serverInfo
        else:
            self.ens_server = sqlgraph.DBServerInfo(
                host='ensembldb.ensembl.org', port=5306, user='anonymous')
        self.ucsc_db = ucsc_genome_name.split('.')[-1]
        self.ens_db = self.get_ensembl_db_name(ens_species)
        # Connect to all the necessary tables.
        self.ucsc_ensGene_trans = sqlgraph.SQLTable('%s.ensGene' %
                                                    self.ucsc_db,
                                                   serverInfo=self.ucsc_server,
                                                    primaryKey='name',
                                                  itemClass=UCSCSeqIntervalRow)
        self.ucsc_ensGene_gene = sqlgraph.SQLTable('%s.ensGene' % self.ucsc_db,
                                                   serverInfo=self.ucsc_server,
                                                   primaryKey='name2',
                                                  itemClass=UCSCSeqIntervalRow)
        self.ucsc_ensGtp = sqlgraph.SQLTable('%s.ensGtp' % self.ucsc_db,
                                             serverInfo=self.ucsc_server,
                                             primaryKey='protein')
        self.ens_exon_stable_id = sqlgraph.SQLTable('%s.exon_stable_id'
                                                    % self.ens_db,
                                                    serverInfo=self.ens_server,
                                                    primaryKey='stable_id')
        self.ens_transcript_stable_id = sqlgraph.SQLTable(
            '%s.transcript_stable_id' % self.ens_db,
            serverInfo=self.ens_server, primaryKey='stable_id')
        # We will need this too.
        self.genome_seq = worldbase(ucsc_genome_name)
        # Initialise all cache variables.
        self.trans_db = None
        self.gene_db = None
        self.prot_db = None
        self.exon_db = None

    def get_ensembl_db_name(self, ens_prefix):
        '''Used by __init__(), obtains Ensembl database name matching
        the specified UCSC genome version'''
        ucsc_versions = sqlgraph.SQLTable('hgFixed.trackVersion',
                                          serverInfo=self.ucsc_server,
                                          primaryKey='db')
        ens_version = ucsc_versions[self.ucsc_db].version
        if ens_prefix is None:
            # Note: this assumes 'source' in hgFixed.trackVersion contains
            # the URI of the Ensembl data set and that the last path component
            # of that URI is the species name of that data set.
            ens_prefix = ucsc_versions[self.ucsc_db].source.split('/')[-2]
        cursor = self.ens_server.cursor()
        cursor.execute("show databases like '%s_core_%s_%%'" % (ens_prefix,
                                                                ens_version))
        return cursor.fetchall()[0][0]

    def transcript_database(self):
        'Return an AnnotationDB of transcript annotations.'
        if self.trans_db is None:
            self.trans_db = annotation.AnnotationDB(self.ucsc_ensGene_trans,
                                                    self.genome_seq,
                                                    checkFirstID=False,
                                                    sliceAttrDict=dict(
                                                        id='chrom',
                                                        start='txStart',
                                                        stop='txEnd'),
                                      itemClass=EnsemblTranscriptAnnotationSeq)
        return self.trans_db

    def gene_database(self):
        'Return an AnnotationDB of gene annotations.'
        if self.gene_db is None:
            self.gene_db = annotation.AnnotationDB(self.ucsc_ensGene_gene,
                                                   self.genome_seq,
                                                   checkFirstID=False,
                                                   sliceAttrDict=dict(
                                                       id='chrom',
                                                       start='txStart',
                                                       stop='txEnd'))
        return self.gene_db

    def protein_database(self):
        'Return an AnnotationDB of protein annotations.'
        if self.prot_db is None:
            self.protein_transcript_id_map = sqlgraph.MapView(self.ucsc_ensGtp,
                self.ucsc_ensGene_trans, 'select transcript from %s.ensGtp \
                where protein=%%s' % self.ucsc_db, inverseSQL='select protein \
                from %s.ensGtp where transcript=%%s' % self.ucsc_db)
            protein_slicedb = EnsemblProteinSliceDB(self, '%s.ensGene' %
                                                    self.ucsc_db,
                                                   serverInfo=self.ucsc_server,
                                                    primaryKey='name',
                                                  itemClass=UCSCSeqIntervalRow)
            self.prot_db = annotation.AnnotationDB(protein_slicedb,
                                                   self.genome_seq,
                                                   checkFirstID=False,
                                                   sliceAttrDict=dict(
                                                       id='chrom',
                                                       start='txStart',
                                                       stop='txEnd'))
            return self.prot_db

    def exon_database(self):
        'Return an AnnotationDB of exon annotations.'
        if self.exon_db is None:
            self.ens_transcripts_of_exons_map = sqlgraph.MapView(
                self.ens_exon_stable_id, self.ucsc_ensGene_trans, """\
select trans.stable_id from %s.exon_stable_id exon, \
%s.transcript_stable_id trans, %s.exon_transcript et where \
exon.exon_id=et.exon_id and trans.transcript_id=et.transcript_id and \
exon.stable_id=%%s""" % (self.ens_db, self.ens_db, self.ens_db))
            self.ens_exons_in_transcripts_map = sqlgraph.GraphView(
                self.ens_transcript_stable_id, self.ens_exon_stable_id, """\
select exon.stable_id from %s.exon_stable_id exon, %s.transcript_stable_id \
trans, %s.exon_transcript et where exon.exon_id=et.exon_id and \
trans.transcript_id=et.transcript_id and trans.stable_id=%%s order by \
et.rank""" % (self.ens_db, self.ens_db, self.ens_db))
            exon_slicedb = EnsemblOnDemandSliceDB(self)
            self.exon_db = annotation.AnnotationDB(exon_slicedb,
                                                   self.genome_seq,
                                                   checkFirstID=False,
                                                   sliceAttrDict=dict(id=1,
                                                                      start=2,
                                                                      stop=3,
                                                                orientation=4))
        return self.exon_db

class EnsemblTranscriptAnnotationSeqDescr(object):

    def __get__(self, obj, objtype):
        '''Concatenate exon sequences of a transcript to obtain
        its sequence.'''
        exon_count = obj.exonCount
        exon_starts = obj.exonStarts.split(',')[:exon_count]
        exon_ends = obj.exonEnds.split(',')[:exon_count]
        trans_seq = sequence.absoluteSlice(obj._anno_seq, int(exon_starts[0]),
                                           int(exon_ends[0]))
        for i in range(1, exon_count):
            trans_seq += sequence.absoluteSlice(obj._anno_seq,
                                                int(exon_starts[i]),
                                                int(exon_ends[i]))
        return trans_seq    # FIXME does this actually do what it's supposed to?

class EnsemblTranscriptAnnotationSeq(annotation.AnnotationSeq):
    '''An AnnotationSeq class for transcript annotations, implementing
    a custom 'sequence' property.'''
    sequence = EnsemblTranscriptAnnotationSeqDescr()

class EnsemblProteinSliceDB(sqlgraph.SQLTable):
    '''A sliceDB class for protein annotations. Basically, an SQLTable
    pointing to transcript data along with transparent mapping of keys.'''

    def __init__(self, res, *args, **kwargs):
        self.res = res
        sqlgraph.SQLTable.__init__(self, *args, **kwargs)

    def __getitem__(self, k):
        tid = self.res.protein_transcript_id_map[self.res.ucsc_ensGtp[k]].name
        return sqlgraph.SQLTable.__getitem__(self, tid)

    def keys(self):
        prot_keys = []
        trans_keys = SQLTable.keys(self)
        for tid in trans_keys:
            pid = (~self.res.protein_transcript_id_map[
                self.res.ucsc_ensGene[tid]]).name
            prot_keys.append(pid)
        return prot_keys

class EnsemblOnDemandSliceDB(object, UserDict.DictMixin):

    def __init__(self, res):
        self.data = {}
        self.res = res

    def __getitem__(self, k):
        try:
            return self.data[k]
        except KeyError:
            # Not cached yet, extract the exon from transcript data
            transcript = self.res.ens_transcripts_of_exons_map[
                self.res.ens_exon_stable_id[k]]
            transcript_exons = self.get_transcript_exons(transcript)
            # Cache all exons from that transcript to save time in the future.
            for exon in transcript_exons:
                self.data[exon[0]] = exon
            self.res.genome_seq.cacheHint({transcript.id: (transcript.txStart,
                                                           transcript.txEnd)},
                                          self)
            return self.data[k]

    def __setitem__(self, k, v):
        '''Method required by UserDict.DictMixin. Throws an exception
        (read-only sliceDB).'''
        raise ReadOnlyError('EnsemblOnDemandSliceDB is read-only')

    def __delitem__(self, k):
        '''Method required by UserDict.DictMixin. Throws an exception
        (read-only sliceDB).'''
        raise ReadOnlyError('EnsemblOnDemandSliceDB is read-only')

    def keys(self):
        'Returns keys present in the cache. FIXME: add support for SQL ones?'
        return self.data.keys()

    def get_ensembl_exon_ids(self, transcript_id):
        '''Obtain a list of stable IDs of exons associated with the
        specified transcript, ordered by rank.'''
        matching_edges = self.res.ens_exons_in_transcripts_map[
            self.res.ens_transcript_stable_id[transcript_id]]
        ids = []
        for exon in matching_edges.keys():
            ids.append(exon.stable_id)
        return ids

    def get_transcript_exons(self, trans_tuple):
        '''Parse the Ensembl transcript from UCSC provided via
        trans_tuple, extract exon data from it and return it
        as a list of tuples.'''
        transcript_id = trans_tuple.name
        chromosome = trans_tuple.chrom
        exon_count = trans_tuple.exonCount
        exon_starts = trans_tuple.exonStarts.split(',')[:exon_count]
        exon_ends = trans_tuple.exonEnds.split(',')[:exon_count]
        exons = []
        exon_ids = self.get_ensembl_exon_ids(transcript_id)
        for i in range(0, exon_count):
            e = (
                exon_ids[i],
                chromosome,
                exon_starts[i],
                exon_ends[i],
                trans_tuple.orientation)
            exons.append(e)
        return exons
<commit_msg>Added code for opening the UCSC ensPep table as a peptide SequenceDB.<commit_after>import UserDict

from pygr import annotation, seqdb, sequence, sqlgraph, worldbase
from pygr.dbfile import ReadOnlyError

class UCSCStrandDescr(object):

    def __get__(self, obj, objtype):
        if obj.strand == '+':
            return 1
        else:
            return -1

class UCSCSeqIntervalRow(sqlgraph.TupleO):
    orientation = UCSCStrandDescr()

class UCSCProteinSeq(sqlgraph.ProteinSQLSequence):
    '''Representation of UCSC protein-sequence tables such as ensGene,
    which lack the length column.'''

    def __len__(self):
        return self._select('length(seq)')

class UCSCEnsemblInterface(object):

    def __init__(self, ucsc_genome_name, ens_species=None, ucsc_serverInfo=None,
                ens_serverInfo=None):
        '''Set up everything needed to produce UCSC/Ensembl
        annotation databases. ucsc_genome_name should follow the worldbase
        naming convention. If ens_species is not specified, we will try
        to autodetect it.'''
        # Connect to both servers and prepare database names.
        if ucsc_serverInfo is not None:
            self.ucsc_server = ucsc_serverInfo
        else:
            self.ucsc_server = sqlgraph.DBServerInfo(
                host='genome-mysql.cse.ucsc.edu', user='genome')
        if ens_serverInfo is not None:
            self.ens_server = ens_serverInfo
        else:
            self.ens_server = sqlgraph.DBServerInfo(
                host='ensembldb.ensembl.org', port=5306, user='anonymous')
        self.ucsc_db = ucsc_genome_name.split('.')[-1]
        self.ens_db = self.get_ensembl_db_name(ens_species)
        # Connect to all the necessary tables.
        self.ucsc_ensGene_trans = sqlgraph.SQLTable('%s.ensGene' %
                                                    self.ucsc_db,
                                                   serverInfo=self.ucsc_server,
                                                    primaryKey='name',
                                                  itemClass=UCSCSeqIntervalRow)
        self.ucsc_ensGene_gene = sqlgraph.SQLTable('%s.ensGene' % self.ucsc_db,
                                                   serverInfo=self.ucsc_server,
                                                   primaryKey='name2',
                                                  itemClass=UCSCSeqIntervalRow)
        self.ucsc_ensGtp = sqlgraph.SQLTable('%s.ensGtp' % self.ucsc_db,
                                             serverInfo=self.ucsc_server,
                                             primaryKey='protein')
        self.ucsc_ensPep = sqlgraph.SQLTable('%s.ensPep' % self.ucsc_db,
                                             serverInfo=self.ucsc_server,
                                             itemClass=UCSCProteinSeq,
                                             itemSliceClass=seqdb.SeqDBSlice)
        self.ens_exon_stable_id = sqlgraph.SQLTable('%s.exon_stable_id'
                                                    % self.ens_db,
                                                    serverInfo=self.ens_server,
                                                    primaryKey='stable_id')
        self.ens_transcript_stable_id = sqlgraph.SQLTable(
            '%s.transcript_stable_id' % self.ens_db,
            serverInfo=self.ens_server, primaryKey='stable_id')
        # We will need this too.
        self.genome_seq = worldbase(ucsc_genome_name)
        # Initialise all cache variables.
        self.trans_db = None
        self.gene_db = None
        self.prot_db = None
        self.exon_db = None

    def get_ensembl_db_name(self, ens_prefix):
        '''Used by __init__(), obtains Ensembl database name matching
        the specified UCSC genome version'''
        ucsc_versions = sqlgraph.SQLTable('hgFixed.trackVersion',
                                          serverInfo=self.ucsc_server,
                                          primaryKey='db')
        ens_version = ucsc_versions[self.ucsc_db].version
        if ens_prefix is None:
            # Note: this assumes 'source' in hgFixed.trackVersion contains
            # the URI of the Ensembl data set and that the last path component
            # of that URI is the species name of that data set.
            ens_prefix = ucsc_versions[self.ucsc_db].source.split('/')[-2]
        cursor = self.ens_server.cursor()
        cursor.execute("show databases like '%s_core_%s_%%'" % (ens_prefix,
                                                                ens_version))
        return cursor.fetchall()[0][0]

    def transcript_database(self):
        'Return an AnnotationDB of transcript annotations.'
        if self.trans_db is None:
            self.trans_db = annotation.AnnotationDB(self.ucsc_ensGene_trans,
                                                    self.genome_seq,
                                                    checkFirstID=False,
                                                    sliceAttrDict=dict(
                                                        id='chrom',
                                                        start='txStart',
                                                        stop='txEnd'),
                                      itemClass=EnsemblTranscriptAnnotationSeq)
        return self.trans_db

    def gene_database(self):
        'Return an AnnotationDB of gene annotations.'
        if self.gene_db is None:
            self.gene_db = annotation.AnnotationDB(self.ucsc_ensGene_gene,
                                                   self.genome_seq,
                                                   checkFirstID=False,
                                                   sliceAttrDict=dict(
                                                       id='chrom',
                                                       start='txStart',
                                                       stop='txEnd'))
        return self.gene_db

    def protein_database(self):
        'Return an AnnotationDB of protein annotations.'
        if self.prot_db is None:
            self.protein_transcript_id_map = sqlgraph.MapView(self.ucsc_ensGtp,
                self.ucsc_ensGene_trans, 'select transcript from %s.ensGtp \
                where protein=%%s' % self.ucsc_db, inverseSQL='select protein \
                from %s.ensGtp where transcript=%%s' % self.ucsc_db)
            protein_slicedb = EnsemblProteinSliceDB(self, '%s.ensGene' %
                                                    self.ucsc_db,
                                                   serverInfo=self.ucsc_server,
                                                    primaryKey='name',
                                                  itemClass=UCSCSeqIntervalRow)
            self.prot_db = annotation.AnnotationDB(protein_slicedb,
                                                   self.genome_seq,
                                                   checkFirstID=False,
                                                   sliceAttrDict=dict(
                                                       id='chrom',
                                                       start='txStart',
                                                       stop='txEnd'))
            return self.prot_db

    def exon_database(self):
        'Return an AnnotationDB of exon annotations.'
        if self.exon_db is None:
            self.ens_transcripts_of_exons_map = sqlgraph.MapView(
                self.ens_exon_stable_id, self.ucsc_ensGene_trans, """\
select trans.stable_id from %s.exon_stable_id exon, \
%s.transcript_stable_id trans, %s.exon_transcript et where \
exon.exon_id=et.exon_id and trans.transcript_id=et.transcript_id and \
exon.stable_id=%%s""" % (self.ens_db, self.ens_db, self.ens_db))
            self.ens_exons_in_transcripts_map = sqlgraph.GraphView(
                self.ens_transcript_stable_id, self.ens_exon_stable_id, """\
select exon.stable_id from %s.exon_stable_id exon, %s.transcript_stable_id \
trans, %s.exon_transcript et where exon.exon_id=et.exon_id and \
trans.transcript_id=et.transcript_id and trans.stable_id=%%s order by \
et.rank""" % (self.ens_db, self.ens_db, self.ens_db))
            exon_slicedb = EnsemblOnDemandSliceDB(self)
            self.exon_db = annotation.AnnotationDB(exon_slicedb,
                                                   self.genome_seq,
                                                   checkFirstID=False,
                                                   sliceAttrDict=dict(id=1,
                                                                      start=2,
                                                                      stop=3,
                                                                orientation=4))
        return self.exon_db

class EnsemblTranscriptAnnotationSeqDescr(object):

    def __get__(self, obj, objtype):
        '''Concatenate exon sequences of a transcript to obtain
        its sequence.'''
        exon_count = obj.exonCount
        exon_starts = obj.exonStarts.split(',')[:exon_count]
        exon_ends = obj.exonEnds.split(',')[:exon_count]
        trans_seq = sequence.absoluteSlice(obj._anno_seq, int(exon_starts[0]),
                                           int(exon_ends[0]))
        for i in range(1, exon_count):
            trans_seq += sequence.absoluteSlice(obj._anno_seq,
                                                int(exon_starts[i]),
                                                int(exon_ends[i]))
        return trans_seq    # FIXME does this actually do what it's supposed to?

class EnsemblTranscriptAnnotationSeq(annotation.AnnotationSeq):
    '''An AnnotationSeq class for transcript annotations, implementing
    a custom 'sequence' property.'''
    sequence = EnsemblTranscriptAnnotationSeqDescr()

class EnsemblProteinSliceDB(sqlgraph.SQLTable):
    '''A sliceDB class for protein annotations. Basically, an SQLTable
    pointing to transcript data along with transparent mapping of keys.'''

    def __init__(self, res, *args, **kwargs):
        self.res = res
        sqlgraph.SQLTable.__init__(self, *args, **kwargs)

    def __getitem__(self, k):
        tid = self.res.protein_transcript_id_map[self.res.ucsc_ensGtp[k]].name
        return sqlgraph.SQLTable.__getitem__(self, tid)

    def keys(self):
        prot_keys = []
        trans_keys = SQLTable.keys(self)
        for tid in trans_keys:
            pid = (~self.res.protein_transcript_id_map[
                self.res.ucsc_ensGene[tid]]).name
            prot_keys.append(pid)
        return prot_keys

class EnsemblOnDemandSliceDB(object, UserDict.DictMixin):

    def __init__(self, res):
        self.data = {}
        self.res = res

    def __getitem__(self, k):
        try:
            return self.data[k]
        except KeyError:
            # Not cached yet, extract the exon from transcript data
            transcript = self.res.ens_transcripts_of_exons_map[
                self.res.ens_exon_stable_id[k]]
            transcript_exons = self.get_transcript_exons(transcript)
            # Cache all exons from that transcript to save time in the future.
            for exon in transcript_exons:
                self.data[exon[0]] = exon
            self.res.genome_seq.cacheHint({transcript.id: (transcript.txStart,
                                                           transcript.txEnd)},
                                          self)
            return self.data[k]

    def __setitem__(self, k, v):
        '''Method required by UserDict.DictMixin. Throws an exception
        (read-only sliceDB).'''
        raise ReadOnlyError('EnsemblOnDemandSliceDB is read-only')

    def __delitem__(self, k):
        '''Method required by UserDict.DictMixin. Throws an exception
        (read-only sliceDB).'''
        raise ReadOnlyError('EnsemblOnDemandSliceDB is read-only')

    def keys(self):
        'Returns keys present in the cache. FIXME: add support for SQL ones?'
        return self.data.keys()

    def get_ensembl_exon_ids(self, transcript_id):
        '''Obtain a list of stable IDs of exons associated with the
        specified transcript, ordered by rank.'''
        matching_edges = self.res.ens_exons_in_transcripts_map[
            self.res.ens_transcript_stable_id[transcript_id]]
        ids = []
        for exon in matching_edges.keys():
            ids.append(exon.stable_id)
        return ids

    def get_transcript_exons(self, trans_tuple):
        '''Parse the Ensembl transcript from UCSC provided via
        trans_tuple, extract exon data from it and return it
        as a list of tuples.'''
        transcript_id = trans_tuple.name
        chromosome = trans_tuple.chrom
        exon_count = trans_tuple.exonCount
        exon_starts = trans_tuple.exonStarts.split(',')[:exon_count]
        exon_ends = trans_tuple.exonEnds.split(',')[:exon_count]
        exons = []
        exon_ids = self.get_ensembl_exon_ids(transcript_id)
        for i in range(0, exon_count):
            e = (
                exon_ids[i],
                chromosome,
                exon_starts[i],
                exon_ends[i],
                trans_tuple.orientation)
            exons.append(e)
        return exons
<|endoftext|>


<commit_before>import logging

class MidiToOmx:
    def __init__(self, options = {}):
        # config
        self.options = options
        # attributes
        self.midi_input = None
        self.omxvideo = None
        self.logger = logging.getLogger(__name__)
        self.logger.setLevel(logging.DEBUG if 'verbose' in options and options['verbose'] else logging.INFO)

    def __del__(self):
        self.destroy()

    def setup(self, midi_input, omxvideo):
        self.destroy()
        self.midi_input = midi_input
        self.omxvideo = omxvideo
        # register callback
        if self.midi_input:
            self.midi_input.messageEvent += self._onMidiMessage
        self.logger.debug('midi-to-omx component loaded')

    def destroy(self):
        if self.midi_input:
            self.midi_input.messageEvent -= self._onMidiMessage
            self.midi_input = None
        # self.omxvideo = None

    def _onMidiMessage(self, msg):
        action, params = self._midiToOmx(msg)

        if not action:
            self.logger.debug('no omx action specified for midi note {0},{1}'.format(msg[0][0], msg[0][1]))
            return

        if not self.omxvideo:
            self.logger.warning('no mxvideo component specified to perform action on')
            return

        if action == 'play':
            self.omxvideo.play()
            return

        if action == 'start' and len(params) > 0:
            self.omxvideo.start(params[0])
            return

        if action == 'load' and len(params) > 0:
            self.omxvideo.load(params[0])
            return

        if action == 'stop':
            self.omxvideo.stop()
            return

        if action == 'pause':
            self.omxvideo.pause()
            return

        if action == 'toggle':
            self.omxvideo.toggle()

        if action == 'seek' and len(params) > 0:
            self.omxvideo.seek(params[0])
            return

        if action == 'speed' and len(params) > 0:
            self.omxvideo.speed(params[0])
            return

        self.logger.debug('midi note ({0}, {1}) did not map to any omx action'.format(msg[0][0], msg[0][1]))

    def _midiToOmx(self, msg):
        if not msg[0][0] in self.options:
            return (None, [])

        cur = self.options[msg[0][0]]

        if not msg[0][1] in cur:
            return (None, [])

        cur = cur[msg[0][1]]

        return (cur['action'] if 'action' in cur else None, cur['params'] if 'params' in cur else [])
<commit_msg>return after toggle action in midi-to-omx component<commit_after>import logging

class MidiToOmx:
    def __init__(self, options = {}):
        # config
        self.options = options
        # attributes
        self.midi_input = None
        self.omxvideo = None
        self.logger = logging.getLogger(__name__)
        self.logger.setLevel(logging.DEBUG if 'verbose' in options and options['verbose'] else logging.INFO)

    def __del__(self):
        self.destroy()

    def setup(self, midi_input, omxvideo):
        self.destroy()
        self.midi_input = midi_input
        self.omxvideo = omxvideo
        # register callback
        if self.midi_input:
            self.midi_input.messageEvent += self._onMidiMessage
        self.logger.debug('midi-to-omx component loaded')

    def destroy(self):
        if self.midi_input:
            self.midi_input.messageEvent -= self._onMidiMessage
            self.midi_input = None
        # self.omxvideo = None

    def _onMidiMessage(self, msg):
        action, params = self._midiToOmx(msg)

        if not action:
            self.logger.debug('no omx action specified for midi note {0},{1}'.format(msg[0][0], msg[0][1]))
            return

        if not self.omxvideo:
            self.logger.warning('no mxvideo component specified to perform action on')
            return

        if action == 'play':
            self.omxvideo.play()
            return

        if action == 'start' and len(params) > 0:
            self.omxvideo.start(params[0])
            return

        if action == 'load' and len(params) > 0:
            self.omxvideo.load(params[0])
            return

        if action == 'stop':
            self.omxvideo.stop()
            return

        if action == 'pause':
            self.omxvideo.pause()
            return

        if action == 'toggle':
            self.omxvideo.toggle()
            return

        if action == 'seek' and len(params) > 0:
            self.omxvideo.seek(params[0])
            return

        if action == 'speed' and len(params) > 0:
            self.omxvideo.speed(params[0])
            return

        self.logger.debug('midi note ({0}, {1}) did not map to any omx action'.format(msg[0][0], msg[0][1]))

    def _midiToOmx(self, msg):
        if not msg[0][0] in self.options:
            return (None, [])

        cur = self.options[msg[0][0]]

        if not msg[0][1] in cur:
            return (None, [])

        cur = cur[msg[0][1]]

        return (cur['action'] if 'action' in cur else None, cur['params'] if 'params' in cur else [])
<|endoftext|>


<commit_before>from datetime import date
from pywps.Process import WPSProcess

import logging
logger = logging.getLogger(__name__)

class AnalogsProcess(WPSProcess):

  def __init__(self):
    # definition of this process
    WPSProcess.__init__(self,
      identifier = "analogs",
      title="Days with analog pressure pattern",
      version = "0.2",
      metadata= [
              {"title": "Institut Pierre Simon Laplace", "href": "https://www.ipsl.fr/en/"}
              ],
      abstract="Search for days with analog pressure pattern",
      statusSupported=True,
      storeSupported=True
      )

    self.resource = self.addComplexInput(
      identifier="resource",
      title="Resource",
      abstract="URL to netCDF file",
      minOccurs=0,
      maxOccurs=1000,
      maxmegabites=5000,
      formats=[{"mimeType":"application/x-netcdf"}],
      )

    self.experiment = self.addLiteralInput(
      identifier="experiment",
      title="Data experiment",
      abstract="Choose the experiment (if 'None' is selected, provide a resource)",
      default="NCEP",
      type=type(''),
      minOccurs=1,
      maxOccurs=1,
      allowedValues=['None','NCEP']
      )

    #self.region = self.addBBoxInput(
      #identifier="region",
      #title="Region",
      #abstract="coordinates to define the region: (minlon,minlat,maxlon,maxlat)",
      ##default='-80,50,22.5,70', #"-80,22.5,50,70",
      ##type=type(''),
      #minOccurs=1,
      #maxOccurs=1,
      #crss=["EPSG:4326"]
      #)

    self.region = self.addLiteralInput(
      identifier="region",
      title="Region",
      abstract="coordinates to define the region: (minlon,minlat,maxlon,maxlat)",
      default='-80,50,22.5,70', #"-80,22.5,50,70",
      minOccurs=1,
      maxOccurs=1,
      type=type(''),
      )

    self.dateSt = self.addLiteralInput(
      identifier="dateSt",
      title="Start date of analyse period",
      abstract="This is a Date: 2013-07-15",
      default="2013-07-15",
      type=type(date(2013,7,15)),
      minOccurs=1,
      maxOccurs=1,
      )

    self.dateEn = self.addLiteralInput(
      identifier="dateEn",
      title="End date of analyse period",
      abstract="This is a Date: 2013-12-31",
      default="2014-12-31",
      type=type(date(2014,12,31)),
      minOccurs=1,
      maxOccurs=1,
      )

    self.refSt = self.addLiteralInput(
      identifier="refSt",
      title="Start reference period",
      abstract="Start YEAR of reference period",
      default="1955-01-01",
      type=type(date(1955,01,01)),
      minOccurs=1,
      maxOccurs=1,
      )

    self.refEn = self.addLiteralInput(
      identifier="refEn",
      title="End reference period",
      abstract="End YEAR of reference period",
      default="1957-12-31",
      type=type(date(1957,12,31)),
      minOccurs=1,
      maxOccurs=1,
      )

    self.normalize = self.addLiteralInput(
      identifier="normalize",
      title="normalisation",
      abstract="Normalize by substraction of annual cycle",
      default='base',
      type=type(''),
      minOccurs=1,
      maxOccurs=1,
      allowedValues=['None','base','sim','own']
        )

    self.distance = self.addLiteralInput(
      identifier="dist",
      title="Distance",
      abstract="Distance function to define analogues",
      default='euclidean',
      type=type(''),
      minOccurs=1,
      maxOccurs=1,
      allowedValues=['euclidean','mahalanobis','cosine','of']
        )

    self.outformat = self.addLiteralInput(
      identifier="outformat",
      title="output file format",
      abstract="Choose the output format for the analog output file",
      default="ascii",
      type=type(''),
      minOccurs=1,
      maxOccurs=1,
      allowedValues=['ascii','netCDF4']
      )

    self.timewin = self.addLiteralInput(
      identifier="timewin",
      title="Time window",
      abstract="Nr of days following the analog day",
      default=30,
      type=type(1),
      minOccurs=0,
      maxOccurs=1,
      )

    self.variable = self.addLiteralInput(
      identifier="variable",
      title="Variable",
      abstract="Variable name in resource",
      default='slp',
      type=type(''),
      minOccurs=0,
      maxOccurs=1,
      )

    # self.seacyc = self.addLiteralInput(
    #   identifier="seacyc",
    #   title="Seasonal Cycle",
    #   abstract="normalized by the Seasonal Cycle",
    #   default=True,
    #   type=type(boolean),
    #   minOccurs=0,
    #   maxOccurs=1,
    #   )

      # #seacyc=True,
      # cycsmooth=91,
      # nanalog=20,
      # seasonwin=30,
      # distfun='rms',
      # calccor=True,
      # silent=False)

    ### ###################
    # define the outputs
    ### ###################

    self.config = self.addComplexOutput(
      identifier="config",
      title="Config File",
      abstract="Config file used for the Fortran process",
      formats=[{"mimeType":"text/plain"}],
      asReference=True,
      )

    self.analogs = self.addComplexOutput(
      identifier="analogs",
      title="Analogs File",
      abstract="mulit-column text file",
      formats=[{"mimeType":"text/plain"}],
      asReference=True,
      )

  def execute(self):
    from os import system, path
    from tempfile import mkstemp
    from flyingpigeon import analogs
    import datetime as dt

    from flyingpigeon.ocgis_module import call
    from flyingpigeon.weatherregimes import get_NCEP

    self.status.set('execution started at : %s '  % dt.datetime.now(),5)

    refSt = self.getInputValues(identifier='refSt')
    refEn = self.getInputValues(identifier='refEn')
    dateSt = self.getInputValues(identifier='dateSt')
    dateEn = self.getInputValues(identifier='dateEn')

    normalize = self.getInputValues(identifier='normalize')[0]
    if normalize == 'None':
      seacyc = False
    else:
      seacyc = True

    distance = self.getInputValues(identifier='dist')[0]
    outformat = self.getInputValues(identifier='outformat')[0]

    if outformat == 'ascii':
      outformat = '.txt'
    elif outformat == 'netCDF':
      outformat = '.nc'
    else:
      logger.error('output format not valid')

    refSt = dt.datetime.strptime(refSt[0],'%Y-%m-%d')
    refEn = dt.datetime.strptime(refEn[0],'%Y-%m-%d')
    dateSt = dt.datetime.strptime(dateSt[0],'%Y-%m-%d')
    dateEn = dt.datetime.strptime(dateEn[0],'%Y-%m-%d')

    timewin = int(self.getInputValues(identifier='timewin')[0])

    start = min( refSt, dateSt )
    end = max( refEn, dateEn )

    #################
    # get input data
    #################

    try:
      experiment = self.getInputValues(identifier='experiment')[0]
      if experiment == 'NCEP':
        input = get_NCEP(start = start.year, end = end.year )
      elif   experiment == 'None':
        input = self.getInputValues(identifier='resource')
      else:
        logger.error('input experiment not found')

      region = self.getInputValues(identifier='region')[0]
      nc_subset = analogs.subset(resource=input,
                                 bbox=region)
    except Exception as e :
      msg = 'failed to fetch input files %s' % e
      logger.error(msg)
      raise Exception(msg)

    ########################
    # input data preperation
    ########################

    try:
      archive = call(resource=nc_subset, time_range=[refSt , refEn])
      simulation = call(resource=nc_subset, time_range=[dateSt , dateEn])
      if seacyc == True:
        analogs.seacyc(archive, simulation, method=normalize)

    except Exception as e:
      msg = 'failed to prepare archive and simulation files %s ' % e
      logger.debug(msg)
      raise Exception(msg)

    ip, output = mkstemp(dir='.',suffix='.txt')
    output_file =  path.abspath(output)
    files=[path.abspath(archive), path.abspath(simulation), output_file]

    ############################
    # generating the config file
    ############################

    try:
      config_file = analogs.get_configfile(files=files,
        timewin=timewin,
        varname='slp',
        seacyc=seacyc,
        cycsmooth=91,
        nanalog=20,
        seasonwin=30,
        distfun=distance,
        outformat=outformat,
        calccor=True,
        silent=False)
    except Exception as e:
      msg = 'failed to generate config file %s ' % e
      logger.debug(msg)
      raise Exception(msg)

    #######################
    # CASTf90 call
    #######################

    try:
      #self.status.set('execution of CASTf90', 50)
      cmd = 'analogue.out %s' % path.relpath(config_file)
      system(cmd)
    except Exception as e:
      msg = 'CASTf90 failed %s ' % e
      logger.error(msg)
      raise Exception(msg)

    self.status.set('preparting output', 99)
    self.config.setValue( config_file )
    self.analogs.setValue( output_file )
    self.status.set('execution ended', 100)
<commit_msg>pull output and errors of castf90 to logfile<commit_after>from datetime import date
from pywps.Process import WPSProcess

import logging
logger = logging.getLogger(__name__)

class AnalogsProcess(WPSProcess):

  def __init__(self):
    # definition of this process
    WPSProcess.__init__(self,
      identifier = "analogs",
      title="Days with analog pressure pattern",
      version = "0.2",
      metadata= [
              {"title": "Institut Pierre Simon Laplace", "href": "https://www.ipsl.fr/en/"}
              ],
      abstract="Search for days with analog pressure pattern",
      statusSupported=True,
      storeSupported=True
      )

    #self.resource = self.addComplexInput(
      #identifier="resource",
      #title="Resource",
      #abstract="URL to netCDF file",
      #minOccurs=0,
      #maxOccurs=1000,
      #maxmegabites=5000,
      #formats=[{"mimeType":"application/x-netcdf"}],
      #)

    self.experiment = self.addLiteralInput(
      identifier="experiment",
      title="Data experiment",
      abstract="Choose the experiment (if 'None' is selected, provide a resource)",
      default="NCEP",
      type=type(''),
      minOccurs=1,
      maxOccurs=1,
      allowedValues=['None','NCEP']
      )

    #self.region = self.addBBoxInput(
      #identifier="region",
      #title="Region",
      #abstract="coordinates to define the region: (minlon,minlat,maxlon,maxlat)",
      ##default='-80,50,22.5,70', #"-80,22.5,50,70",
      ##type=type(''),
      #minOccurs=1,
      #maxOccurs=1,
      #crss=["EPSG:4326"]
      #)

    self.region = self.addLiteralInput(
      identifier="region",
      title="Region",
      abstract="coordinates to define the region: (minlon,minlat,maxlon,maxlat)",
      default='-80,50,22.5,70', #"-80,22.5,50,70",
      minOccurs=1,
      maxOccurs=1,
      type=type(''),
      )

    self.dateSt = self.addLiteralInput(
      identifier="dateSt",
      title="Start date of analyse period",
      abstract="This is a Date: 2013-07-15",
      default="2013-07-15",
      type=type(date(2013,7,15)),
      minOccurs=1,
      maxOccurs=1,
      )

    self.dateEn = self.addLiteralInput(
      identifier="dateEn",
      title="End date of analyse period",
      abstract="This is a Date: 2013-12-31",
      default="2014-12-31",
      type=type(date(2014,12,31)),
      minOccurs=1,
      maxOccurs=1,
      )

    self.refSt = self.addLiteralInput(
      identifier="refSt",
      title="Start reference period",
      abstract="Start YEAR of reference period",
      default="1955-01-01",
      type=type(date(1955,01,01)),
      minOccurs=1,
      maxOccurs=1,
      )

    self.refEn = self.addLiteralInput(
      identifier="refEn",
      title="End reference period",
      abstract="End YEAR of reference period",
      default="1957-12-31",
      type=type(date(1957,12,31)),
      minOccurs=1,
      maxOccurs=1,
      )

    self.normalize = self.addLiteralInput(
      identifier="normalize",
      title="normalisation",
      abstract="Normalize by substraction of annual cycle",
      default='base',
      type=type(''),
      minOccurs=1,
      maxOccurs=1,
      allowedValues=['None','base','sim','own']
        )

    self.distance = self.addLiteralInput(
      identifier="dist",
      title="Distance",
      abstract="Distance function to define analogues",
      default='euclidean',
      type=type(''),
      minOccurs=1,
      maxOccurs=1,
      allowedValues=['euclidean','mahalanobis','cosine','of']
        )

    self.outformat = self.addLiteralInput(
      identifier="outformat",
      title="output file format",
      abstract="Choose the output format for the analog output file",
      default="ascii",
      type=type(''),
      minOccurs=1,
      maxOccurs=1,
      allowedValues=['ascii','netCDF4']
      )

    self.timewin = self.addLiteralInput(
      identifier="timewin",
      title="Time window",
      abstract="Nr of days following the analog day",
      default=30,
      type=type(1),
      minOccurs=0,
      maxOccurs=1,
      )

    self.variable = self.addLiteralInput(
      identifier="variable",
      title="Variable",
      abstract="Variable name in resource",
      default='slp',
      type=type(''),
      minOccurs=0,
      maxOccurs=1,
      )

    # self.seacyc = self.addLiteralInput(
    #   identifier="seacyc",
    #   title="Seasonal Cycle",
    #   abstract="normalized by the Seasonal Cycle",
    #   default=True,
    #   type=type(boolean),
    #   minOccurs=0,
    #   maxOccurs=1,
    #   )

      # #seacyc=True,
      # cycsmooth=91,
      # nanalog=20,
      # seasonwin=30,
      # distfun='rms',
      # calccor=True,
      # silent=False)

    ### ###################
    # define the outputs
    ### ###################

    self.config = self.addComplexOutput(
      identifier="config",
      title="Config File",
      abstract="Config file used for the Fortran process",
      formats=[{"mimeType":"text/plain"}],
      asReference=True,
      )

    self.analogs = self.addComplexOutput(
      identifier="analogs",
      title="Analogs File",
      abstract="mulit-column text file",
      formats=[{"mimeType":"text/plain"}],
      asReference=True,
      )

  def execute(self):
    from os import path
    from tempfile import mkstemp
    from flyingpigeon import analogs
    import datetime as dt

    from flyingpigeon.ocgis_module import call
    from flyingpigeon.weatherregimes import get_NCEP

    self.status.set('execution started at : %s '  % dt.datetime.now(),5)

    refSt = self.getInputValues(identifier='refSt')
    refEn = self.getInputValues(identifier='refEn')
    dateSt = self.getInputValues(identifier='dateSt')
    dateEn = self.getInputValues(identifier='dateEn')

    normalize = self.getInputValues(identifier='normalize')[0]
    if normalize == 'None':
      seacyc = False
    else:
      seacyc = True

    distance = self.getInputValues(identifier='dist')[0]
    outformat = self.getInputValues(identifier='outformat')[0]

    if outformat == 'ascii':
      outformat = '.txt'
    elif outformat == 'netCDF':
      outformat = '.nc'
    else:
      logger.error('output format not valid')

    refSt = dt.datetime.strptime(refSt[0],'%Y-%m-%d')
    refEn = dt.datetime.strptime(refEn[0],'%Y-%m-%d')
    dateSt = dt.datetime.strptime(dateSt[0],'%Y-%m-%d')
    dateEn = dt.datetime.strptime(dateEn[0],'%Y-%m-%d')

    timewin = int(self.getInputValues(identifier='timewin')[0])

    start = min( refSt, dateSt )
    end = max( refEn, dateEn )

    #################
    # get input data
    #################

    try:
      experiment = self.getInputValues(identifier='experiment')[0]
      if experiment == 'NCEP':
        input = get_NCEP(start = start.year, end = end.year )
      elif   experiment == 'None':
        input = self.getInputValues(identifier='resource')
      else:
        logger.error('input experiment not found')

      region = self.getInputValues(identifier='region')[0]
      nc_subset = analogs.subset(resource=input,
                                 bbox=region)
    except Exception as e :
      msg = 'failed to fetch input files %s' % e
      logger.error(msg)
      raise Exception(msg)

    ########################
    # input data preperation
    ########################

    try:
      archive = call(resource=nc_subset, time_range=[refSt , refEn])
      simulation = call(resource=nc_subset, time_range=[dateSt , dateEn])
      if seacyc == True:
        analogs.seacyc(archive, simulation, method=normalize)

    except Exception as e:
      msg = 'failed to prepare archive and simulation files %s ' % e
      logger.debug(msg)
      raise Exception(msg)

    ip, output = mkstemp(dir='.',suffix='.txt')
    output_file =  path.abspath(output)
    files=[path.abspath(archive), path.abspath(simulation), output_file]

    ############################
    # generating the config file
    ############################

    try:
      config_file = analogs.get_configfile(files=files,
        timewin=timewin,
        varname='slp',
        seacyc=seacyc,
        cycsmooth=91,
        nanalog=20,
        seasonwin=30,
        distfun=distance,
        outformat=outformat,
        calccor=True,
        silent=False)
    except Exception as e:
      msg = 'failed to generate config file %s ' % e
      logger.debug(msg)
      raise Exception(msg)

    #######################
    # CASTf90 call
    #######################
    import subprocess
    import shlex

    try:
      #self.status.set('execution of CASTf90', 50)
      cmd = 'analogue.out %s' % path.relpath(config_file)
      #system(cmd)
      args = shlex.split(cmd)
      output,error = subprocess.Popen(args, stdout = subprocess.PIPE, stderr= subprocess.PIPE).communicate()
      logger.info('analogue.out info:\n %s ' % output)
      logger.debug('analogue.out errors:\n %s ' % error)
    except Exception as e:
      msg = 'CASTf90 failed %s ' % e
      logger.error(msg)
      raise Exception(msg)

    self.status.set('preparting output', 99)
    self.config.setValue( config_file )
    self.analogs.setValue( output_file )
    self.status.set('execution ended', 100)
<|endoftext|>


<commit_before>import sys
from io import BytesIO
from contextlib import contextmanager
from PIL import Image

from django.core.files.uploadedfile import InMemoryUploadedFile

from .. import settings
from .. import models

from .models import *

def get_image_data():
    io = BytesIO()
    size = (200,200)
    color = (255,0,0,0)
    image = Image.new("RGBA", size, color)
    image.save(io, format='JPEG')
    io.seek(0)
    return io

def get_image_in_memory_data():
    io = get_image_data()
    image_file = InMemoryUploadedFile(io, None, 'foo.jpg',
                            'jpeg', sys.getsizeof(io), None)
    image_file.seek(0)
    return image_file

def create_image_file(name):
    io = get_image_data()
    with open(name, 'wb') as f:
        f.write(io.read())

def get_image_size(path):
    with Image.open(path) as img:
        return img.size

@contextmanager
def patch_settings(settings_dict):
    saved_settings = {}
    for key, value in settings_dict.items():
        saved_settings[key] = settings.CONF[key]
        settings.CONF[key] = value
    yield
    for key, value in saved_settings.items():
        settings.CONF[key] = value

def clean_db():
    TestModel.objects.all().delete()
    AnotherTestModel.objects.all().delete()
    WrongTestModel.objects.all().delete()
    models.Image.objects.all().delete()
<commit_msg>Changed patch_settings using dict.update<commit_after>import sys
from io import BytesIO
from contextlib import contextmanager
from PIL import Image

from django.core.files.uploadedfile import InMemoryUploadedFile

from .. import settings
from .. import models

from .models import *

def get_image_data():
    io = BytesIO()
    size = (200,200)
    color = (255,0,0,0)
    image = Image.new("RGBA", size, color)
    image.save(io, format='JPEG')
    io.seek(0)
    return io

def get_image_in_memory_data():
    io = get_image_data()
    image_file = InMemoryUploadedFile(io, None, 'foo.jpg',
                            'jpeg', sys.getsizeof(io), None)
    image_file.seek(0)
    return image_file

def create_image_file(name):
    io = get_image_data()
    with open(name, 'wb') as f:
        f.write(io.read())

def get_image_size(path):
    with Image.open(path) as img:
        return img.size

@contextmanager
def patch_settings(settings_dict):
    saved_settings = {}
    for key, value in settings_dict.items():
        saved_settings[key] = settings.CONF[key]
        settings.CONF[key] = value
    yield
    settings.CONF.update(saved_settings)

def clean_db():
    TestModel.objects.all().delete()
    AnotherTestModel.objects.all().delete()
    WrongTestModel.objects.all().delete()
    models.Image.objects.all().delete()
<|endoftext|>


<commit_before>"""
Processes for Species distribution
Author: Nils Hempelmann (nils.hempelmann@lsce.ipsl.fr)
"""

from pywps.Process import WPSProcess
import logging
logger = logging.getLogger(__name__)

from flyingpigeon.sdm import _SDMINDICES_

class sdmcsvProcess(WPSProcess):
    def __init__(self):
        WPSProcess.__init__(
            self,
            identifier = "sdm_csv",
            title = "SDM -- CSV table",
            version = "0.9",
            metadata=[
                {"title":"LWF", "href": "http://www.lwf.bayern.de/"},
                {"title":"Documentation", "href":"http://flyingpigeon.readthedocs.io/en/latest/descriptions/index.html#species-distribution-model"},
	              {"title":"Journal of Climatology", "href":"http://www.hindawi.com/journals/jcli/2013/787250/"},
	              {"title":"Tutorial", "href":"http://flyingpigeon.readthedocs.io/en/latest/tutorials/sdm.html"},
                ],
            abstract="Species distribution model for tree species based on GBIF presence/absence data and climate model data. Indices will be calculated while processing",
            statusSupported=True,
            storeSupported=True
            )

        # Literal Input Data
        # ------------------
        self.resources = self.addComplexInput(
            identifier="resources",
            title="NetCDF File",
            abstract="NetCDF File",
            minOccurs=1,
            maxOccurs=500,
            maxmegabites=50000,
            formats=[{"mimeType":"application/x-netcdf"}],
            )

        self.gbif = self.addComplexInput(
            identifier="gbif",
            title="GBIF csv file",
            abstract="GBIF table (csv) with tree occurence (output of 'GBIF data fetch' process )",
            minOccurs=1,
            maxOccurs=1,
            #maxmegabites=50,
            formats=[{"mimeType":"text/csv"}],
            )

        self.input_indices = self.addLiteralInput(
            identifier="input_indices",
            title="Indices",
            abstract="Climate indices related to growth conditions of tree species",
            default=['TG_JJA', 'TNn_Jan'],
            type=type(''),
            minOccurs=1,
            maxOccurs=10,
            allowedValues=_SDMINDICES_
            )

        self.period = self.addLiteralInput(
            identifier="period",
            title="Reference period",
            abstract="Reference period for climate conditions (all = entire timeseries)",
            default="all",
            type=type(''),
            minOccurs=1,
            maxOccurs=1,
            allowedValues=['all','1951-1980', '1961-1990', '1971-2000','1981-2010']
            )

        self.archive_format = self.addLiteralInput(
            identifier="archive_format",
            title="Archive format",
            abstract="Result files will be compressed into archives. Choose an appropriate format",
            default="tar",
            type=type(''),
            minOccurs=1,
            maxOccurs=1,
            allowedValues=['zip','tar']
            )

        ###########
        ### OUTPUTS
        ###########

        self.output_csv = self.addComplexOutput(
            identifier="output_csv",
            title="Tree species table",
            abstract="Extracted CSV file containing the tree species table",
            formats=[{"mimeType":"text/csv"}],
            asReference=True,
            )

        self.output_gbif = self.addComplexOutput(
            identifier="output_gbif",
            title="Graphic of GBIF coordinates",
            abstract="PNG graphic file showing the presence of tree species according to the CSV file",
            formats=[{"mimeType":"image/png"}],
            asReference=True,
            )

        self.output_PA = self.addComplexOutput(
            identifier="output_PA",
            title="Graphic of PA mask",
            abstract="PNG graphic file showing PA mask generated based on netCDF spatial increment",
            formats=[{"mimeType":"image/png"}],
            asReference=True,
            )

        self.output_indices = self.addComplexOutput(
            identifier="output_indices",
            title="Climate indices for growth conditions over all timesteps (3D)",
            abstract="Archive (tar/zip) containing calculated climate indices as netCDF files",
            formats=[{"mimeType":"application/x-tar"}, {"mimeType":"application/zip"}],
            asReference=True,
            )

        self.output_reference = self.addComplexOutput(
            identifier="output_reference",
            title="Climate indices for growth conditions of reference period (2D)",
            abstract="Archive (tar/zip) containing calculated climate indices as netCDF files",
            formats=[{"mimeType":"application/x-tar"}, {"mimeType":"application/zip"}],
            asReference=True,
            )

        self.output_prediction = self.addComplexOutput(
            identifier="output_prediction",
            title="predicted growth conditions",
            abstract="Archive (tar/zip) containing the netCDF files of the predicted growth conditions",
            formats=[{"mimeType":"application/x-tar"}, {"mimeType":"application/zip"}],
            asReference=True,
            )

        self.output_info = self.addComplexOutput(
            identifier="output_info",
            title="GAM statistics information",
            abstract="Graphics and information of the learning statistics",
            formats=[{"mimeType":"image/png"}],
            asReference=True,
            )

    def execute(self):
      from os.path import basename

      from flyingpigeon import sdm
      from flyingpigeon.utils import archive

      self.status.set('Start process', 0)

      try:
        logger.info('reading the arguments')
        resources = self.getInputValues(identifier='resources')
        csv_file = self.getInputValues(identifier='gbif')[0]
        #period = self.period.getValue()
        period = self.getInputValues(identifier='period')
        period = period[0]
        #indices = self.input_indices.getValue()
        indices = self.getInputValues(identifier='input_indices')
        logger.debug("indices = %s", indices)
        archive_format = self.archive_format.getValue()
      except Exception as e:
        logger.error('failed to read in the arguments %s ' % e)
      logger.info('indices %s ' % indices)

      try:
        self.status.set('read in latlon coordinates of tree observations', 10)
        latlon = sdm.latlon_gbifcsv(csv_file)
      except Exception as e:
        logger.exception('failed to extract the latlon points')

      try:
        self.status.set('plot map', 80)
        from flyingpigeon.visualisation import map_gbifoccurrences

        # latlon = sdm.latlon_gbifdic(gbifdic)
        occurence_map = map_gbifoccurrences(latlon)
      except Exception as e:
        logger.exception('failed to plot occurence map %s' % e)

      try:
        self.status.set('generating the PA mask', 20)
        PAmask = sdm.get_PAmask(coordinates=latlon, domain='EUR-11')
        logger.info('PA mask sucessfully generated')
      except Exception as e:
        logger.exception('failed to generate the PA mask: %s' % e)

      try:
        self.status.set('Ploting PA mask', 25)
        from flyingpigeon.visualisation import map_PAmask
        PAmask_png = map_PAmask(PAmask)
      except Exception as e:
        logger.exception('failed to plot the PA mask: %s' % e )

      #################################
      ### calculate the climate indices
      #################################

      # get the indices
      ncs_indices = None
      try:
        self.status.set('start calculation of climate indices for %s' % indices, 30 )
        ncs_indices = sdm.get_indices(resources=resources, indices=indices)
        logger.info('indice calculation done')
      except:
        msg = 'failed to calculate indices'
        logger.exception(msg)
        raise Exception(msg)

      try:
        archive_indices = archive(ncs_indices , format=archive_format)
        logger.info('indices 3D added to tarfile')
      except:
        msg = 'failed adding indices to tar'
        logger.exception(msg)
        raise Exception(msg)

      indices_dic = None
      try:
        # sort indices
        indices_dic = sdm.sort_indices(ncs_indices)
        logger.info('indice files sorted for %s Datasets' % len(indices_dic.keys()))
      except:
        msg = 'failed to sort indices'
        logger.exception(msg)
        raise Exception(msg)

      ncs_references = []
      species_files = []
      statistics_info = []

      for count,key in enumerate(indices_dic.keys()):
        try:
          self.status.set('Start processing of %s ' % key, 40 + count * 10)

          ncs = indices_dic[key]

          logger.info('with %s files' % len(ncs))

          try:
            ncs_references.extend(sdm.get_reference(ncs_indices=ncs, period=period))
            logger.info('reference indice calculated %s ' % ncs_references)
          except:
            msg = 'failed adding ref indices to tar'
            logger.exception(msg)
            raise Exception(msg)

          # for nc_reference in ncs_references:
          #   tar_reference.add(nc_reference,
          #       arcname = basename(nc_reference))# nc_reference.replace(os.path.abspath(os.path.curdir), ""))

          # logger.info('reference indices added to tarfile')

        except:
          msg = 'failed to calculate reference indices.'
          logger.exception(msg)
          raise Exception(msg)

        archive_references = None
        try:
          archive_references = archive(ncs_references , format=archive_format)
          logger.info('indices 2D added to archive')
        except:
          msg = 'failed adding 2D indices to archive'
          logger.exception(msg)
          raise Exception(msg)

        archive_predicion = None
        try:
          archive_predicion = archive(species_files , format=archive_format)
          logger.info('species_files added to archive')
        except:
          msg = 'failed adding species_files indices to archive'
          logger.exception(msg)
          raise Exception(msg)

        try:
          gam_model, predict_gam, gam_info = sdm.get_gam(ncs_references,PAmask)
          statistics_info.append(gam_info)
          self.status.set('GAM sucessfully trained', 70)
        except:
          msg = 'failed to train GAM'
          logger.exception(msg)
          raise Exception(msg)

        try:
          prediction = sdm.get_prediction(gam_model, ncs_indices)
          self.status.set('prediction done', 80)
        except:
          msg = 'failed to predict'
          logger.exception(msg)
          raise Exception(msg)

        try:
          from numpy import invert, isnan, nan, broadcast_arrays, array, zeros, linspace, meshgrid
          mask = invert(isnan(PAmask))
          mask = broadcast_arrays(prediction, mask)[1]
          prediction[mask==False] = nan
          self.status.set('land sea mask for predicted data', 90)
        except:
          logger.exception('failed to mask predicted data')

        try:
          species_files.append(sdm.write_to_file(ncs_indices[0], prediction))

          logger.info('Favourabillity written to file')
          #tar_prediction.add(species_file,
           #               arcname = basename(species_file))#.replace(os.path.abspath(os.path.curdir), ""))
        except:
          msg = 'failed to write species file'
          logger.exception(msg)
          raise Exception(msg)

      from flyingpigeon.visualisation import concat_images
      statistics_infos = None
      try:
        statistics_infos = concat_images(statistics_info, orientation='v')
      except:
        msg = 'failed to concat images'
        logger.exception(msg)
        raise Exception(msg)

      self.output_csv.setValue( csv_file )
      self.output_gbif.setValue( occurence_map )
      self.output_PA.setValue( PAmask_png )
      self.output_indices.setValue( archive_indices )
      self.output_reference.setValue (archive_references )
      self.output_prediction.setValue (archive_predicion )
      self.output_info.setValue( statistics_infos )

      self.status.set('done', 100)

      # # try:
      # #   # open tar files
      # #   tar_reference = tarfile.open('reference.tar', "w")
      # #   tar_indices = tarfile.open('indices.tar', "w")

      # #   tar_info = tarfile.open('info.tar', "w")
      # #   tar_prediction = tarfile.open('prediction.tar', "w")

      # #   logger.info('tar files prepared')
      # # except:
      # #   msg = 'tar file preparation failed'
      # #   logger.exception(msg)
      # #   raise Exception(msg)

      # # try:
      # #   #tar_indices.close()
      # #   #tar_reference.close()

      # #   tar_prediction.close()
      # #   #tar_info.close()

      # #   logger.info('tar files closed')
      # # except:
      # #   logger.exception('tar file closing failed')
      # #   raise Exception
<commit_msg>pep8 conform wps_sdm_csv.py<commit_after>"""
Processes for Species distribution
Author: Nils Hempelmann (nils.hempelmann@lsce.ipsl.fr)
"""

from pywps.Process import WPSProcess
from flyingpigeon.sdm import _SDMINDICES_
import logging
logger = logging.getLogger(__name__)

class sdmcsvProcess(WPSProcess):
    def __init__(self):
        WPSProcess.__init__(
            self,
            identifier="sdm_csv",
            title="SDM -- CSV table",
            version="0.9",
            metadata=[
                {"title": "LWF",
                 "href": "http://www.lwf.bayern.de/"},
                {"title": "Doc",
                 "href": "http://flyingpigeon.readthedocs.io/en/latest/\
                        descriptions/index.html#species-distribution-model"},
                {"title": "Paper",
                 "href": "http://www.hindawi.com/journals/jcli/2013/787250/"},
                {"title": "Tutorial",
                 "href": "http://flyingpigeon.readthedocs.io/en/latest/\
                 tutorials/sdm.html"},
                ],
            abstract="Species distribution model for tree species based on\
             GBIF presence/absence data and climate model data. Indices will\
             be calculated while processing",
            statusSupported=True,
            storeSupported=True
            )

        # Literal Input Data
        # ------------------
        self.resources = self.addComplexInput(
            identifier="resources",
            title="NetCDF File",
            abstract="NetCDF File",
            minOccurs=1,
            maxOccurs=500,
            maxmegabites=50000,
            formats=[{"mimeType": "application/x-netcdf"}],
            )

        self.gbif = self.addComplexInput(
            identifier="gbif",
            title="GBIF csv file",
            abstract="GBIF table (csv) with tree occurence \
            (output of 'GBIF data fetch' process )",
            minOccurs=1,
            maxOccurs=1,
            # maxmegabites=50,
            formats=[{"mimeType": "text/csv"}],
            )

        self.input_indices = self.addLiteralInput(
            identifier="input_indices",
            title="Indices",
            abstract="Climate indices related to growth conditions",
            # default=['TG_JJA', 'TNn_Jan'],
            type=type(''),
            minOccurs=1,
            maxOccurs=10,
            allowedValues=_SDMINDICES_
            )

        self.period = self.addLiteralInput(
            identifier="period",
            title="Reference period",
            abstract="Reference period for climate conditions\
            (all = entire timeseries)",
            default="all",
            type=type(''),
            minOccurs=1,
            maxOccurs=1,
            allowedValues=['all', '1951-1980', '1961-1990',
                           '1971-2000', '1981-2010']
            )

        self.archive_format = self.addLiteralInput(
            identifier="archive_format",
            title="Archive format",
            abstract="Result files will be compressed into archives.\
                Choose an appropriate format",
            default="tar",
            type=type(''),
            minOccurs=1,
            maxOccurs=1,
            allowedValues=['zip', 'tar']
            )

        ###########
        # OUTPUTS
        ###########

        self.output_csv = self.addComplexOutput(
            identifier="output_csv",
            title="Tree species table",
            abstract="Extracted CSV file containing the tree species table",
            formats=[{"mimeType": "text/csv"}],
            asReference=True,
            )

        self.output_gbif = self.addComplexOutput(
            identifier="output_gbif",
            title="Graphic of GBIF coordinates",
            abstract="PNG graphic file showing the presence of tree species\
             according to the CSV file",
            formats=[{"mimeType": "image/png"}],
            asReference=True,
            )

        self.output_PA = self.addComplexOutput(
            identifier="output_PA",
            title="Graphic of PA mask",
            abstract="PNG graphic file showing PA mask generated based on\
             netCDF spatial increment",
            formats=[{"mimeType": "image/png"}],
            asReference=True,
            )

        self.output_indices = self.addComplexOutput(
            identifier="output_indices",
            title="Climate indices for growth conditions over all timesteps",
            abstract="Archive (tar/zip) containing calculated climate indices",
            formats=[{"mimeType": "application/x-tar"},
                     {"mimeType": "application/zip"}],
            asReference=True,
            )

        self.output_reference = self.addComplexOutput(
            identifier="output_reference",
            title="Climate indices for growth conditions of reference period",
            abstract="Archive (tar/zip) containing calculated climate indices",
            formats=[{"mimeType": "application/x-tar"},
                     {"mimeType": "application/zip"}],
            asReference=True,
            )

        self.output_prediction = self.addComplexOutput(
            identifier="output_prediction",
            title="predicted growth conditions",
            abstract="Archive containing files of the predicted\
             growth conditions",
            formats=[{"mimeType": "application/x-tar"},
                     {"mimeType": "application/zip"}],
            asReference=True,
            )

        self.output_info = self.addComplexOutput(
            identifier="output_info",
            title="GAM statistics information",
            abstract="Graphics and information of the learning statistics",
            formats=[{"mimeType": "image/png"}],
            asReference=True,
            )

    def execute(self):
        from os.path import basename
        from flyingpigeon import sdm
        from flyingpigeon.utils import archive
        self.status.set('Start process', 0)

        try:
            logger.info('reading the arguments')
            resources = self.getInputValues(identifier='resources')
            csv_file = self.getInputValues(identifier='gbif')[0]
            period = self.getInputValues(identifier='period')
            period = period[0]
            indices = self.getInputValues(identifier='input_indices')
            archive_format = self.archive_format.getValue()
            logger.info('indices %s ' % indices)
        except Exception as e:
            logger.error('failed to read in the arguments %s ' % e)

        try:
            self.status.set('read in latlon coordinates', 10)
            latlon = sdm.latlon_gbifcsv(csv_file)
        except Exception as e:
            logger.exception('failed to extract the latlon points')

        try:
            self.status.set('plot map', 80)
            from flyingpigeon.visualisation import map_gbifoccurrences

            # latlon = sdm.latlon_gbifdic(gbifdic)
            occurence_map = map_gbifoccurrences(latlon)
        except Exception as e:
            logger.exception('failed to plot occurence map %s' % e)

        try:
            self.status.set('generating the PA mask', 20)
            PAmask = sdm.get_PAmask(coordinates=latlon, domain='EUR-11')
            logger.info('PA mask sucessfully generated')
        except Exception as e:
            logger.exception('failed to generate the PA mask: %s' % e)

        try:
            self.status.set('Ploting PA mask', 25)
            from flyingpigeon.visualisation import map_PAmask
            PAmask_png = map_PAmask(PAmask)
        except Exception as e:
            logger.exception('failed to plot the PA mask: %s' % e)

        #################################
        # calculate the climate indices
        #################################

        # get the indices
        ncs_indices = None
        try:
            self.status.set('start calculation of climate indices for %s'
                            % indices, 30)
            ncs_indices = sdm.get_indices(resources=resources, indices=indices)
            logger.info('indice calculation done')
        except:
            msg = 'failed to calculate indices'
            logger.exception(msg)
            raise Exception(msg)

        try:
            archive_indices = archive(ncs_indices, format=archive_format)
            logger.info('indices 3D added to tarfile')
        except:
            msg = 'failed adding indices to tar'
            logger.exception(msg)
            raise Exception(msg)

        indices_dic = None
        try:
            # sort indices
            indices_dic = sdm.sort_indices(ncs_indices)
            logger.info('indice files sorted for %s Datasets' %
                        len(indices_dic.keys()))
        except:
            msg = 'failed to sort indices'
            logger.exception(msg)
            raise Exception(msg)

        ncs_references = []
        species_files = []
        statistics_info = []
        for count, key in enumerate(indices_dic.keys()):
            try:
                self.status.set('Start processing of %s ' % key, 40 + count*10)
                ncs = indices_dic[key]
                logger.info('with %s files' % len(ncs))
                try:
                    ncs_references.extend(sdm.get_reference(ncs_indices=ncs,
                                                            period=period))
                    logger.info('reference indice calculated %s '
                                % ncs_references)
                except:
                    msg = 'failed adding ref indices to tar'
                    logger.exception(msg)
                raise Exception(msg)
            except:
                msg = 'failed to calculate reference indices.'
                logger.exception(msg)
                raise Exception(msg)

        archive_references = None
        try:
            archive_references = archive(ncs_references, format=archive_format)
            logger.info('indices 2D added to archive')
        except:
            msg = 'failed adding 2D indices to archive'
            logger.exception(msg)
            raise Exception(msg)

        archive_predicion = None
        try:
            archive_predicion = archive(species_files, format=archive_format)
            logger.info('species_files added to archive')
        except:
            msg = 'failed adding species_files indices to archive'
            logger.exception(msg)
            raise Exception(msg)

        try:
            gam_model, predict_gam, gam_info = sdm.get_gam(ncs_references,
                                                           PAmask)
            statistics_info.append(gam_info)
            self.status.set('GAM sucessfully trained', 70)
        except:
            msg = 'failed to train GAM'
            logger.exception(msg)
            raise Exception(msg)

        try:
            prediction = sdm.get_prediction(gam_model, ncs_indices)
            self.status.set('prediction done', 80)
        except:
            msg = 'failed to predict'
            logger.exception(msg)
            raise Exception(msg)

        try:
            from numpy import invert, isnan, nan, broadcast_arrays, array,\
             zeros, linspace, meshgrid
            mask = invert(isnan(PAmask))
            mask = broadcast_arrays(prediction, mask)[1]
            prediction[mask==False] = nan
            self.status.set('land sea mask for predicted data', 90)
        except:
            logger.exception('failed to mask predicted data')

        try:
            species_files.append(sdm.write_to_file(ncs_indices[0], prediction))
            logger.info('Favourabillity written to file')
        except:
            msg = 'failed to write species file'
            logger.exception(msg)
            raise Exception(msg)

        from flyingpigeon.visualisation import concat_images
        statistics_infos = None
        try:
            statistics_infos = concat_images(statistics_info, orientation='v')
        except:
            msg = 'failed to concat images'
            logger.exception(msg)
            raise Exception(msg)

        self.output_csv.setValue(csv_file)
        self.output_gbif.setValue(occurence_map)
        self.output_PA.setValue(PAmask_png)
        self.output_indices.setValue(archive_indices)
        self.output_reference.setValue(archive_references)
        self.output_prediction.setValue(archive_predicion)
        self.output_info.setValue(statistics_infos)

        self.status.set('done', 100)
<|endoftext|>