<commit_before>import datetime
from django.db import models
from django.conf import settings
from django.db.models import signals
from django.db.models.query import QuerySet
from django.contrib.auth.models import User
from django.utils.translation import ugettext_lazy as _
from profiles.models import Profile
from django.db.models import F, Q
from django.db.models import Avg, Max, Min, Count

class MessageManager(models.Manager):

    def inbox_for(self, user, read=None, only_unreplied=None):
        """
        Returns all messages that were received by the given user and are not
        marked as deleted.
        """
        inbox = self.filter(
            user=user,
            deleted_at__isnull=True,
        )

        # this makes sure that a just created message does not show up in inbox of the same user
        inbox = inbox.exclude(Q(thread__creator=user)&Q(thread__replied=False))

        if read != None:
            if read == True:
                # read messages have read_at set to a later value then last message of the thread
                inbox = inbox.exclude(read_at__isnull=True)\
                            .filter(read_at__gt=F("thread__latest_msg__sent_at"))
            else:
                # unread threads are the ones that either have not been read at all or before the last message arrived
                inbox = inbox.filter(Q(read_at__isnull=True)
                                    |Q(read_at__lt=F("thread__latest_msg__sent_at")))

        if unreplied != None:
            if unreplied == True:
                inbox = inbox.filter(Q(replied_at__isnull=True)
                                    |Q(replied_at__lt=F("thread__latest_msg__sent_at")))

        return inbox

    def outbox_for(self, user):
        """
        Returns all messages that were sent by the given user and are not
        marked as deleted.
        """
        return self.filter(
            user=user,
            replied_at__isnull=False,
            deleted_at__isnull=True,
        )

    def trash_for(self, user):
        """
        Returns all messages that were either received or sent by the given
        user and are marked as deleted.
        """
        return self.filter(
            user=user,
            deleted_at__isnull=False,
        )

class Message(models.Model):
    """
    A private message from user to user
    """
    body = models.TextField(_("body"))
    sender = models.ForeignKey(User, related_name='sent_messages', blank=True, null=True, verbose_name=_("sender"))
    parent_msg = models.ForeignKey('self', related_name='next_messages', blank=True, null=True, verbose_name=_("parent message"))
    sent_at = models.DateTimeField(_("sent at"), auto_now_add=True)

    def __unicode__(self):
        return "%s - %s" % (str(self.sender), self.sent_at)

    def save(self, **kwargs):
        if not self.id:
            self.sent_at = datetime.datetime.now()
        super(Message, self).save(**kwargs)

    class Meta:
        ordering = ['-sent_at']
        verbose_name = _("Message")
        verbose_name_plural = _("Messages")

class Thread(models.Model):
    """
    A linear conversation between two or more Users
    """
    subject = models.CharField(_("Subject"), max_length=120)
    latest_msg = models.ForeignKey(Message, related_name='thread_latest', verbose_name=_("Latest message"))
    all_msgs = models.ManyToManyField(Message, related_name='thread', verbose_name=_("Messages"))
    # the following fields are used to filter out messages that have not been replied to in the inbox
    creator = models.ForeignKey(User, related_name='created_threads', verbose_name=_("creator"))
    replied = models.BooleanField(editable=False, default=False)

    def __unicode__(self):
        return self.subject

    def get_absolute_url(self):
        return ('messages_detail', [self.id])
    get_absolute_url = models.permalink(get_absolute_url)

    class Meta:
        ordering = ['latest_msg']
        verbose_name = _("Thread")
        verbose_name_plural = _("Threads")

class Participant(models.Model):
    """
    Thread manager for each participant
    """
    thread = models.ForeignKey(Thread, related_name='participants', verbose_name=_("message thread"))
    user = models.ForeignKey(User, related_name='threads', verbose_name=_("participant users"))
    read_at = models.DateTimeField(_("read at"), null=True, blank=True)
    replied_at = models.DateTimeField(_("replied at"), null=True, blank=True)
    deleted_at = models.DateTimeField(_("deleted at"), null=True, blank=True)

    objects = MessageManager()

    def new(self):
        """returns whether the recipient has read the message or not"""
        if self.read_at is None or self.read_at < self.thread.latest_msg.sent_at:
            return True
        return False

    def replied(self):
        """returns whether the recipient has replied the message or not"""
        if self.replied_at is None \
            or self.replied_at < self.thread.latest_msg.sent_at:
            return True
        return False

    def others(self):
        """returns the other participants of the thread"""
        return self.thread.participants.exclude(user=self.user)

    def __unicode__(self):
        return "%s - %s" % (str(self.user), self.thread.subject)

    class Meta:
        ordering = ['thread']
        verbose_name = _("participant")
        verbose_name_plural = _("participants")

def inbox_count_for(user):
    """
    returns the number of unread messages for the given user but does not
    mark them seen
    """
    return Participant.objects.filter(user=user, read_at__isnull=True, deleted_at__isnull=True).count()

from threaded_messages.utils import new_message_email
signals.post_save.connect(new_message_email, sender=Message)
<commit_msg>small fix<commit_after>import datetime
from django.db import models
from django.conf import settings
from django.db.models import signals
from django.db.models.query import QuerySet
from django.contrib.auth.models import User
from django.utils.translation import ugettext_lazy as _
from profiles.models import Profile
from django.db.models import F, Q
from django.db.models import Avg, Max, Min, Count

class MessageManager(models.Manager):

    def inbox_for(self, user, read=None, only_unreplied=None):
        """
        Returns all messages that were received by the given user and are not
        marked as deleted.
        """
        inbox = self.filter(
            user=user,
            deleted_at__isnull=True,
        )

        # this makes sure that a just created message does not show up in inbox of the same user
        inbox = inbox.exclude(Q(thread__creator=user)&Q(thread__replied=False))

        if read != None:
            if read == True:
                # read messages have read_at set to a later value then last message of the thread
                inbox = inbox.exclude(read_at__isnull=True)\
                            .filter(read_at__gt=F("thread__latest_msg__sent_at"))
            else:
                # unread threads are the ones that either have not been read at all or before the last message arrived
                inbox = inbox.filter(Q(read_at__isnull=True)
                                    |Q(read_at__lt=F("thread__latest_msg__sent_at")))

        if only_unreplied != None:
            if only_unreplied == True:
                inbox = inbox.filter(Q(replied_at__isnull=True)
                                    |Q(replied_at__lt=F("thread__latest_msg__sent_at")))

        return inbox

    def outbox_for(self, user):
        """
        Returns all messages that were sent by the given user and are not
        marked as deleted.
        """
        return self.filter(
            user=user,
            replied_at__isnull=False,
            deleted_at__isnull=True,
        )

    def trash_for(self, user):
        """
        Returns all messages that were either received or sent by the given
        user and are marked as deleted.
        """
        return self.filter(
            user=user,
            deleted_at__isnull=False,
        )

class Message(models.Model):
    """
    A private message from user to user
    """
    body = models.TextField(_("body"))
    sender = models.ForeignKey(User, related_name='sent_messages', blank=True, null=True, verbose_name=_("sender"))
    parent_msg = models.ForeignKey('self', related_name='next_messages', blank=True, null=True, verbose_name=_("parent message"))
    sent_at = models.DateTimeField(_("sent at"), auto_now_add=True)

    def __unicode__(self):
        return "%s - %s" % (str(self.sender), self.sent_at)

    def save(self, **kwargs):
        if not self.id:
            self.sent_at = datetime.datetime.now()
        super(Message, self).save(**kwargs)

    class Meta:
        ordering = ['-sent_at']
        verbose_name = _("Message")
        verbose_name_plural = _("Messages")

class Thread(models.Model):
    """
    A linear conversation between two or more Users
    """
    subject = models.CharField(_("Subject"), max_length=120)
    latest_msg = models.ForeignKey(Message, related_name='thread_latest', verbose_name=_("Latest message"))
    all_msgs = models.ManyToManyField(Message, related_name='thread', verbose_name=_("Messages"))
    # the following fields are used to filter out messages that have not been replied to in the inbox
    creator = models.ForeignKey(User, related_name='created_threads', verbose_name=_("creator"))
    replied = models.BooleanField(editable=False, default=False)

    def __unicode__(self):
        return self.subject

    def get_absolute_url(self):
        return ('messages_detail', [self.id])
    get_absolute_url = models.permalink(get_absolute_url)

    class Meta:
        ordering = ['latest_msg']
        verbose_name = _("Thread")
        verbose_name_plural = _("Threads")

class Participant(models.Model):
    """
    Thread manager for each participant
    """
    thread = models.ForeignKey(Thread, related_name='participants', verbose_name=_("message thread"))
    user = models.ForeignKey(User, related_name='threads', verbose_name=_("participant users"))
    read_at = models.DateTimeField(_("read at"), null=True, blank=True)
    replied_at = models.DateTimeField(_("replied at"), null=True, blank=True)
    deleted_at = models.DateTimeField(_("deleted at"), null=True, blank=True)

    objects = MessageManager()

    def new(self):
        """returns whether the recipient has read the message or not"""
        if self.read_at is None or self.read_at < self.thread.latest_msg.sent_at:
            return True
        return False

    def replied(self):
        """returns whether the recipient has replied the message or not"""
        if self.replied_at is None \
            or self.replied_at < self.thread.latest_msg.sent_at:
            return True
        return False

    def others(self):
        """returns the other participants of the thread"""
        return self.thread.participants.exclude(user=self.user)

    def __unicode__(self):
        return "%s - %s" % (str(self.user), self.thread.subject)

    class Meta:
        ordering = ['thread']
        verbose_name = _("participant")
        verbose_name_plural = _("participants")

def inbox_count_for(user):
    """
    returns the number of unread messages for the given user but does not
    mark them seen
    """
    return Participant.objects.filter(user=user, read_at__isnull=True, deleted_at__isnull=True).count()

from threaded_messages.utils import new_message_email
signals.post_save.connect(new_message_email, sender=Message)
<|endoftext|>


<commit_before>__version__ = '0.5'<commit_msg>Bump to 0.6-dev<commit_after>__version__ = '0.6-dev'<|endoftext|>


<commit_before>from threeML.pluginPrototype import pluginPrototype

from threeML.models.Parameter import Parameter

from threeML.io.fileUtils import fileExistingAndReadable, sanitizeFilename

from threeML.pyModelInterface import pyToCppModelInterface

from hawc import liff_3ML

import os, sys, collections

import matplotlib.pyplot as plt
from matplotlib import gridspec

import numpy

defaultMinChannel = 0
defaultMaxChannel = 9

__instrument_name = "HAWC"

class HAWCLike( pluginPrototype ):

    def __init__( self, name, maptree, response, ntransits, **kwargs ):

        #This controls if the likeHAWC class should load the entire
        #map or just a small disc around a source (faster).
        #Default is the latter, which is way faster. LIFF will decide
        #autonomously which ROI to use depending on the source model

        self.fullsky = False

        if 'fullsky' in kwargs.keys():

            self.fullsky = bool( kwargs['fullsky'] )

        self.name = str( name )

        #Sanitize files in input (expand variables and so on)

        self.maptree = os.path.abspath( sanitizeFilename( maptree ) )

        self.response = os.path.abspath( sanitizeFilename( response ) )

        #
        self.ntransits = ntransits

        #Check that they exists and can be read

        if not fileExistingAndReadable( self.maptree ):

            raise IOError("MapTree %s does not exist or is not readable" % maptree)

        if not fileExistingAndReadable( self.response ):

            raise IOError("Response %s does not exist or is not readable" % response)

        #Post-pone the creation of the LIFF instance to when
        #we have the likelihood model

        self.instanced = False

        #Default value for minChannel and maxChannel

        self.minChannel = int( defaultMinChannel )
        self.maxChannel = int( defaultMaxChannel )

        #By default the fit of the CommonNorm is deactivated

        self.deactivateCommonNorm()

        #This is to keep track of whether the user defined a ROI or not

        self.roi_ra = None

        #Further setup

        self.__setup()

    def setROI(self, ra, dec, radius):

        self.roi_ra = ra
        self.roi_dec = dec

        self.roi_radius = radius

    def __setup(self):

        #I put this here so I can use it both from the __init__ both from
        #the __setstate__ methods

        #Create the dictionary of nuisance parameters

        self.nuisanceParameters = collections.OrderedDict()
        self.nuisanceParameters['CommonNorm'] = Parameter("CommonNorm",1.0,0.5,1.5,0.01,
                                                           fixed=True,nuisance=True)

    def __getstate__(self):

        #This method is used by pickle before attempting to pickle the class

        #Return only the objects needed to recreate the class
        #IN particular, we do NOT return the theLikeHAWC class,
        #which is not pickeable. It will instead be recreated
        #on the other side

        d = {}

        d['name']= self.name
        d['maptree'] = self.maptree
        d['response'] = self.response
        d['ntransits'] = self.ntransits
        d['model'] = self.model
        d['minChannel'] = self.minChannel
        d['maxChannel'] = self.maxChannel

        d['roi_ra'] = self.roi_ra

        if self.roi_ra is not None:

            d['roi_dec'] = self.roi_dec
            d['roi_radius'] = self.roi_radius

        return d

    def __setstate__( self, state ):

        #This is used by pickle to recreate the class on the remote
        #side
        name = state['name']
        maptree = state['maptree']
        response = state['response']
        ntransits = state['ntransits']

        #Now report the class to its state

        self.__init__( name, maptree, response, ntransits )

        if state['roi_ra'] is not None:

            self.setROI( state['roi_ra'], state['roi_dec'], state['roi_radius'] )

        self.setActiveMeasurements( state['minChannel'], state['maxChannel'] )

        self.setModel( state['model'] )

    def setActiveMeasurements( self, minChannel, maxChannel ):

        self.minChannel = int( minChannel )
        self.maxChannel = int( maxChannel )

        if self.instanced:

            sys.stderr.write("Since the plugins was already used before, the change in active measurements" +
                             "will not be effective until you create a new JointLikelihood or Bayesian" +
                             "instance")

    def setModel( self, LikelihoodModelInstance ):
        '''
        Set the model to be used in the joint minimization. Must be a LikelihoodModel instance.
        '''

        #Instance the python - C++ bridge
        self.model = LikelihoodModelInstance

        self.pymodel = pyToCppModelInterface( self.model )

        #Now init the HAWC LIFF software

        try:

            #Load all sky
            #(ROI will be defined later)

            self.theLikeHAWC = liff_3ML.LikeHAWC(self.maptree,
                                                 self.ntransits,
                                                 self.response,
                                                 self.pymodel,
                                                 self.minChannel,
                                                 self.maxChannel,
                                                 self.fullsky)

            if self.roi_ra is None and self.fullsky:

                raise RuntimeError("You have to define a ROI with the setROI method")

            if self.roi_ra is not None and self.fullsky:

                self.theLikeHAWC.SetROI( self.roi_ra, self.roi_dec, self.roi_radius )

        except:

            print("Could not instance the LikeHAWC class from LIFF. " +
                               "Check that HAWC software is working")

            raise

        else:

            self.instanced = True

        #Now set a callback in the CommonNorm parameter, so that if the user or the fit
        #engine or the Bayesian sampler change the CommonNorm value, the change will be
        #propagated to the LikeHAWC instance

        self.nuisanceParameters['CommonNorm'].setCallback( self._CommonNormCallback )

    def _CommonNormCallback( self ):

        self.theLikeHAWC.SetCommonNorm( self.nuisanceParameters['CommonNorm'].value )

    def getName(self):
        '''
        Return a name for this dataset (likely set during the constructor)
        '''
        return self.name

    def activateCommonNorm( self ):

        self.fitCommonNorm = True

    def deactivateCommonNorm( self ):

        self.fitCommonNorm = False

    def getLogLike(self):

        '''
        Return the value of the log-likelihood with the current values for the
        parameters
        '''

        self.pymodel.update()

        logL = self.theLikeHAWC.getLogLike( self.fitCommonNorm )

        return logL

    def getNuisanceParameters(self):
        '''
        Return a list of nuisance parameters. Return an empty list if there
        are no nuisance parameters
        '''

        return self.nuisanceParameters.keys()

    def innerFit(self):

        self.theLikeHAWC.SetBackgroundNormFree( self.fitCommonNorm )

        self.pymodel.update()

        logL = self.theLikeHAWC.getLogLike( self.fitCommonNorm )

        self.nuisanceParameters['CommonNorm'].setValue( self.theLikeHAWC.CommonNorm() )

        return logL

    def display( self, radius = 2.0 ):

        figs = []

        nsrc = self.model.getNumberOfPointSources()

        for srcid in range(nsrc):

            ra, dec = self.model.getPointSourcePosition( srcid )

            model = numpy.array( self.theLikeHAWC.GetTopHatExpectedExcesses( ra, dec, radius ) )

            signal = numpy.array(self.theLikeHAWC.GetTopHatExcesses( ra, dec, radius ) )

            bkg = numpy.array( self.theLikeHAWC.GetTopHatBackgrounds( ra, dec, radius ) )

            total = signal + bkg

            fig = plt.figure()

            gs = gridspec.GridSpec(2,1, height_ratios=[2,1])
            gs.update(hspace=0)

            sub = plt.subplot( gs[0] )

            nHitBins = numpy.arange( self.minChannel, self.maxChannel + 1 )

            sub.errorbar( nHitBins, total, yerr=numpy.sqrt(total),
                          capsize=0, color='black', label='Observation',
                          fmt='.')

            sub.plot( nHitBins, model + bkg, label='Model + bkg')

            plt.legend( bbox_to_anchor=(1.05, 1), loc=2, numpoints=1)

            #Residuals

            sub1 = plt.subplot( gs[1] )

            #Using model variance to account for low statistic

            resid = ( signal - model ) / model

            sub1.axhline( 0, linestyle='--' )

            sub1.errorbar( nHitBins, resid,
                           yerr=numpy.sqrt(total) / model,
                           capsize=0, fmt='.' )

            sub.set_xlim( [nHitBins.min() - 0.5, nHitBins.max() + 0.5] )

            sub.set_yscale("log", nonposy='clip')

            sub.set_ylabel("Counts per bin")

            #sub1.set_xscale("log")

            sub1.set_xlabel("Analysis bin")

            sub1.set_ylabel("(Mo. - data) / mo.")

            sub1.set_xlim( [nHitBins.min() - 0.5, nHitBins.max() + 0.5] )

            sub.set_xticks( [] )
            sub1.set_xticks( nHitBins )

            figs.append(fig)

        return figs

<commit_msg>Changed label of y axis in the display() method<commit_after>from threeML.pluginPrototype import pluginPrototype

from threeML.models.Parameter import Parameter

from threeML.io.fileUtils import fileExistingAndReadable, sanitizeFilename

from threeML.pyModelInterface import pyToCppModelInterface

from hawc import liff_3ML

import os, sys, collections

import matplotlib.pyplot as plt
from matplotlib import gridspec

import numpy

defaultMinChannel = 0
defaultMaxChannel = 9

__instrument_name = "HAWC"

class HAWCLike( pluginPrototype ):

    def __init__( self, name, maptree, response, ntransits, **kwargs ):

        #This controls if the likeHAWC class should load the entire
        #map or just a small disc around a source (faster).
        #Default is the latter, which is way faster. LIFF will decide
        #autonomously which ROI to use depending on the source model

        self.fullsky = False

        if 'fullsky' in kwargs.keys():

            self.fullsky = bool( kwargs['fullsky'] )

        self.name = str( name )

        #Sanitize files in input (expand variables and so on)

        self.maptree = os.path.abspath( sanitizeFilename( maptree ) )

        self.response = os.path.abspath( sanitizeFilename( response ) )

        #
        self.ntransits = ntransits

        #Check that they exists and can be read

        if not fileExistingAndReadable( self.maptree ):

            raise IOError("MapTree %s does not exist or is not readable" % maptree)

        if not fileExistingAndReadable( self.response ):

            raise IOError("Response %s does not exist or is not readable" % response)

        #Post-pone the creation of the LIFF instance to when
        #we have the likelihood model

        self.instanced = False

        #Default value for minChannel and maxChannel

        self.minChannel = int( defaultMinChannel )
        self.maxChannel = int( defaultMaxChannel )

        #By default the fit of the CommonNorm is deactivated

        self.deactivateCommonNorm()

        #This is to keep track of whether the user defined a ROI or not

        self.roi_ra = None

        #Further setup

        self.__setup()

    def setROI(self, ra, dec, radius):

        self.roi_ra = ra
        self.roi_dec = dec

        self.roi_radius = radius

    def __setup(self):

        #I put this here so I can use it both from the __init__ both from
        #the __setstate__ methods

        #Create the dictionary of nuisance parameters

        self.nuisanceParameters = collections.OrderedDict()
        self.nuisanceParameters['CommonNorm'] = Parameter("CommonNorm",1.0,0.5,1.5,0.01,
                                                           fixed=True,nuisance=True)

    def __getstate__(self):

        #This method is used by pickle before attempting to pickle the class

        #Return only the objects needed to recreate the class
        #IN particular, we do NOT return the theLikeHAWC class,
        #which is not pickeable. It will instead be recreated
        #on the other side

        d = {}

        d['name']= self.name
        d['maptree'] = self.maptree
        d['response'] = self.response
        d['ntransits'] = self.ntransits
        d['model'] = self.model
        d['minChannel'] = self.minChannel
        d['maxChannel'] = self.maxChannel

        d['roi_ra'] = self.roi_ra

        if self.roi_ra is not None:

            d['roi_dec'] = self.roi_dec
            d['roi_radius'] = self.roi_radius

        return d

    def __setstate__( self, state ):

        #This is used by pickle to recreate the class on the remote
        #side
        name = state['name']
        maptree = state['maptree']
        response = state['response']
        ntransits = state['ntransits']

        #Now report the class to its state

        self.__init__( name, maptree, response, ntransits )

        if state['roi_ra'] is not None:

            self.setROI( state['roi_ra'], state['roi_dec'], state['roi_radius'] )

        self.setActiveMeasurements( state['minChannel'], state['maxChannel'] )

        self.setModel( state['model'] )

    def setActiveMeasurements( self, minChannel, maxChannel ):

        self.minChannel = int( minChannel )
        self.maxChannel = int( maxChannel )

        if self.instanced:

            sys.stderr.write("Since the plugins was already used before, the change in active measurements" +
                             "will not be effective until you create a new JointLikelihood or Bayesian" +
                             "instance")

    def setModel( self, LikelihoodModelInstance ):
        '''
        Set the model to be used in the joint minimization. Must be a LikelihoodModel instance.
        '''

        #Instance the python - C++ bridge
        self.model = LikelihoodModelInstance

        self.pymodel = pyToCppModelInterface( self.model )

        #Now init the HAWC LIFF software

        try:

            #Load all sky
            #(ROI will be defined later)

            self.theLikeHAWC = liff_3ML.LikeHAWC(self.maptree,
                                                 self.ntransits,
                                                 self.response,
                                                 self.pymodel,
                                                 self.minChannel,
                                                 self.maxChannel,
                                                 self.fullsky)

            if self.roi_ra is None and self.fullsky:

                raise RuntimeError("You have to define a ROI with the setROI method")

            if self.roi_ra is not None and self.fullsky:

                self.theLikeHAWC.SetROI( self.roi_ra, self.roi_dec, self.roi_radius )

        except:

            print("Could not instance the LikeHAWC class from LIFF. " +
                               "Check that HAWC software is working")

            raise

        else:

            self.instanced = True

        #Now set a callback in the CommonNorm parameter, so that if the user or the fit
        #engine or the Bayesian sampler change the CommonNorm value, the change will be
        #propagated to the LikeHAWC instance

        self.nuisanceParameters['CommonNorm'].setCallback( self._CommonNormCallback )

    def _CommonNormCallback( self ):

        self.theLikeHAWC.SetCommonNorm( self.nuisanceParameters['CommonNorm'].value )

    def getName(self):
        '''
        Return a name for this dataset (likely set during the constructor)
        '''
        return self.name

    def activateCommonNorm( self ):

        self.fitCommonNorm = True

    def deactivateCommonNorm( self ):

        self.fitCommonNorm = False

    def getLogLike(self):

        '''
        Return the value of the log-likelihood with the current values for the
        parameters
        '''

        self.pymodel.update()

        logL = self.theLikeHAWC.getLogLike( self.fitCommonNorm )

        return logL

    def getNuisanceParameters(self):
        '''
        Return a list of nuisance parameters. Return an empty list if there
        are no nuisance parameters
        '''

        return self.nuisanceParameters.keys()

    def innerFit(self):

        self.theLikeHAWC.SetBackgroundNormFree( self.fitCommonNorm )

        self.pymodel.update()

        logL = self.theLikeHAWC.getLogLike( self.fitCommonNorm )

        self.nuisanceParameters['CommonNorm'].setValue( self.theLikeHAWC.CommonNorm() )

        return logL

    def display( self, radius = 2.0 ):

        figs = []

        nsrc = self.model.getNumberOfPointSources()

        for srcid in range(nsrc):

            ra, dec = self.model.getPointSourcePosition( srcid )

            model = numpy.array( self.theLikeHAWC.GetTopHatExpectedExcesses( ra, dec, radius ) )

            signal = numpy.array(self.theLikeHAWC.GetTopHatExcesses( ra, dec, radius ) )

            bkg = numpy.array( self.theLikeHAWC.GetTopHatBackgrounds( ra, dec, radius ) )

            total = signal + bkg

            fig = plt.figure()

            gs = gridspec.GridSpec(2,1, height_ratios=[2,1])
            gs.update(hspace=0)

            sub = plt.subplot( gs[0] )

            nHitBins = numpy.arange( self.minChannel, self.maxChannel + 1 )

            sub.errorbar( nHitBins, total, yerr=numpy.sqrt(total),
                          capsize=0, color='black', label='Observation',
                          fmt='.')

            sub.plot( nHitBins, model + bkg, label='Model + bkg')

            plt.legend( bbox_to_anchor=(1.05, 1), loc=2, numpoints=1)

            #Residuals

            sub1 = plt.subplot( gs[1] )

            #Using model variance to account for low statistic

            resid = ( signal - model ) / model

            sub1.axhline( 0, linestyle='--' )

            sub1.errorbar( nHitBins, resid,
                           yerr=numpy.sqrt(total) / model,
                           capsize=0, fmt='.' )

            sub.set_xlim( [nHitBins.min() - 0.5, nHitBins.max() + 0.5] )

            sub.set_yscale("log", nonposy='clip')

            sub.set_ylabel("Counts per bin")

            #sub1.set_xscale("log")

            sub1.set_xlabel("Analysis bin")

            sub1.set_ylabel(r"$\frac{excess - mod.}{mod.}$",fontsize=20)

            sub1.set_xlim( [nHitBins.min() - 0.5, nHitBins.max() + 0.5] )

            sub.set_xticks( [] )
            sub1.set_xticks( nHitBins )

            figs.append(fig)

        return figs

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright (c) 2014 Parisson SARL
# Copyright (c) 2014 Thomas Fillon <thomas@parisson.com>

# This file is part of TimeSide.

# TimeSide is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 2 of the License, or
# (at your option) any later version.

# TimeSide is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with TimeSide.  If not, see <http://www.gnu.org/licenses/>.
#
# Author : Thomas Fillon <thomas@parisson.com>

import tables
from tempfile import NamedTemporaryFile

class BufferTable(object):
    def __init__(self, array_names=None):
        self._tempfile = NamedTemporaryFile(mode='w', suffix='.h5',
                                            prefix='ts_buf_',
                                            delete=True)
        self.fileh = tables.openFile(self._tempfile.name, mode='w')

        if not array_names:
            array_names = []
        if isinstance(array_names, list):
            self.array_names = array_names
        else:
            self.array_names = [array_names]
        for name in array_names:
            if not isinstance(name, basestring):
                raise(TypeError, 'String argument require in array_names')

    def __getitem__(self, name):
        return self.fileh.root.__getattr__(name)

    #def __set_item__(self, name, value):
    #    self.fileh.root.__setattr__(name, value)

    def append(self, name, new_array):
        try:
            self.fileh.root.__getattr__(name).append([new_array])
        except tables.exceptions.NoSuchNodeError:
            if name not in self.array_names:
                self.array_names.append(name)
             # The following is compatible with pytables 3 only
             #self.fileh.create_earray(where=self.fileh.root,
             #                         name=name,
             #                         obj=[new_array])
            atom = tables.Atom.from_dtype(new_array.dtype)
            if len(new_array.shape) > 1:
                shape = (0, new_array.shape[1])
            else:
                shape = (0,)
            self.fileh.createEArray(where=self.fileh.root,
                                    name=name,
                                    atom=atom,
                                    shape=shape)
            self.append(name, new_array)

    def close(self):
        for name in self.array_names:
            self.fileh.removeNode(self.fileh.root, name)
        self.fileh.close()
        self._tempfile.close()
<commit_msg>fix(tools.buffering): allow proper array concatenation in BufferTable<commit_after># -*- coding: utf-8 -*-

# Copyright (c) 2014 Parisson SARL
# Copyright (c) 2014 Thomas Fillon <thomas@parisson.com>

# This file is part of TimeSide.

# TimeSide is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 2 of the License, or
# (at your option) any later version.

# TimeSide is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with TimeSide.  If not, see <http://www.gnu.org/licenses/>.
#
# Author : Thomas Fillon <thomas@parisson.com>

import tables
from tempfile import NamedTemporaryFile
import numpy as np

class BufferTable(object):
    def __init__(self, array_names=None):
        self._tempfile = NamedTemporaryFile(mode='w', suffix='.h5',
                                            prefix='ts_buf_',
                                            delete=True)
        self.fileh = tables.openFile(self._tempfile.name, mode='w')

        if not array_names:
            array_names = []
        if isinstance(array_names, list):
            self.array_names = array_names
        else:
            self.array_names = [array_names]
        for name in array_names:
            if not isinstance(name, basestring):
                raise(TypeError, 'String argument require in array_names')

    def __getitem__(self, name):
        return self.fileh.root.__getattr__(name)

    #def __set_item__(self, name, value):
    #    self.fileh.root.__setattr__(name, value)

    def append(self, name, new_array):
        try:
            if new_array.shape:
                self.fileh.root.__getattr__(name).append(new_array[np.newaxis,
                                                                   :])
            else:
                self.fileh.root.__getattr__(name).append([new_array])
        except tables.exceptions.NoSuchNodeError:
            if name not in self.array_names:
                self.array_names.append(name)
             # The following is compatible with pytables 3 only
             #self.fileh.create_earray(where=self.fileh.root,
             #                         name=name,
             #                         obj=[new_array])
            atom = tables.Atom.from_dtype(new_array.dtype)
            dim_list = [0]
            dim_list.extend([dim for dim in new_array.shape])
            shape = tuple(dim_list)

            self.fileh.createEArray(where=self.fileh.root,
                                    name=name,
                                    atom=atom,
                                    shape=shape)
            self.append(name, new_array)

    def close(self):
        for name in self.array_names:
            self.fileh.removeNode(self.fileh.root, name)
        self.fileh.close()
        self._tempfile.close()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    tipfyext.jinja2
    ~~~~~~~~~~~~~~~

    Jinja2 template support for Tipfy.

    Learn more about Jinja2 at http://jinja.pocoo.org/2/

    :copyright: 2010 by tipfy.org.
    :license: BSD, see LICENSE.txt for more details.
"""
from jinja2 import Environment, FileSystemLoader, ModuleLoader

from werkzeug import cached_property, import_string

from tipfy.utils import url_for

#: Default configuration values for this module. Keys are:
#:
#: templates_dir
#:     Directory for templates. Default is `templates`.
#:
#: templates_compiled_target
#:     Target for compiled templates. If set, uses the loader for compiled
#:     templates in production. If it ends with a '.zip' it will be treated
#:     as a zip file. Default is None.
#:
#: force_use_compiled
#:     Forces the use of compiled templates even in the development server.
#:
#: environment_args
#:     Keyword arguments used to instantiate the Jinja2 environment. By
#:     default autoescaping is enabled and two extensions are set:
#:     'jinja2.ext.autoescape' and 'jinja2.ext.with_'. For production it may
#:     be a godd idea to set 'auto_reload' to False -- we don't need to check
#:     if templates changed after deployed.
#:
#: after_environment_created
#:     A function called after the environment is created. Can also be defined
#:     as a string to be imported dynamically. Use this to set extra filters,
#:     global variables, extensions etc. It is called passing the environment
#:     as argument.
default_config = {
    'templates_dir': 'templates',
    'templates_compiled_target': None,
    'force_use_compiled': False,
    'environment_args': {
        'autoescape': True,
        'extensions': ['jinja2.ext.autoescape', 'jinja2.ext.with_'],
    },
    'after_environment_created': None,
}

class Jinja2Mixin(object):
    """Mixin that adds ``render_template`` and ``render_response`` methods
    to a :class:`tipfy.RequestHandler`. It will use the request context to
    render templates.
    """
    @cached_property
    def jinja2(self):
        return Jinja2.factory(self.app, 'jinja2')

    def render_template(self, _filename, **context):
        return self.jinja2.render_template(self, _filename, **context)

    def render_response(self, _filename, **context):
        return self.jinja2.render_response(self, _filename, **context)

class Jinja2(object):
    def __init__(self, app, _globals=None, filters=None):
        self.app = app
        config = app.config[__name__]
        kwargs = config['environment_args'].copy()
        enable_i18n = 'jinja2.ext.i18n' in kwargs.get('extensions', [])

        if not kwargs.get('loader'):
            templates_compiled_target = config['templates_compiled_target']
            use_compiled = not app.debug or config['force_use_compiled']

            if templates_compiled_target and use_compiled:
                # Use precompiled templates loaded from a module or zip.
                kwargs['loader'] = ModuleLoader(templates_compiled_target)
            else:
                # Parse templates for every new environment instances.
                kwargs['loader'] = FileSystemLoader(config['templates_dir'])

        # Initialize the environment.
        env = Environment(**kwargs)

        if _globals:
            env.globals.update(_globals)

        if filters:
            env.filters.update(filters)

        if enable_i18n:
            # Install i18n.
            from tipfy import i18n
            env.install_gettext_callables(i18n.gettext, i18n.ngettext,
                newstyle=True)
            env.filters.update({
                'format_date':      i18n.format_date,
                'format_time':      i18n.format_time,
                'format_datetime':  i18n.format_datetime,
                'format_timedelta': i18n.format_timedelta,
            })

        env.globals['url_for'] = url_for

        after_creation_func = config['after_environment_created']
        if after_creation_func:
            if isinstance(after_creation_func, basestring):
                after_creation_func = import_string(after_creation_func)

            after_creation_func(env)

        self.environment = env

    def render(self, _filename, **context):
        """Renders a template and returns a response object.

        :param _filename:
            The template filename, related to the templates directory.
        :param context:
            Keyword arguments used as variables in the rendered template.
            These will override values set in the request context.
       :returns:
            A rendered template.
        """
        return self.environment.get_template(_filename).render(**context)

    def render_template(self, _handler, _filename, **context):
        """Renders a template and returns a response object.

        :param _filename:
            The template filename, related to the templates directory.
        :param context:
            Keyword arguments used as variables in the rendered template.
            These will override values set in the request context.
       :returns:
            A rendered template.
        """
        ctx = _handler.context.copy()
        ctx.update(context)
        return self.render(_filename, **ctx)

    def render_response(self, _handler, _filename, **context):
        """Returns a response object with a rendered template.

        :param _filename:
            The template filename, related to the templates directory.
        :param context:
            Keyword arguments used as variables in the rendered template.
            These will override values set in the request context.
        """
        res = self.render_template(_handler, _filename, **context)
        return self.app.response_class(res)

    def get_template_attribute(self, filename, attribute):
        """Loads a macro (or variable) a template exports.  This can be used to
        invoke a macro from within Python code.  If you for example have a
        template named `_foo.html` with the following contents:

        .. sourcecode:: html+jinja

           {% macro hello(name) %}Hello {{ name }}!{% endmacro %}

        You can access this from Python code like this::

            hello = get_template_attribute('_foo.html', 'hello')
            return hello('World')

        This function is borrowed from `Flask`.

        :param filename:
            The template filename.
        :param attribute:
            The name of the variable of macro to acccess.
        """
        template = self.environment.get_template(filename)
        return getattr(template.module, attribute)

    @classmethod
    def factory(cls, _app, _name, **kwargs):
        if _name not in _app.registry:
            _app.registry[_name] = cls(_app, **kwargs)

        return _app.registry[_name]
<commit_msg>Added a jinja2_class attribute to Jinja2Mixin.<commit_after># -*- coding: utf-8 -*-
"""
    tipfyext.jinja2
    ~~~~~~~~~~~~~~~

    Jinja2 template support for Tipfy.

    Learn more about Jinja2 at http://jinja.pocoo.org/2/

    :copyright: 2010 by tipfy.org.
    :license: BSD, see LICENSE.txt for more details.
"""
from jinja2 import Environment, FileSystemLoader, ModuleLoader

from werkzeug import cached_property, import_string

from tipfy.utils import url_for

#: Default configuration values for this module. Keys are:
#:
#: templates_dir
#:     Directory for templates. Default is `templates`.
#:
#: templates_compiled_target
#:     Target for compiled templates. If set, uses the loader for compiled
#:     templates in production. If it ends with a '.zip' it will be treated
#:     as a zip file. Default is None.
#:
#: force_use_compiled
#:     Forces the use of compiled templates even in the development server.
#:
#: environment_args
#:     Keyword arguments used to instantiate the Jinja2 environment. By
#:     default autoescaping is enabled and two extensions are set:
#:     'jinja2.ext.autoescape' and 'jinja2.ext.with_'. For production it may
#:     be a godd idea to set 'auto_reload' to False -- we don't need to check
#:     if templates changed after deployed.
#:
#: after_environment_created
#:     A function called after the environment is created. Can also be defined
#:     as a string to be imported dynamically. Use this to set extra filters,
#:     global variables, extensions etc. It is called passing the environment
#:     as argument.
default_config = {
    'templates_dir': 'templates',
    'templates_compiled_target': None,
    'force_use_compiled': False,
    'environment_args': {
        'autoescape': True,
        'extensions': ['jinja2.ext.autoescape', 'jinja2.ext.with_'],
    },
    'after_environment_created': None,
}

class Jinja2(object):
    def __init__(self, app, _globals=None, filters=None):
        self.app = app
        config = app.config[__name__]
        kwargs = config['environment_args'].copy()
        enable_i18n = 'jinja2.ext.i18n' in kwargs.get('extensions', [])

        if not kwargs.get('loader'):
            templates_compiled_target = config['templates_compiled_target']
            use_compiled = not app.debug or config['force_use_compiled']

            if templates_compiled_target and use_compiled:
                # Use precompiled templates loaded from a module or zip.
                kwargs['loader'] = ModuleLoader(templates_compiled_target)
            else:
                # Parse templates for every new environment instances.
                kwargs['loader'] = FileSystemLoader(config['templates_dir'])

        # Initialize the environment.
        env = Environment(**kwargs)

        if _globals:
            env.globals.update(_globals)

        if filters:
            env.filters.update(filters)

        if enable_i18n:
            # Install i18n.
            from tipfy import i18n
            env.install_gettext_callables(i18n.gettext, i18n.ngettext,
                newstyle=True)
            env.filters.update({
                'format_date':      i18n.format_date,
                'format_time':      i18n.format_time,
                'format_datetime':  i18n.format_datetime,
                'format_timedelta': i18n.format_timedelta,
            })

        env.globals['url_for'] = url_for

        after_creation_func = config['after_environment_created']
        if after_creation_func:
            if isinstance(after_creation_func, basestring):
                after_creation_func = import_string(after_creation_func)

            after_creation_func(env)

        self.environment = env

    def render(self, _filename, **context):
        """Renders a template and returns a response object.

        :param _filename:
            The template filename, related to the templates directory.
        :param context:
            Keyword arguments used as variables in the rendered template.
            These will override values set in the request context.
       :returns:
            A rendered template.
        """
        return self.environment.get_template(_filename).render(**context)

    def render_template(self, _handler, _filename, **context):
        """Renders a template and returns a response object.

        :param _filename:
            The template filename, related to the templates directory.
        :param context:
            Keyword arguments used as variables in the rendered template.
            These will override values set in the request context.
       :returns:
            A rendered template.
        """
        ctx = _handler.context.copy()
        ctx.update(context)
        return self.render(_filename, **ctx)

    def render_response(self, _handler, _filename, **context):
        """Returns a response object with a rendered template.

        :param _filename:
            The template filename, related to the templates directory.
        :param context:
            Keyword arguments used as variables in the rendered template.
            These will override values set in the request context.
        """
        res = self.render_template(_handler, _filename, **context)
        return self.app.response_class(res)

    def get_template_attribute(self, filename, attribute):
        """Loads a macro (or variable) a template exports.  This can be used to
        invoke a macro from within Python code.  If you for example have a
        template named `_foo.html` with the following contents:

        .. sourcecode:: html+jinja

           {% macro hello(name) %}Hello {{ name }}!{% endmacro %}

        You can access this from Python code like this::

            hello = get_template_attribute('_foo.html', 'hello')
            return hello('World')

        This function is borrowed from `Flask`.

        :param filename:
            The template filename.
        :param attribute:
            The name of the variable of macro to acccess.
        """
        template = self.environment.get_template(filename)
        return getattr(template.module, attribute)

    @classmethod
    def factory(cls, _app, _name, **kwargs):
        if _name not in _app.registry:
            _app.registry[_name] = cls(_app, **kwargs)

        return _app.registry[_name]

class Jinja2Mixin(object):
    """Mixin that adds ``render_template`` and ``render_response`` methods
    to a :class:`tipfy.RequestHandler`. It will use the request context to
    render templates.
    """
    # The Jinja2 creator.
    jinja2_class = Jinja2

    @cached_property
    def jinja2(self):
        return self.jinja2_class.factory(self.app, 'jinja2')

    def render_template(self, _filename, **context):
        return self.jinja2.render_template(self, _filename, **context)

    def render_response(self, _filename, **context):
        return self.jinja2.render_response(self, _filename, **context)
<|endoftext|>


<commit_before># TmLibrary - TissueMAPS library for distibuted image analysis routines.
# Copyright (C) 2016  Markus D. Herrmann, University of Zurich and Robin Hafen
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
import logging
from collections import defaultdict

import tmlib.models as tm
from tmlib.utils import notimplemented
from tmlib.utils import same_docstring_as
from tmlib.errors import NotSupportedError
from sqlalchemy.orm.exc import NoResultFound
from tmlib.errors import JobDescriptionError
from tmlib.workflow.align import registration as reg
from tmlib.workflow.api import ClusterRoutines
from tmlib.workflow import register_step_api

logger = logging.getLogger(__name__)

@register_step_api('align')
class ImageRegistrator(ClusterRoutines):

    '''Class for registeration and alignment of images between multiplexing
    cycles.
    '''

    def __init__(self, experiment_id, verbosity):
        '''
        Parameters
        ----------
        experiment_id: int
            ID of the processed experiment
        verbosity: int
            logging level
        '''
        super(ImageRegistrator, self).__init__(experiment_id, verbosity)

    def create_batches(self, args):
        '''Creates job descriptions for parallel computing.

        Parameters
        ----------
        args: tmlib.align.args.AlignInitArgs
            step-specific arguments

        Returns
        -------
        Dict[str, List[dict] or dict]
            job descriptions

        Raises
        ------
        tmlib.errors.NotSupportedError
            when a plate contains only one cycle
        ValueError
            when `args.ref_wavelength` does not exist across all cycles
        '''
        job_count = 0
        job_descriptions = dict()
        job_descriptions['run'] = list()

        with tm.utils.ExperimentSession(self.experiment_id) as session:

            cycles = session.query(tm.Cycle).all()

            if not(len(cycles) > 1):
                raise NotSupportedError(
                    'Alignment requires more than one cycle.'
                )

            if args.ref_cycle >= len(cycles):
                raise JobDescriptionError(
                    'Cycle index must not exceed total number of cycles.'
                )

            site_ids = session.query(tm.Site.id).\
                order_by(tm.Site.id).\
                all()

            batches = self._create_batches(site_ids, args.batch_size)
            for batch in batches:

                job_count += 1
                input_ids = {
                    'reference_file_ids': list(),
                    'target_file_ids': defaultdict(list)
                }

                for cycle in cycles:

                    n = session.query(tm.ChannelImageFile.id).\
                        join(tm.Cycle).\
                        join(tm.Channel).\
                        filter(tm.Cycle.id == cycle.id).\
                        filter(tm.Channel.wavelength == args.ref_wavelength).\
                        filter(~tm.Site.omitted).\
                        count()

                    if n == 0:
                        raise ValueError(
                            'No image files found for cycle %d and '
                            'wavelength "%s"'
                            % (cycle.id, args.ref_wavelength)
                        )

                    for s in batch:

                        files = session.query(tm.ChannelImageFile.id).\
                            join(tm.Site).\
                            join(tm.Cycle).\
                            join(tm.Channel).\
                            filter(tm.Site.id == s).\
                            filter(tm.Cycle.id == cycle.id).\
                            filter(tm.Channel.wavelength == args.ref_wavelength).\
                            filter(~tm.Site.omitted).\
                            all()

                        if not files:
                            # We don't raise an Execption here, because
                            # there may be situations were an aquisition
                            # failed at a given site in one cycle, but
                            # is present in the other cycles.
                            logger.warning(
                                'no files for site %d and cycle %d',
                                s, cycle.id
                            )
                            continue

                        ids = [f.id for f in files]
                        if cycle.index == args.ref_cycle:
                            input_ids['reference_file_ids'].extend(ids)
                        input_ids['target_file_ids'][cycle.id].extend(ids)

                job_descriptions['run'].append({
                    'id': job_count,
                    'input_ids': input_ids,
                    'illumcorr': args.illumcorr
                })

        return job_descriptions

    @same_docstring_as(ClusterRoutines.delete_previous_job_output)
    def delete_previous_job_output(self):
        logger.info('delete existing site shifts and intersections')
        with tm.utils.ExperimentSession(self.experiment_id) as session:
            session.drop_and_recreate(tm.SiteShift)
            session.drop_and_recreate(tm.SiteIntersection)

    def run_job(self, batch):
        '''Calculates shift and overhang values for the given sites.

        Parameters
        ----------
        batch: dict
            description of the *run* job

        Note
        ----
        If sites contain multiple z-planes, z-stacks are projected to 2D and
        the resulting projections are registered.
        '''
        with tm.utils.ExperimentSession(self.experiment_id) as session:
            reference_file_ids = batch['input_ids']['reference_file_ids']
            target_file_ids = batch['input_ids']['target_file_ids']
            if batch['illumcorr']:
                logger.info('correct images for illumination artifacts')

                rid = reference_file_ids[0]
                reference_file = session.query(tm.ChannelImageFile).get(rid)
                try:
                    logger.debug(
                        'load illumination statistics for channel %d of '
                        'reference cycle %d', reference_file.channel_id,
                        reference_file.cycle_id
                    )
                    illumstats_file = session.query(tm.IllumstatsFile).\
                        filter_by(channel_id=reference_file.channel_id).\
                        one()
                except NoResultFound:
                    raise WorkflowError(
                        'No illumination statistics file found for channel %d'
                        % reference_file.channel_id
                    )
                reference_stats = illumstats_file.get()

                target_stats = dict()
                for cycle_id, tids in target_file_ids.iteritems():
                    target_file = session.query(tm.ChannelImageFile).get(tids[0])
                    try:
                        logger.debug(
                            'load illumination statistics for channel %d of'
                            'target cycle %d', target_file.channel_id,
                            target_file.cycle_id
                        )
                        illumstats_file = session.query(tm.IllumstatsFile).\
                            filter_by(
                                channel_id=target_file.channel_id,
                                cycle_id=target_file.cycle_id
                            ).\
                            one()
                    except NoResultFound:
                        raise WorkflowError(
                            'No illumination statistics file found for '
                            'channel %d'
                            % target_file.channel_id
                        )
                    target_stats[cycle_id] = illumstats_file.get()

            for i, rid in enumerate(reference_file_ids):
                reference_file = session.query(tm.ChannelImageFile).get(rid)
                logger.info('register images at site %d', reference_file.site_id)
                logger.debug('load reference image %d', rid)
                reference_img = reference_file.get()
                if batch['illumcorr']:
                    logger.debug('correct reference image')
                    reference_img = reference_img.correct(reference_stats)
                y_shifts = list()
                x_shifts = list()
                for cycle_id, tids in target_file_ids.iteritems():
                    logger.info('calculate shifts for cycle %s', cycle_id)
                    logger.debug('load target image %d', tids[i])
                    target_file = session.query(tm.ChannelImageFile).get(tids[i])
                    target_img = target_file.get()
                    if batch['illumcorr']:
                        logger.debug('correct target image')
                        target_img = target_img.correct(target_stats[cycle_id])

                    y, x = reg.calculate_shift(
                        target_img.array, reference_img.array
                    )

                    session.get_or_create(
                        tm.SiteShift,
                        x=x, y=y,
                        site_id=target_file.site_id,
                        cycle_id=target_file.cycle_id
                    )

                    y_shifts.append(y)
                    x_shifts.append(x)

                logger.info('calculate intersection of sites across cycles')
                top, bottom, right, left = reg.calculate_overhang(
                    y_shifts, x_shifts
                )

                session.get_or_create(
                    tm.SiteIntersection,
                    upper_overhang=top, lower_overhang=bottom,
                    right_overhang=right, left_overhang=left,
                    site_id=reference_file.site_id
                )

    @notimplemented
    def collect_job_output(self, batch):
        pass
        # TODO: set shifts to zero for sites that were omitted

<commit_msg>Fix bug related to querying illumstat files in align step II<commit_after># TmLibrary - TissueMAPS library for distibuted image analysis routines.
# Copyright (C) 2016  Markus D. Herrmann, University of Zurich and Robin Hafen
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
import logging
from collections import defaultdict

import tmlib.models as tm
from tmlib.utils import notimplemented
from tmlib.utils import same_docstring_as
from tmlib.errors import NotSupportedError
from sqlalchemy.orm.exc import NoResultFound
from tmlib.errors import JobDescriptionError
from tmlib.workflow.align import registration as reg
from tmlib.workflow.api import ClusterRoutines
from tmlib.workflow import register_step_api

logger = logging.getLogger(__name__)

@register_step_api('align')
class ImageRegistrator(ClusterRoutines):

    '''Class for registeration and alignment of images between multiplexing
    cycles.
    '''

    def __init__(self, experiment_id, verbosity):
        '''
        Parameters
        ----------
        experiment_id: int
            ID of the processed experiment
        verbosity: int
            logging level
        '''
        super(ImageRegistrator, self).__init__(experiment_id, verbosity)

    def create_batches(self, args):
        '''Creates job descriptions for parallel computing.

        Parameters
        ----------
        args: tmlib.align.args.AlignInitArgs
            step-specific arguments

        Returns
        -------
        Dict[str, List[dict] or dict]
            job descriptions

        Raises
        ------
        tmlib.errors.NotSupportedError
            when a plate contains only one cycle
        ValueError
            when `args.ref_wavelength` does not exist across all cycles
        '''
        job_count = 0
        job_descriptions = dict()
        job_descriptions['run'] = list()

        with tm.utils.ExperimentSession(self.experiment_id) as session:

            cycles = session.query(tm.Cycle).all()

            if not(len(cycles) > 1):
                raise NotSupportedError(
                    'Alignment requires more than one cycle.'
                )

            if args.ref_cycle >= len(cycles):
                raise JobDescriptionError(
                    'Cycle index must not exceed total number of cycles.'
                )

            site_ids = session.query(tm.Site.id).\
                order_by(tm.Site.id).\
                all()

            batches = self._create_batches(site_ids, args.batch_size)
            for batch in batches:

                job_count += 1
                input_ids = {
                    'reference_file_ids': list(),
                    'target_file_ids': defaultdict(list)
                }

                for cycle in cycles:

                    n = session.query(tm.ChannelImageFile.id).\
                        join(tm.Cycle).\
                        join(tm.Channel).\
                        filter(tm.Cycle.id == cycle.id).\
                        filter(tm.Channel.wavelength == args.ref_wavelength).\
                        filter(~tm.Site.omitted).\
                        count()

                    if n == 0:
                        raise ValueError(
                            'No image files found for cycle %d and '
                            'wavelength "%s"'
                            % (cycle.id, args.ref_wavelength)
                        )

                    for s in batch:

                        files = session.query(tm.ChannelImageFile.id).\
                            join(tm.Site).\
                            join(tm.Cycle).\
                            join(tm.Channel).\
                            filter(tm.Site.id == s).\
                            filter(tm.Cycle.id == cycle.id).\
                            filter(tm.Channel.wavelength == args.ref_wavelength).\
                            filter(~tm.Site.omitted).\
                            all()

                        if not files:
                            # We don't raise an Execption here, because
                            # there may be situations were an aquisition
                            # failed at a given site in one cycle, but
                            # is present in the other cycles.
                            logger.warning(
                                'no files for site %d and cycle %d',
                                s, cycle.id
                            )
                            continue

                        ids = [f.id for f in files]
                        if cycle.index == args.ref_cycle:
                            input_ids['reference_file_ids'].extend(ids)
                        input_ids['target_file_ids'][cycle.id].extend(ids)

                job_descriptions['run'].append({
                    'id': job_count,
                    'input_ids': input_ids,
                    'illumcorr': args.illumcorr
                })

        return job_descriptions

    @same_docstring_as(ClusterRoutines.delete_previous_job_output)
    def delete_previous_job_output(self):
        logger.info('delete existing site shifts and intersections')
        with tm.utils.ExperimentSession(self.experiment_id) as session:
            session.drop_and_recreate(tm.SiteShift)
            session.drop_and_recreate(tm.SiteIntersection)

    def run_job(self, batch):
        '''Calculates shift and overhang values for the given sites.

        Parameters
        ----------
        batch: dict
            description of the *run* job

        Note
        ----
        If sites contain multiple z-planes, z-stacks are projected to 2D and
        the resulting projections are registered.
        '''
        with tm.utils.ExperimentSession(self.experiment_id) as session:
            reference_file_ids = batch['input_ids']['reference_file_ids']
            target_file_ids = batch['input_ids']['target_file_ids']
            if batch['illumcorr']:
                logger.info('correct images for illumination artifacts')

                rid = reference_file_ids[0]
                reference_file = session.query(tm.ChannelImageFile).get(rid)
                try:
                    logger.debug(
                        'load illumination statistics for channel %d of '
                        'reference cycle %d', reference_file.channel_id,
                        reference_file.cycle_id
                    )
                    illumstats_file = session.query(tm.IllumstatsFile).\
                        filter_by(channel_id=reference_file.channel_id).\
                        one()
                except NoResultFound:
                    raise WorkflowError(
                        'No illumination statistics file found for channel %d'
                        % reference_file.channel_id
                    )
                reference_stats = illumstats_file.get()

                target_stats = dict()
                for cycle_id, tids in target_file_ids.iteritems():
                    target_file = session.query(tm.ChannelImageFile).get(tids[0])
                    try:
                        logger.debug(
                            'load illumination statistics for channel %d of'
                            'target cycle %d', target_file.channel_id,
                            target_file.cycle_id
                        )
                        illumstats_file = session.query(tm.IllumstatsFile).\
                            filter_by(channel_id=target_file.channel_id).\
                            one()
                    except NoResultFound:
                        raise WorkflowError(
                            'No illumination statistics file found for '
                            'channel %d'
                            % target_file.channel_id
                        )
                    target_stats[cycle_id] = illumstats_file.get()

            for i, rid in enumerate(reference_file_ids):
                reference_file = session.query(tm.ChannelImageFile).get(rid)
                logger.info('register images at site %d', reference_file.site_id)
                logger.debug('load reference image %d', rid)
                reference_img = reference_file.get()
                if batch['illumcorr']:
                    logger.debug('correct reference image')
                    reference_img = reference_img.correct(reference_stats)
                y_shifts = list()
                x_shifts = list()
                for cycle_id, tids in target_file_ids.iteritems():
                    logger.info('calculate shifts for cycle %s', cycle_id)
                    logger.debug('load target image %d', tids[i])
                    target_file = session.query(tm.ChannelImageFile).get(tids[i])
                    target_img = target_file.get()
                    if batch['illumcorr']:
                        logger.debug('correct target image')
                        target_img = target_img.correct(target_stats[cycle_id])

                    y, x = reg.calculate_shift(
                        target_img.array, reference_img.array
                    )

                    session.get_or_create(
                        tm.SiteShift,
                        x=x, y=y,
                        site_id=target_file.site_id,
                        cycle_id=target_file.cycle_id
                    )

                    y_shifts.append(y)
                    x_shifts.append(x)

                logger.info('calculate intersection of sites across cycles')
                top, bottom, right, left = reg.calculate_overhang(
                    y_shifts, x_shifts
                )

                session.get_or_create(
                    tm.SiteIntersection,
                    upper_overhang=top, lower_overhang=bottom,
                    right_overhang=right, left_overhang=left,
                    site_id=reference_file.site_id
                )

    @notimplemented
    def collect_job_output(self, batch):
        pass
        # TODO: set shifts to zero for sites that were omitted

<|endoftext|>


<commit_before>import tensorflow as tf
import numpy as np

from .base import BaseSampler

class EmceeSampler(BaseSampler):

    def __init__(self, walkers=100, **kwargs):
        try:
            import emcee
        except ImportError:
            raise ImportError("The emcee package needs to be installed in order to use EmceeSampler")

        self.emcee = emcee
        self.walkers = walkers
        super(EmceeSampler, self).__init__(**kwargs)

    def sample(self, variables, cost, gradient=None, samples=None):
        # Check if variables is iterable
        try:
            iter(variables)
        except TypeError:
            raise ValueError("Variables parameter is not iterable")

        inits = self.session.run(variables)

        for v in variables:
            if not isinstance(v, tf.Variable):
                raise ValueError("Parameter {} is not a tensorflow variable".format(v))

        def objective(xs):
            feed_dict = { k: v for k, v in zip(variables, xs) }
            out = self.session.run(cost, feed_dict=feed_dict)
            if np.isnan(out):
                return np.inf
            return -out

        all_inits = self.emcee.utils.sample_ball(inits, [1e-1] * len(inits), self.walkers)
        sampler = self.emcee.EnsembleSampler(self.walkers, len(variables), objective)

        samples = 1 if samples is None else samples
        sampler.random_state = np.random.mtrand.RandomState(np.random.randint(1)).get_state()
        pos, lnprob, rstate = sampler.run_mcmc(all_inits, samples)
        return sampler.chain

<commit_msg>Fix import under Python 2<commit_after>from __future__ import absolute_import

import tensorflow as tf
import numpy as np

from .base import BaseSampler

class EmceeSampler(BaseSampler):

    def __init__(self, walkers=100, **kwargs):
        try:
            import emcee
        except ImportError:
            raise ImportError("The emcee package needs to be installed in order to use EmceeSampler")

        self.emcee = emcee
        self.walkers = walkers
        super(EmceeSampler, self).__init__(**kwargs)

    def sample(self, variables, cost, gradient=None, samples=None):
        # Check if variables is iterable
        try:
            iter(variables)
        except TypeError:
            raise ValueError("Variables parameter is not iterable")

        inits = self.session.run(variables)

        for v in variables:
            if not isinstance(v, tf.Variable):
                raise ValueError("Parameter {} is not a tensorflow variable".format(v))

        def objective(xs):
            feed_dict = { k: v for k, v in zip(variables, xs) }
            out = self.session.run(cost, feed_dict=feed_dict)
            if np.isnan(out):
                return np.inf
            return -out

        all_inits = self.emcee.utils.sample_ball(inits, [1e-1] * len(inits), self.walkers)
        sampler = self.emcee.EnsembleSampler(self.walkers, len(variables), objective)

        samples = 1 if samples is None else samples
        sampler.random_state = np.random.mtrand.RandomState(np.random.randint(1)).get_state()
        pos, lnprob, rstate = sampler.run_mcmc(all_inits, samples)
        return sampler.chain

<|endoftext|>


<commit_before>import logging
log = logging.getLogger(__name__)

from ReportGenerator.Scan import Scan
from ReportGenerator.IRC import IRC
from ReportGenerator.Freq import Freq
from ReportGenerator.Opt import Opt
from ReportGenerator.SinglePoint import SinglePoint
from ReportGenerator.Top import Top_ReportGenerator

class Job(Top_ReportGenerator):

    def __init__(self, we,parsed):
        super().__init__(we,parsed)

    def prepare_for_report(self):
        pass

    def jobtype_html(self):
        P = self.parsed
        jobtype = P.last_value('P_jobtype')
        if not jobtype:
            return 'Job type not determined'
        if isinstance(jobtype,list):
            sx = " ".join(jobtype).upper()
        else:
            sx = jobtype.upper()
        sx = self.strong_tag(sx)
        if P.last_value('P_term_ok')== 'True':
            return sx
        else:
            return self.color_tag(sx, 'err')

    def level_of_theory_html(self):
        method = self.parsed.last_value('P_wftype')
        basis = self.parsed.last_value('P_basis')
        lot = "%s / %s" % (method,basis)
        return self.color_tag(lot.upper(), 'lot')

    def fragmentation_html(self):
        P = self.parsed
        if P.last_value('P_wftype_fragment') is not None:
            lot2 = 'Fragmentation scheme: ' + P.last_value('P_wftype_fragment')
            return self.color_tag(lot2, 'lot')

    def symmetry_html(self):
        return "Symmetry: %s\n" % self.parsed.last_value('P_sym')

    def charge_mult_html(self):
        charge = "Charge: %s; " % self.parsed.last_value('P_charge')
        mult = "Mult: %s\n" % self.parsed.last_value('P_mult')
        return charge + mult

    def open_shell_html(self):
        if self.parsed.last_value('P_open_shell')== 'True':
            return "Open Shell; S2= %s,\n" % self.parsed.last_value('P_S2')

    def scf_energy_html(self):
        try:
            return "Last SCF Energy= %-11.6f\n" % float(self.parsed.last_value('P_scf_e'))
        except TypeError:
            return "Last SCF Energy %s\n" % self.color_tag('N/A','err')

    def scf_failed_html(self):
        if self.parsed.last_value('P_scf_notconv')== 'True':
            return self.color_tag('SCF did not converge!', 'err')

    def solvent_html(self):
        P = self.parsed
        if P.last_value('P_solvent') is not None:
            sx = 'Solvation: '
            if P.last_value('P_solv_model') is not None:
                sx += '%s(%s)' % (P.last_value('P_solv_model'), P.last_value('P_solvent'))
            return self.color_tag(sx, 'lot')

    def coupled_cluster_html(self):
        if self.parsed.last_value('P_ccsd_pTp_energy') is not None:
            return "Last CCSD(T) Energy= %-11.6f\n" % float(self.parsed.last_value('P_ccsd_pTp_energy'))

    def t1_diagnostics_html(self):
        if self.parsed.last_value('P_T1_diagnostic') is not None:
            T1 = float(self.parsed.last_value('P_T1_diagnostic'))
            s_T1 = "T1 diagnostic= %.3f\n" % T1
            if T1 >= 0.025:
                s_T1 = self.color_tag(s_T1, 'err')
            return s_T1

    def report_text(self):
        pass

    def report_html(self):
        self.add_left(self.we.initiate_jmol_applet()) # initialize an JSMol applet
        out_html = [
            self.jobtype_html(),
            self.level_of_theory_html(),
            self.fragmentation_html(),
            self.symmetry_html(),
            self.charge_mult_html(),
            self.open_shell_html(),
            self.scf_energy_html(),
            self.scf_failed_html(),
            self.solvent_html(),
            self.coupled_cluster_html(),
            self.t1_diagnostics_html()
        ]
        [self.add_right(s + self.br_tag) for s in out_html if s]

        P = self.parsed
        jobtype = P.last_value('P_jobtype')
        if not jobtype:
            return self.get_cells()

        # P is wrapped in the aggregation order: scan > irc > opt;
        for name,Gen in {'scan':Scan,'irc':IRC,'opt':Opt}.items():
            if name in jobtype:
                self.add_both(Gen(self.we, P).report())
            else:
                P = P.data[0]

        if 'freq' in jobtype and not ('opt' in jobtype and P.last_value('P_opt_ok')!='True'):
            self.add_both(Freq(self.we, P).report())

        if 'sp' in jobtype:
            self.add_both(SinglePoint(self.we, P).report())

        return self.get_cells()<commit_msg>fixed a bug with the job type orders<commit_after>import logging
log = logging.getLogger(__name__)

from ReportGenerator.Scan import Scan
from ReportGenerator.IRC import IRC
from ReportGenerator.Freq import Freq
from ReportGenerator.Opt import Opt
from ReportGenerator.SinglePoint import SinglePoint
from ReportGenerator.Top import Top_ReportGenerator

class Job(Top_ReportGenerator):

    def __init__(self, we,parsed):
        super().__init__(we,parsed)

    def prepare_for_report(self):
        pass

    def jobtype_html(self):
        P = self.parsed
        jobtype = P.last_value('P_jobtype')
        if not jobtype:
            return 'Job type not determined'
        if isinstance(jobtype,list):
            sx = " ".join(jobtype).upper()
        else:
            sx = jobtype.upper()
        sx = self.strong_tag(sx)
        if P.last_value('P_term_ok')== 'True':
            return sx
        else:
            return self.color_tag(sx, 'err')

    def level_of_theory_html(self):
        method = self.parsed.last_value('P_wftype')
        basis = self.parsed.last_value('P_basis')
        lot = "%s / %s" % (method,basis)
        return self.color_tag(lot.upper(), 'lot')

    def fragmentation_html(self):
        P = self.parsed
        if P.last_value('P_wftype_fragment') is not None:
            lot2 = 'Fragmentation scheme: ' + P.last_value('P_wftype_fragment')
            return self.color_tag(lot2, 'lot')

    def symmetry_html(self):
        return "Symmetry: %s\n" % self.parsed.last_value('P_sym')

    def charge_mult_html(self):
        charge = "Charge: %s; " % self.parsed.last_value('P_charge')
        mult = "Mult: %s\n" % self.parsed.last_value('P_mult')
        return charge + mult

    def open_shell_html(self):
        if self.parsed.last_value('P_open_shell')== 'True':
            return "Open Shell; S2= %s,\n" % self.parsed.last_value('P_S2')

    def scf_energy_html(self):
        try:
            return "Last SCF Energy= %-11.6f\n" % float(self.parsed.last_value('P_scf_e'))
        except TypeError:
            return "Last SCF Energy %s\n" % self.color_tag('N/A','err')

    def scf_failed_html(self):
        if self.parsed.last_value('P_scf_notconv')== 'True':
            return self.color_tag('SCF did not converge!', 'err')

    def solvent_html(self):
        P = self.parsed
        if P.last_value('P_solvent') is not None:
            sx = 'Solvation: '
            if P.last_value('P_solv_model') is not None:
                sx += '%s(%s)' % (P.last_value('P_solv_model'), P.last_value('P_solvent'))
            return self.color_tag(sx, 'lot')

    def coupled_cluster_html(self):
        if self.parsed.last_value('P_ccsd_pTp_energy') is not None:
            return "Last CCSD(T) Energy= %-11.6f\n" % float(self.parsed.last_value('P_ccsd_pTp_energy'))

    def t1_diagnostics_html(self):
        if self.parsed.last_value('P_T1_diagnostic') is not None:
            T1 = float(self.parsed.last_value('P_T1_diagnostic'))
            s_T1 = "T1 diagnostic= %.3f\n" % T1
            if T1 >= 0.025:
                s_T1 = self.color_tag(s_T1, 'err')
            return s_T1

    def report_text(self):
        pass

    def report_html(self):
        self.add_left(self.we.initiate_jmol_applet()) # initialize an JSMol applet
        out_html = [
            self.jobtype_html(),
            self.level_of_theory_html(),
            self.fragmentation_html(),
            self.symmetry_html(),
            self.charge_mult_html(),
            self.open_shell_html(),
            self.scf_energy_html(),
            self.scf_failed_html(),
            self.solvent_html(),
            self.coupled_cluster_html(),
            self.t1_diagnostics_html()
        ]
        [self.add_right(s + self.br_tag) for s in out_html if s]

        P = self.parsed
        jobtype = P.last_value('P_jobtype')
        if not jobtype:
            return self.get_cells()

        # P is wrapped in the aggregation order: scan > irc > opt;
        for name,Gen in zip(('scan','irc','opt'),(Scan,IRC,Opt)):
            if name in jobtype:
                self.add_both(Gen(self.we, P).report())
            else:
                P = P.data[0]

        if 'freq' in jobtype and not ('opt' in jobtype and P.last_value('P_opt_ok')!='True'):
            self.add_both(Freq(self.we, P).report())

        if 'sp' in jobtype:
            self.add_both(SinglePoint(self.we, P).report())

        return self.get_cells()<|endoftext|>


<commit_before>import unittest
import saliweb.test
import os
import re

# Import the modloop frontend with mocks
modloop = saliweb.test.import_mocked_frontend("modloop", __file__,
                                              '../../frontend')

class Tests(saliweb.test.TestCase):
    """Check submit page"""

    def test_submit_page(self):
        """Test submit page"""
        with saliweb.test.temporary_directory() as t:
            incoming = os.path.join(t, 'incoming')
            os.mkdir(incoming)
            modloop.app.config['DIRECTORIES_INCOMING'] = incoming
            c = modloop.app.test_client()
            rv = c.post('/job')
            self.assertEqual(rv.status_code, 400)  # no license key

            modkey = saliweb.test.get_modeller_key()
            data = {'modkey': modkey}
            rv = c.post('/job', data=data)
            self.assertEqual(rv.status_code, 400)  # no loops

            data['loops'] = '1::1::'
            rv = c.post('/job', data=data)
            self.assertEqual(rv.status_code, 400)  # no pdb file

            pdbf = os.path.join(t, 'test.pdb')
            with open(pdbf, 'w') as fh:
                fh.write(
                    "REMARK\n"
                    "ATOM      2  CA  ALA     1      26.711  14.576   5.091\n")

            # Successful submission (no email)
            data['pdb'] = open(pdbf, 'rb')
            rv = c.post('/job', data=data)
            self.assertEqual(rv.status_code, 200)
            r = re.compile(b'Your job has been submitted.*'
                           b'You can check on your job',
                           re.MULTILINE | re.DOTALL)
            self.assertRegex(rv.data, r)

            # Successful submission (with email)
            data['email'] = 'test@test.com'
            data['pdb'] = open(pdbf, 'rb')
            rv = c.post('/job', data=data)
            self.assertEqual(rv.status_code, 200)
            r = re.compile(
                b'Your job has been submitted.*You will be notified.*'
                b'You can check on your job', re.MULTILINE | re.DOTALL)
            self.assertRegex(rv.data, r)

    def test_check_loop_selection(self):
        """Test check_loop_selection()"""
        modloop.submit.check_loop_selection("anything")
        self.assertRaises(saliweb.frontend.InputValidationError,
                          modloop.submit.check_loop_selection, "")

    def test_check_pdb_name(self):
        """Test check_pdb_name()"""
        modloop.submit.check_pdb_name("anything")
        self.assertRaises(saliweb.frontend.InputValidationError,
                          modloop.submit.check_pdb_name, "")

    def test_parse_loop_selection(self):
        """Test parse_loop_selection()"""
        (loops, start_res, start_id, end_res, end_id,
         loop_data) = modloop.submit.parse_loop_selection('1::5::16:A:30:A:')
        self.assertEqual(loop_data, ['1', ' ', '5', ' ', '16', 'A', '30', 'A'])
        self.assertEqual(start_res, [1, 16])
        self.assertEqual(start_id, [' ', 'A'])
        self.assertEqual(end_res, [5, 30])
        self.assertEqual(end_id, [' ', 'A'])

        # Wrong number of colons
        self.assertRaises(
            saliweb.frontend.InputValidationError,
            modloop.submit.parse_loop_selection, '1::5::16:A:30:')

        # Loop that spans chains
        self.assertRaises(
            saliweb.frontend.InputValidationError,
            modloop.submit.parse_loop_selection, '1::5::16:A:20:B:')

        # Loop too long
        self.assertRaises(
            saliweb.frontend.InputValidationError,
            modloop.submit.parse_loop_selection, '1::50::16:A:20:B:')

        # Non-numeric residue
        self.assertRaises(
            saliweb.frontend.InputValidationError,
            modloop.submit.parse_loop_selection, '1::5A::16:A:20:B:')

        # Loop of negative length
        self.assertRaises(
            saliweb.frontend.InputValidationError,
            modloop.submit.parse_loop_selection, '10::5::16:A:20:B:')

        # Too many residues
        self.assertRaises(
            saliweb.frontend.InputValidationError,
            modloop.submit.parse_loop_selection, '1::10::20:A:31:A:')

        # No residues
        self.assertRaises(
            saliweb.frontend.InputValidationError,
            modloop.submit.parse_loop_selection, '')

        # Empty first residue
        self.assertRaises(
            saliweb.frontend.InputValidationError,
            modloop.submit.parse_loop_selection, '::10::20:A:31:A:')

    def test_read_pdb_file(self):
        """Test read_pdb_file()"""
        with saliweb.test.temporary_directory() as tmpdir:
            pdb = os.path.join(tmpdir, 'test.pdb')
            with open(pdb, 'w') as fh:
                for chain in (' ', 'A'):
                    for resid in range(1, 11):
                        fh.write(
                            "ATOM      1  CA  ALA %1s%4d      "
                            "18.511  -1.416  15.632  1.00  6.84           C\n"
                            % (chain, resid))

            # Successful read
            with open(pdb, 'rb') as fh:
                contents = modloop.submit.read_pdb_file(
                    fh, 2, [1, 1], [' ', 'A'], [5, 5], [' ', 'A'])
            r = re.compile(
                b'^ATOM\\s+1\\s+CA\\s+ALA     1.*ATOM\\s+1\\s+CA\\s+ALA A  10',
                re.MULTILINE | re.DOTALL)
            self.assertRegex(b"".join(contents), r)

            # Loop not found in ATOM records
            with open(pdb, 'rb') as fh:
                self.assertRaises(
                    saliweb.frontend.InputValidationError,
                    modloop.submit.read_pdb_file, fh, 2, [1, 1], [' ', 'A'],
                    [5, 15], [' ', 'A'])

if __name__ == '__main__':
    unittest.main()
<commit_msg>Use tempfile.TemporaryDirectory in tests<commit_after>import unittest
import saliweb.test
import tempfile
import os
import re

# Import the modloop frontend with mocks
modloop = saliweb.test.import_mocked_frontend("modloop", __file__,
                                              '../../frontend')

class Tests(saliweb.test.TestCase):
    """Check submit page"""

    def test_submit_page(self):
        """Test submit page"""
        with tempfile.TemporaryDirectory() as t:
            incoming = os.path.join(t, 'incoming')
            os.mkdir(incoming)
            modloop.app.config['DIRECTORIES_INCOMING'] = incoming
            c = modloop.app.test_client()
            rv = c.post('/job')
            self.assertEqual(rv.status_code, 400)  # no license key

            modkey = saliweb.test.get_modeller_key()
            data = {'modkey': modkey}
            rv = c.post('/job', data=data)
            self.assertEqual(rv.status_code, 400)  # no loops

            data['loops'] = '1::1::'
            rv = c.post('/job', data=data)
            self.assertEqual(rv.status_code, 400)  # no pdb file

            pdbf = os.path.join(t, 'test.pdb')
            with open(pdbf, 'w') as fh:
                fh.write(
                    "REMARK\n"
                    "ATOM      2  CA  ALA     1      26.711  14.576   5.091\n")

            # Successful submission (no email)
            data['pdb'] = open(pdbf, 'rb')
            rv = c.post('/job', data=data)
            self.assertEqual(rv.status_code, 200)
            r = re.compile(b'Your job has been submitted.*'
                           b'You can check on your job',
                           re.MULTILINE | re.DOTALL)
            self.assertRegex(rv.data, r)

            # Successful submission (with email)
            data['email'] = 'test@test.com'
            data['pdb'] = open(pdbf, 'rb')
            rv = c.post('/job', data=data)
            self.assertEqual(rv.status_code, 200)
            r = re.compile(
                b'Your job has been submitted.*You will be notified.*'
                b'You can check on your job', re.MULTILINE | re.DOTALL)
            self.assertRegex(rv.data, r)

    def test_check_loop_selection(self):
        """Test check_loop_selection()"""
        modloop.submit.check_loop_selection("anything")
        self.assertRaises(saliweb.frontend.InputValidationError,
                          modloop.submit.check_loop_selection, "")

    def test_check_pdb_name(self):
        """Test check_pdb_name()"""
        modloop.submit.check_pdb_name("anything")
        self.assertRaises(saliweb.frontend.InputValidationError,
                          modloop.submit.check_pdb_name, "")

    def test_parse_loop_selection(self):
        """Test parse_loop_selection()"""
        (loops, start_res, start_id, end_res, end_id,
         loop_data) = modloop.submit.parse_loop_selection('1::5::16:A:30:A:')
        self.assertEqual(loop_data, ['1', ' ', '5', ' ', '16', 'A', '30', 'A'])
        self.assertEqual(start_res, [1, 16])
        self.assertEqual(start_id, [' ', 'A'])
        self.assertEqual(end_res, [5, 30])
        self.assertEqual(end_id, [' ', 'A'])

        # Wrong number of colons
        self.assertRaises(
            saliweb.frontend.InputValidationError,
            modloop.submit.parse_loop_selection, '1::5::16:A:30:')

        # Loop that spans chains
        self.assertRaises(
            saliweb.frontend.InputValidationError,
            modloop.submit.parse_loop_selection, '1::5::16:A:20:B:')

        # Loop too long
        self.assertRaises(
            saliweb.frontend.InputValidationError,
            modloop.submit.parse_loop_selection, '1::50::16:A:20:B:')

        # Non-numeric residue
        self.assertRaises(
            saliweb.frontend.InputValidationError,
            modloop.submit.parse_loop_selection, '1::5A::16:A:20:B:')

        # Loop of negative length
        self.assertRaises(
            saliweb.frontend.InputValidationError,
            modloop.submit.parse_loop_selection, '10::5::16:A:20:B:')

        # Too many residues
        self.assertRaises(
            saliweb.frontend.InputValidationError,
            modloop.submit.parse_loop_selection, '1::10::20:A:31:A:')

        # No residues
        self.assertRaises(
            saliweb.frontend.InputValidationError,
            modloop.submit.parse_loop_selection, '')

        # Empty first residue
        self.assertRaises(
            saliweb.frontend.InputValidationError,
            modloop.submit.parse_loop_selection, '::10::20:A:31:A:')

    def test_read_pdb_file(self):
        """Test read_pdb_file()"""
        with tempfile.TemporaryDirectory() as tmpdir:
            pdb = os.path.join(tmpdir, 'test.pdb')
            with open(pdb, 'w') as fh:
                for chain in (' ', 'A'):
                    for resid in range(1, 11):
                        fh.write(
                            "ATOM      1  CA  ALA %1s%4d      "
                            "18.511  -1.416  15.632  1.00  6.84           C\n"
                            % (chain, resid))

            # Successful read
            with open(pdb, 'rb') as fh:
                contents = modloop.submit.read_pdb_file(
                    fh, 2, [1, 1], [' ', 'A'], [5, 5], [' ', 'A'])
            r = re.compile(
                b'^ATOM\\s+1\\s+CA\\s+ALA     1.*ATOM\\s+1\\s+CA\\s+ALA A  10',
                re.MULTILINE | re.DOTALL)
            self.assertRegex(b"".join(contents), r)

            # Loop not found in ATOM records
            with open(pdb, 'rb') as fh:
                self.assertRaises(
                    saliweb.frontend.InputValidationError,
                    modloop.submit.read_pdb_file, fh, 2, [1, 1], [' ', 'A'],
                    [5, 15], [' ', 'A'])

if __name__ == '__main__':
    unittest.main()
<|endoftext|>