<commit_before>import collections
import os
import sys

import matplotlib.pyplot as plt
import numpy as np
from astromodels import Parameter
from cthreeML.pyModelInterfaceCache import pyToCppModelInterfaceCache
from hawc import liff_3ML
from matplotlib import gridspec

from threeML.io.file_utils import file_existing_and_readable, sanitize_filename
from threeML.plugin_prototype import PluginPrototype

defaultMinChannel = 0
defaultMaxChannel = 9

__instrument_name = "HAWC"

class HAWCLike(PluginPrototype):

    def __init__(self, name, maptree, response, n_transits=None, **kwargs):

        # This controls if the likeHAWC class should load the entire
        # map or just a small disc around a source (faster).
        # Default is the latter, which is way faster. LIFF will decide
        # autonomously which ROI to use depending on the source model

        self.fullsky = False

        if 'fullsky' in kwargs.keys():
            self.fullsky = bool(kwargs['fullsky'])

        self.name = str(name)

        # Sanitize files in input (expand variables and so on)

        self.maptree = os.path.abspath(sanitize_filename(maptree))

        self.response = os.path.abspath(sanitize_filename(response))

        # Check that they exists and can be read

        if not file_existing_and_readable(self.maptree):
            raise IOError("MapTree %s does not exist or is not readable" % maptree)

        if not file_existing_and_readable(self.response):
            raise IOError("Response %s does not exist or is not readable" % response)

        # Post-pone the creation of the LIFF instance to when
        # we have the likelihood model

        self.instanced = False

        # Number of transits
        if n_transits is not None:

            self._n_transits = float(n_transits)

        else:

            self._n_transits = None

            # Default value for minChannel and maxChannel

        self.minChannel = int(defaultMinChannel)
        self.maxChannel = int(defaultMaxChannel)

        # By default the fit of the CommonNorm is deactivated

        self.deactivate_CommonNorm()

        # This is to keep track of whether the user defined a ROI or not

        self.roi_ra = None

        # Further setup

        self.__setup()

    def set_ROI(self, ra, dec, radius, fixedROI=False):

        self.roi_ra = ra
        self.roi_dec = dec

        self.roi_radius = radius

        self.fixedROI = fixedROI

    def __setup(self):

        # I put this here so I can use it both from the __init__ both from
        # the __setstate__ methods

        # Create the dictionary of nuisance parameters

        self.nuisanceParameters = collections.OrderedDict()
        self.nuisanceParameters['CommonNorm'] = Parameter("CommonNorm", 1.0, min_value=0.5, max_value=1.5,
                                                          delta=0.01)
        self.nuisanceParameters['CommonNorm'].fix = True

    def __getstate__(self):

        # This method is used by pickle before attempting to pickle the class

        # Return only the objects needed to recreate the class
        # IN particular, we do NOT return the theLikeHAWC class,
        # which is not pickeable. It will instead be recreated
        # on the other side

        d = {}

        d['name'] = self.name
        d['maptree'] = self.maptree
        d['response'] = self.response
        d['model'] = self.model
        d['n_transits'] = self._n_transits
        d['minChannel'] = self.minChannel
        d['maxChannel'] = self.maxChannel

        d['roi_ra'] = self.roi_ra

        if self.roi_ra is not None:
            d['roi_dec'] = self.roi_dec
            d['roi_radius'] = self.roi_radius

        return d

    def __setstate__(self, state):

        # This is used by pickle to recreate the class on the remote
        # side
        name = state['name']
        maptree = state['maptree']
        response = state['response']
        ntransits = state['n_transits']

        self._n_transits = ntransits

        # Now report the class to its state

        self.__init__(name, maptree, response)

        if state['roi_ra'] is not None:
            self.set_ROI(state['roi_ra'], state['roi_dec'], state['roi_radius'], state['fixedROI'])

        self.set_active_measurements(state['minChannel'], state['maxChannel'])

        self.set_model(state['model'])

    def set_active_measurements(self, minChannel, maxChannel):

        self.minChannel = int(minChannel)
        self.maxChannel = int(maxChannel)

        if self.instanced:
            sys.stderr.write("Since the plugins was already used before, the change in active measurements" +
                             "will not be effective until you create a new JointLikelihood or Bayesian" +
                             "instance")

    def set_model(self, likelihood_model_instance):
        """
        Set the model to be used in the joint minimization. Must be a LikelihoodModel instance.
        """

        # Instance the python - C++ bridge

        self.model = likelihood_model_instance

        self.pymodel = pyToCppModelInterfaceCache()

        # Set boundaries for extended source
        # NOTE: we assume that these boundaries do not change during the fit

        for id in range(self.model.get_number_of_extended_sources()):

            lon_min, lon_max, lat_min, lat_max = self.model.get_extended_source_boundaries(id)

            self.pymodel.setExtSourceBoundaries(id, lon_min, lon_max, lat_min, lat_max)

        # Now init the HAWC LIFF software

        try:

            # Load all sky
            # (ROI will be defined later)

            if self._n_transits is None:

                self.theLikeHAWC = liff_3ML.LikeHAWC(self.maptree,
                                                     self.response,
                                                     self.pymodel,
                                                     self.minChannel,
                                                     self.maxChannel,
                                                     self.fullsky)

            else:

                self.theLikeHAWC = liff_3ML.LikeHAWC(self.maptree,
                                                     self._n_transits,
                                                     self.response,
                                                     self.pymodel,
                                                     self.minChannel,
                                                     self.maxChannel,
                                                     self.fullsky)

            if self.roi_ra is None and self.fullsky:
                raise RuntimeError("You have to define a ROI with the setROI method")

            if self.roi_ra is not None and self.fullsky:
                self.theLikeHAWC.SetROI(self.roi_ra, self.roi_dec, self.roi_radius, self.fixedROI)

        except:

            print("Could not instance the LikeHAWC class from LIFF. " +
                  "Check that HAWC software is working")

            raise

        else:

            self.instanced = True

        # Now set a callback in the CommonNorm parameter, so that if the user or the fit
        # engine or the Bayesian sampler change the CommonNorm value, the change will be
        # propagated to the LikeHAWC instance

        self.nuisanceParameters['CommonNorm'].add_callback(self._CommonNormCallback)

        # Update to start the computation of positions and energies inside LiFF

        self.theLikeHAWC.UpdateSources()

        # Get the energies needed by LiFF (the same for all sources)

        self._energies = self.theLikeHAWC.GetEnergies(False)

    def _CommonNormCallback(self, value):

        self.theLikeHAWC.SetCommonNorm(value)

    def get_name(self):
        '''
        Return a name for this dataset (likely set during the constructor)
        '''
        return self.name

    def activate_CommonNorm(self):

        self.fitCommonNorm = True

    def deactivate_CommonNorm(self):

        self.fitCommonNorm = False

    def _fill_model_cache(self):

        n_extended = self.model.get_number_of_extended_sources()

        for id in range(n_extended):

            # Get the positions for this extended source
            positions = np.array(self.theLikeHAWC.GetPositions(id, False))

            ras = positions[:, 0]
            decs = positions[:, 1]

            # Get the energies for this extended source

            cube = self.model.get_extended_source_fluxes(self, id, ras, decs, self._energies)

            # We need to multiply by 1000 because the cube is in "per keV" while
            # LiFF needs "per MeV"

            self.pymodel.setExtSourceCube(id, cube * 1000.0, ras, decs)

    def get_log_like(self):

        '''
        Return the value of the log-likelihood with the current values for the
        parameters
        '''

        self.pymodel.update()

        self._fill_model_cache()

        logL = self.theLikeHAWC.getLogLike(self.fitCommonNorm)

        return logL

    def calc_TS(self):

        '''
        Return the value of the log-likelihood test statistic, defined as
        2*[log(LL_model) - log(LL_bkg)]
        '''

        self.pymodel.update()

        TS = self.theLikeHAWC.calcTS(self.fitCommonNorm)

        return TS

    def get_nuisance_parameters(self):
        '''
        Return a list of nuisance parameters. Return an empty list if there
        are no nuisance parameters
        '''

        return self.nuisanceParameters.keys()

    def inner_fit(self):

        self.theLikeHAWC.SetBackgroundNormFree(self.fitCommonNorm)

        self.pymodel.update()

        logL = self.theLikeHAWC.getLogLike(self.fitCommonNorm)

        self.nuisanceParameters['CommonNorm'].value = self.theLikeHAWC.CommonNorm()

        return logL

    def display(self, radius=0.5):

        figs = []

        nsrc = self.model.get_number_of_point_sources()

        for srcid in range(nsrc):
            ra, dec = self.model.get_point_source_position(srcid)

            model = np.array(self.theLikeHAWC.GetTopHatExpectedExcesses(ra, dec, radius))

            signal = np.array(self.theLikeHAWC.GetTopHatExcesses(ra, dec, radius))

            bkg = np.array(self.theLikeHAWC.GetTopHatBackgrounds(ra, dec, radius))

            total = signal + bkg

            fig = plt.figure()

            gs = gridspec.GridSpec(2, 1, height_ratios=[2, 1])
            gs.update(hspace=0)

            sub = plt.subplot(gs[0])

            nHitBins = np.arange(self.minChannel, self.maxChannel + 1)

            sub.errorbar(nHitBins, total, yerr=np.sqrt(total),
                         capsize=0, color='black', label='Observation',
                         fmt='.')

            sub.plot(nHitBins, model + bkg, label='Model + bkg')

            plt.legend(bbox_to_anchor=(1.05, 1), loc=2, numpoints=1)

            # Residuals

            sub1 = plt.subplot(gs[1])

            # Using model variance to account for low statistic

            resid = (signal - model) / model

            sub1.axhline(0, linestyle='--')

            sub1.errorbar(nHitBins, resid,
                          yerr=np.sqrt(total) / model,
                          capsize=0, fmt='.')

            sub.set_xlim([nHitBins.min() - 0.5, nHitBins.max() + 0.5])

            sub.set_yscale("log", nonposy='clip')

            sub.set_ylabel("Counts per bin")

            # sub1.set_xscale("log")

            sub1.set_xlabel("Analysis bin")

            sub1.set_ylabel(r"$\frac{excess - mod.}{mod.}$", fontsize=20)

            sub1.set_xlim([nHitBins.min() - 0.5, nHitBins.max() + 0.5])

            sub.set_xticks([])
            sub1.set_xticks(nHitBins)

            figs.append(fig)

        return figs

    def write_model_map(self, fileName):

        self.theLikeHAWC.WriteModelMap(fileName)

    def write_residual_map(self, fileName):

        self.theLikeHAWC.WriteResidualMap(fileName)
<commit_msg>Implementing the new pyToCpp interface<commit_after>import collections
import os
import sys

import matplotlib.pyplot as plt
import numpy as np
from astromodels import Parameter
from cthreeML.pyModelInterfaceCache import pyToCppModelInterfaceCache
from hawc import liff_3ML
from matplotlib import gridspec

from threeML.io.file_utils import file_existing_and_readable, sanitize_filename
from threeML.plugin_prototype import PluginPrototype

defaultMinChannel = 0
defaultMaxChannel = 9

__instrument_name = "HAWC"

class HAWCLike(PluginPrototype):

    def __init__(self, name, maptree, response, n_transits=None, **kwargs):

        # This controls if the likeHAWC class should load the entire
        # map or just a small disc around a source (faster).
        # Default is the latter, which is way faster. LIFF will decide
        # autonomously which ROI to use depending on the source model

        self.fullsky = False

        if 'fullsky' in kwargs.keys():
            self.fullsky = bool(kwargs['fullsky'])

        self.name = str(name)

        # Sanitize files in input (expand variables and so on)

        self.maptree = os.path.abspath(sanitize_filename(maptree))

        self.response = os.path.abspath(sanitize_filename(response))

        # Check that they exists and can be read

        if not file_existing_and_readable(self.maptree):
            raise IOError("MapTree %s does not exist or is not readable" % maptree)

        if not file_existing_and_readable(self.response):
            raise IOError("Response %s does not exist or is not readable" % response)

        # Post-pone the creation of the LIFF instance to when
        # we have the likelihood model

        self.instanced = False

        # Number of transits
        if n_transits is not None:

            self._n_transits = float(n_transits)

        else:

            self._n_transits = None

            # Default value for minChannel and maxChannel

        self.minChannel = int(defaultMinChannel)
        self.maxChannel = int(defaultMaxChannel)

        # By default the fit of the CommonNorm is deactivated

        self.deactivate_CommonNorm()

        # This is to keep track of whether the user defined a ROI or not

        self.roi_ra = None

        # Further setup

        self.__setup()

    def set_ROI(self, ra, dec, radius, fixedROI=False):

        self.roi_ra = ra
        self.roi_dec = dec

        self.roi_radius = radius

        self.fixedROI = fixedROI

    def __setup(self):

        # I put this here so I can use it both from the __init__ both from
        # the __setstate__ methods

        # Create the dictionary of nuisance parameters

        self.nuisanceParameters = collections.OrderedDict()
        self.nuisanceParameters['CommonNorm'] = Parameter("CommonNorm", 1.0, min_value=0.5, max_value=1.5,
                                                          delta=0.01)
        self.nuisanceParameters['CommonNorm'].fix = True

    def __getstate__(self):

        # This method is used by pickle before attempting to pickle the class

        # Return only the objects needed to recreate the class
        # IN particular, we do NOT return the theLikeHAWC class,
        # which is not pickeable. It will instead be recreated
        # on the other side

        d = {}

        d['name'] = self.name
        d['maptree'] = self.maptree
        d['response'] = self.response
        d['model'] = self.model
        d['n_transits'] = self._n_transits
        d['minChannel'] = self.minChannel
        d['maxChannel'] = self.maxChannel

        d['roi_ra'] = self.roi_ra

        if self.roi_ra is not None:
            d['roi_dec'] = self.roi_dec
            d['roi_radius'] = self.roi_radius

        return d

    def __setstate__(self, state):

        # This is used by pickle to recreate the class on the remote
        # side
        name = state['name']
        maptree = state['maptree']
        response = state['response']
        ntransits = state['n_transits']

        self._n_transits = ntransits

        # Now report the class to its state

        self.__init__(name, maptree, response)

        if state['roi_ra'] is not None:
            self.set_ROI(state['roi_ra'], state['roi_dec'], state['roi_radius'], state['fixedROI'])

        self.set_active_measurements(state['minChannel'], state['maxChannel'])

        self.set_model(state['model'])

    def set_active_measurements(self, minChannel, maxChannel):

        self.minChannel = int(minChannel)
        self.maxChannel = int(maxChannel)

        if self.instanced:
            sys.stderr.write("Since the plugins was already used before, the change in active measurements" +
                             "will not be effective until you create a new JointLikelihood or Bayesian" +
                             "instance")

    def set_model(self, likelihood_model_instance):
        """
        Set the model to be used in the joint minimization. Must be a LikelihoodModel instance.
        """

        # Instance the python - C++ bridge

        self.model = likelihood_model_instance

        self.pymodel = pyToCppModelInterfaceCache()

        # Set boundaries for extended source
        # NOTE: we assume that these boundaries do not change during the fit

        for id in range(self.model.get_number_of_extended_sources()):

            lon_min, lon_max, lat_min, lat_max = self.model.get_extended_source_boundaries(id)

            self.pymodel.setExtSourceBoundaries(id, lon_min, lon_max, lat_min, lat_max)

        # Now init the HAWC LIFF software

        try:

            # Load all sky
            # (ROI will be defined later)

            if self._n_transits is None:

                self.theLikeHAWC = liff_3ML.LikeHAWC(self.maptree,
                                                     self.response,
                                                     self.pymodel,
                                                     self.minChannel,
                                                     self.maxChannel,
                                                     self.fullsky)

            else:

                self.theLikeHAWC = liff_3ML.LikeHAWC(self.maptree,
                                                     self._n_transits,
                                                     self.response,
                                                     self.pymodel,
                                                     self.minChannel,
                                                     self.maxChannel,
                                                     self.fullsky)

            if self.roi_ra is None and self.fullsky:
                raise RuntimeError("You have to define a ROI with the setROI method")

            if self.roi_ra is not None and self.fullsky:
                self.theLikeHAWC.SetROI(self.roi_ra, self.roi_dec, self.roi_radius, self.fixedROI)

        except:

            print("Could not instance the LikeHAWC class from LIFF. " +
                  "Check that HAWC software is working")

            raise

        else:

            self.instanced = True

        # Now set a callback in the CommonNorm parameter, so that if the user or the fit
        # engine or the Bayesian sampler change the CommonNorm value, the change will be
        # propagated to the LikeHAWC instance

        self.nuisanceParameters['CommonNorm'].add_callback(self._CommonNormCallback)

        # Update to start the computation of positions and energies inside LiFF

        self.theLikeHAWC.UpdateSources()

        # Get the energies needed by LiFF (the same for all sources)

        self._energies = self.theLikeHAWC.GetEnergies(False)

    def _CommonNormCallback(self, value):

        self.theLikeHAWC.SetCommonNorm(value)

    def get_name(self):
        '''
        Return a name for this dataset (likely set during the constructor)
        '''
        return self.name

    def activate_CommonNorm(self):

        self.fitCommonNorm = True

    def deactivate_CommonNorm(self):

        self.fitCommonNorm = False

    def _fill_model_cache(self):

        n_extended = self.model.get_number_of_extended_sources()

        for id in range(n_extended):

            # Get the positions for this extended source
            positions = np.array(self.theLikeHAWC.GetPositions(id, False))

            ras = positions[:, 0]
            decs = positions[:, 1]

            # Get the energies for this extended source

            cube = self.model.get_extended_source_fluxes(self, id, ras, decs, self._energies)

            # We need to multiply by 1000 because the cube is in "per keV" while
            # LiFF needs "per MeV"

            self.pymodel.setExtSourceCube(id, cube * 1000.0, ras, decs)

    def get_log_like(self):

        '''
        Return the value of the log-likelihood with the current values for the
        parameters
        '''

        self.pymodel.reset()

        self._fill_model_cache()

        logL = self.theLikeHAWC.getLogLike(self.fitCommonNorm)

        return logL

    def calc_TS(self):

        '''
        Return the value of the log-likelihood test statistic, defined as
        2*[log(LL_model) - log(LL_bkg)]
        '''

        self.pymodel.update()

        TS = self.theLikeHAWC.calcTS(self.fitCommonNorm)

        return TS

    def get_nuisance_parameters(self):
        '''
        Return a list of nuisance parameters. Return an empty list if there
        are no nuisance parameters
        '''

        return self.nuisanceParameters.keys()

    def inner_fit(self):

        self.theLikeHAWC.SetBackgroundNormFree(self.fitCommonNorm)

        self.pymodel.update()

        logL = self.theLikeHAWC.getLogLike(self.fitCommonNorm)

        self.nuisanceParameters['CommonNorm'].value = self.theLikeHAWC.CommonNorm()

        return logL

    def display(self, radius=0.5):

        figs = []

        nsrc = self.model.get_number_of_point_sources()

        for srcid in range(nsrc):
            ra, dec = self.model.get_point_source_position(srcid)

            model = np.array(self.theLikeHAWC.GetTopHatExpectedExcesses(ra, dec, radius))

            signal = np.array(self.theLikeHAWC.GetTopHatExcesses(ra, dec, radius))

            bkg = np.array(self.theLikeHAWC.GetTopHatBackgrounds(ra, dec, radius))

            total = signal + bkg

            fig = plt.figure()

            gs = gridspec.GridSpec(2, 1, height_ratios=[2, 1])
            gs.update(hspace=0)

            sub = plt.subplot(gs[0])

            nHitBins = np.arange(self.minChannel, self.maxChannel + 1)

            sub.errorbar(nHitBins, total, yerr=np.sqrt(total),
                         capsize=0, color='black', label='Observation',
                         fmt='.')

            sub.plot(nHitBins, model + bkg, label='Model + bkg')

            plt.legend(bbox_to_anchor=(1.05, 1), loc=2, numpoints=1)

            # Residuals

            sub1 = plt.subplot(gs[1])

            # Using model variance to account for low statistic

            resid = (signal - model) / model

            sub1.axhline(0, linestyle='--')

            sub1.errorbar(nHitBins, resid,
                          yerr=np.sqrt(total) / model,
                          capsize=0, fmt='.')

            sub.set_xlim([nHitBins.min() - 0.5, nHitBins.max() + 0.5])

            sub.set_yscale("log", nonposy='clip')

            sub.set_ylabel("Counts per bin")

            # sub1.set_xscale("log")

            sub1.set_xlabel("Analysis bin")

            sub1.set_ylabel(r"$\frac{excess - mod.}{mod.}$", fontsize=20)

            sub1.set_xlim([nHitBins.min() - 0.5, nHitBins.max() + 0.5])

            sub.set_xticks([])
            sub1.set_xticks(nHitBins)

            figs.append(fig)

        return figs

    def write_model_map(self, fileName):

        self.theLikeHAWC.WriteModelMap(fileName)

    def write_residual_map(self, fileName):

        self.theLikeHAWC.WriteResidualMap(fileName)
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

# thumbor imaging service
# https://github.com/thumbor/thumbor/wiki

# Licensed under the MIT license:
# http://www.opensource.org/licenses/mit-license
# Copyright (c) 2011 globo.com timehome@corp.globo.com

from pexif import ExifSegment

try:
    import cairosvg
except:
    cairosvg = None

try:
    from pyexiv2 import ImageMetadata
    METADATA_AVAILABLE = True
except ImportError:
    METADATA_AVAILABLE = False

from thumbor.utils import logger, EXTENSION

WEBP_SIDE_LIMIT = 16383

class MultipleEngine:

    def __init__(self, source_engine):
        self.frame_engines = []
        self.source_engine = source_engine

    def add_frame(self, frame):
        frame_engine = self.source_engine.__class__(self.source_engine.context)
        frame_engine.extension = self.source_engine.extension
        frame_engine.source_width = self.source_engine.source_width
        frame_engine.source_height = self.source_engine.source_height
        frame_engine.image = frame

        self.frame_engines.append(frame_engine)

    def read(self, extension=None, quality=None):
        return self.source_engine.read_multiple(
            [frame_engine.image for frame_engine in self.frame_engines],
            extension)

    def size(self):
        return self.frame_engines[0].size

    def do_many(self, name):
        def exec_func(*args, **kwargs):
            result = []
            for frame_engine in self.frame_engines:
                result.append(getattr(frame_engine, name)(*args, **kwargs))
            return result
        return exec_func

class BaseEngine(object):

    def __init__(self, context):
        self.context = context
        self.image = None
        self.extension = None
        self.source_width = None
        self.source_height = None
        self.icc_profile = None
        self.frame_count = 1
        self.metadata = None

    @classmethod
    def get_mimetype(cls, buffer):
        mime = None

        # magic number detection
        if buffer.startswith('GIF8'):
            mime = 'image/gif'
        elif buffer.startswith('\x89PNG\r\n\x1a\n'):
            mime = 'image/png'
        elif buffer.startswith('\xff\xd8'):
            mime = 'image/jpeg'
        elif buffer.startswith('WEBP', 8):
            mime = 'image/webp'
        elif buffer.startswith('\x00\x00\x00\x0c'):
            mime = 'image/jp2'
        elif buffer.startswith('\x00\x00\x00 ftyp'):
            mime = 'video/mp4'
        elif buffer.startswith('\x1aE\xdf\xa3'):
            mime = 'video/webm'
        elif buffer.startswith('<svg'):
            mime = 'image/svg+xml'
        return mime

    def wrap(self, multiple_engine):
        for method_name in ['resize', 'crop', 'flip_vertically',
                            'flip_horizontally']:
            setattr(self, method_name, multiple_engine.do_many(method_name))
        setattr(self, 'read', multiple_engine.read)

    def is_multiple(self):
        return hasattr(self, 'multiple_engine') \
               and self.multiple_engine is not None

    def frame_engines(self):
        return self.multiple_engine.frame_engines

    def convert_svg_to_png(self, buffer):
        if not cairosvg:
            msg = """[BaseEngine] convert_svg_to_png failed cairosvg not
            imported (if you want svg conversion to png please install cairosvg)
            """
            logger.error(msg)
            return buffer

        buffer = cairosvg.svg2png(bytestring=buffer)
        mime = self.get_mimetype(buffer)
        self.extension = EXTENSION.get(mime, '.jpg')
        self.transformed_body = buffer
        return buffer

    def load(self, buffer, extension):
        self.extension = extension

        if extension is None:
            mime = self.get_mimetype(buffer)
            self.extension = EXTENSION.get(mime, '.jpg')

        if self.extension == '.svg':
            buffer = self.convert_svg_to_png(buffer)

        image_or_frames = self.create_image(buffer)

        if METADATA_AVAILABLE:
            try:
                self.metadata = ImageMetadata.from_buffer(buffer)
                self.metadata.read()
            except Exception as e:
                logger.error('Error reading image metadata: %s' % e)

        if self.context.config.ALLOW_ANIMATED_GIFS and isinstance(
                image_or_frames, (list, tuple)):
            self.image = image_or_frames[0]
            if len(image_or_frames) > 1:
                self.multiple_engine = MultipleEngine(self)
                for frame in image_or_frames:
                    self.multiple_engine.add_frame(frame)
                self.wrap(self.multiple_engine)
        else:
            self.image = image_or_frames

        if self.source_width is None:
            self.source_width = self.size[0]
        if self.source_height is None:
            self.source_height = self.size[1]

    @property
    def size(self):
        if self.is_multiple():
            return self.multiple_engine.size()
        return self.image.size

    def can_convert_to_webp(self):
        return self.size[0] <= WEBP_SIDE_LIMIT \
               and self.size[1] <= WEBP_SIDE_LIMIT

    def normalize(self):
        width, height = self.size
        self.source_width = width
        self.source_height = height

        if width > self.context.config.MAX_WIDTH \
                or height > self.context.config.MAX_HEIGHT:
            width_diff = width - self.context.config.MAX_WIDTH
            height_diff = height - self.context.config.MAX_HEIGHT
            if self.context.config.MAX_WIDTH and width_diff > height_diff:
                height = self.get_proportional_height(
                        self.context.config.MAX_WIDTH)
                self.resize(self.context.config.MAX_WIDTH, height)
                return True
            elif self.context.config.MAX_HEIGHT and height_diff > width_diff:
                width = self.get_proportional_width(
                        self.context.config.MAX_HEIGHT)
                self.resize(width, self.context.config.MAX_HEIGHT)
                return True

        return False

    def get_proportional_width(self, new_height):
        width, height = self.size
        return round(float(new_height) * width / height, 0)

    def get_proportional_height(self, new_width):
        width, height = self.size
        return round(float(new_width) * height / width, 0)

    def _get_exif_segment(self):
        try:
            segment = ExifSegment(None, None, self.exif, 'ro')
        except Exception:
            logger.exception('Ignored error handling exif for reorientation')
        else:
            return segment
        return None

    def get_orientation(self):
        """
        Returns the image orientation of the buffer image or None
        if it is undefined. Gets the original value from the Exif tag.
        If the buffer has been rotated, then the value is adjusted to 1.
        :return: Orientation value (1 - 8)
        :rtype: int or None
        """
        if (not hasattr(self, 'exif')) or self.exif is None:
            return None

        segment = self._get_exif_segment()
        if segment:
            orientation = segment.primary['Orientation']
            if orientation:
                return orientation[0]
        return None

    def reorientate(self, override_exif=True):
        """
        Rotates the image in the buffer so that it is oriented correctly.
        If override_exif is True (default) then the metadata
        orientation is adjusted as well.
        :param override_exif: If the metadata should be adjusted as well.
        :type override_exif: Boolean
        """

        orientation = self.get_orientation()

        if orientation == 2:
            self.flip_horizontally()
        elif orientation == 3:
            self.rotate(180)
        elif orientation == 4:
            self.flip_vertically()
        elif orientation == 5:
            # Horizontal Mirror + Rotation 270 CCW
            self.flip_vertically()
            self.rotate(270)
        elif orientation == 6:
            self.rotate(270)
        elif orientation == 7:
            # Vertical Mirror + Rotation 270 CCW
            self.flip_horizontally()
            self.rotate(270)
        elif orientation == 8:
            self.rotate(90)

        if orientation != 1 and override_exif:
            segment = self._get_exif_segment()
            if segment:
                segment.primary['Orientation'] = [1]
                self.exif = segment.get_data()

    def gen_image(self):
        raise NotImplementedError()

    def create_image(self):
        raise NotImplementedError()

    def crop(self):
        raise NotImplementedError()

    def resize(self):
        raise NotImplementedError()

    def focus(self, points):
        pass

    def flip_horizontally(self):
        raise NotImplementedError()

    def flip_vertically(self):
        raise NotImplementedError()

    def rotate(self, amount):
        """
        Rotates the image the given amount CCW.
        :param amount: Amount to rotate in degrees.
        :type amount: int
        """
        pass

    def read(self, extension, quality):
        raise NotImplementedError()

    def get_image_data(self):
        raise NotImplementedError()

    def set_image_data(self, data):
        raise NotImplementedError()

    def get_image_mode(self):
        """ Possible return values should be: RGB, RBG, GRB, GBR,
            BRG, BGR, RGBA, AGBR, ...  """
        raise NotImplementedError()

    def paste(self):
        raise NotImplementedError()

    def enable_alpha(self):
        raise NotImplementedError()

    def image_data_as_rgb(self, update_image=True):
        raise NotImplementedError()

    def strip_icc(self):
        pass

    def extract_cover(self):
        raise NotImplementedError()
<commit_msg>Remove spuprious instance variable<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

# thumbor imaging service
# https://github.com/thumbor/thumbor/wiki

# Licensed under the MIT license:
# http://www.opensource.org/licenses/mit-license
# Copyright (c) 2011 globo.com timehome@corp.globo.com

from pexif import ExifSegment

try:
    import cairosvg
except:
    cairosvg = None

try:
    from pyexiv2 import ImageMetadata
    METADATA_AVAILABLE = True
except ImportError:
    METADATA_AVAILABLE = False

from thumbor.utils import logger, EXTENSION

WEBP_SIDE_LIMIT = 16383

class MultipleEngine:

    def __init__(self, source_engine):
        self.frame_engines = []
        self.source_engine = source_engine

    def add_frame(self, frame):
        frame_engine = self.source_engine.__class__(self.source_engine.context)
        frame_engine.extension = self.source_engine.extension
        frame_engine.source_width = self.source_engine.source_width
        frame_engine.source_height = self.source_engine.source_height
        frame_engine.image = frame

        self.frame_engines.append(frame_engine)

    def read(self, extension=None, quality=None):
        return self.source_engine.read_multiple(
            [frame_engine.image for frame_engine in self.frame_engines],
            extension)

    def size(self):
        return self.frame_engines[0].size

    def do_many(self, name):
        def exec_func(*args, **kwargs):
            result = []
            for frame_engine in self.frame_engines:
                result.append(getattr(frame_engine, name)(*args, **kwargs))
            return result
        return exec_func

class BaseEngine(object):

    def __init__(self, context):
        self.context = context
        self.image = None
        self.extension = None
        self.source_width = None
        self.source_height = None
        self.icc_profile = None
        self.frame_count = 1
        self.metadata = None

    @classmethod
    def get_mimetype(cls, buffer):
        mime = None

        # magic number detection
        if buffer.startswith('GIF8'):
            mime = 'image/gif'
        elif buffer.startswith('\x89PNG\r\n\x1a\n'):
            mime = 'image/png'
        elif buffer.startswith('\xff\xd8'):
            mime = 'image/jpeg'
        elif buffer.startswith('WEBP', 8):
            mime = 'image/webp'
        elif buffer.startswith('\x00\x00\x00\x0c'):
            mime = 'image/jp2'
        elif buffer.startswith('\x00\x00\x00 ftyp'):
            mime = 'video/mp4'
        elif buffer.startswith('\x1aE\xdf\xa3'):
            mime = 'video/webm'
        elif buffer.startswith('<svg'):
            mime = 'image/svg+xml'
        return mime

    def wrap(self, multiple_engine):
        for method_name in ['resize', 'crop', 'flip_vertically',
                            'flip_horizontally']:
            setattr(self, method_name, multiple_engine.do_many(method_name))
        setattr(self, 'read', multiple_engine.read)

    def is_multiple(self):
        return hasattr(self, 'multiple_engine') \
               and self.multiple_engine is not None

    def frame_engines(self):
        return self.multiple_engine.frame_engines

    def convert_svg_to_png(self, buffer):
        if not cairosvg:
            msg = """[BaseEngine] convert_svg_to_png failed cairosvg not
            imported (if you want svg conversion to png please install cairosvg)
            """
            logger.error(msg)
            return buffer

        buffer = cairosvg.svg2png(bytestring=buffer)
        mime = self.get_mimetype(buffer)
        self.extension = EXTENSION.get(mime, '.jpg')
        return buffer

    def load(self, buffer, extension):
        self.extension = extension

        if extension is None:
            mime = self.get_mimetype(buffer)
            self.extension = EXTENSION.get(mime, '.jpg')

        if self.extension == '.svg':
            buffer = self.convert_svg_to_png(buffer)

        image_or_frames = self.create_image(buffer)

        if METADATA_AVAILABLE:
            try:
                self.metadata = ImageMetadata.from_buffer(buffer)
                self.metadata.read()
            except Exception as e:
                logger.error('Error reading image metadata: %s' % e)

        if self.context.config.ALLOW_ANIMATED_GIFS and isinstance(
                image_or_frames, (list, tuple)):
            self.image = image_or_frames[0]
            if len(image_or_frames) > 1:
                self.multiple_engine = MultipleEngine(self)
                for frame in image_or_frames:
                    self.multiple_engine.add_frame(frame)
                self.wrap(self.multiple_engine)
        else:
            self.image = image_or_frames

        if self.source_width is None:
            self.source_width = self.size[0]
        if self.source_height is None:
            self.source_height = self.size[1]

    @property
    def size(self):
        if self.is_multiple():
            return self.multiple_engine.size()
        return self.image.size

    def can_convert_to_webp(self):
        return self.size[0] <= WEBP_SIDE_LIMIT \
               and self.size[1] <= WEBP_SIDE_LIMIT

    def normalize(self):
        width, height = self.size
        self.source_width = width
        self.source_height = height

        if width > self.context.config.MAX_WIDTH \
                or height > self.context.config.MAX_HEIGHT:
            width_diff = width - self.context.config.MAX_WIDTH
            height_diff = height - self.context.config.MAX_HEIGHT
            if self.context.config.MAX_WIDTH and width_diff > height_diff:
                height = self.get_proportional_height(
                        self.context.config.MAX_WIDTH)
                self.resize(self.context.config.MAX_WIDTH, height)
                return True
            elif self.context.config.MAX_HEIGHT and height_diff > width_diff:
                width = self.get_proportional_width(
                        self.context.config.MAX_HEIGHT)
                self.resize(width, self.context.config.MAX_HEIGHT)
                return True

        return False

    def get_proportional_width(self, new_height):
        width, height = self.size
        return round(float(new_height) * width / height, 0)

    def get_proportional_height(self, new_width):
        width, height = self.size
        return round(float(new_width) * height / width, 0)

    def _get_exif_segment(self):
        try:
            segment = ExifSegment(None, None, self.exif, 'ro')
        except Exception:
            logger.exception('Ignored error handling exif for reorientation')
        else:
            return segment
        return None

    def get_orientation(self):
        """
        Returns the image orientation of the buffer image or None
        if it is undefined. Gets the original value from the Exif tag.
        If the buffer has been rotated, then the value is adjusted to 1.
        :return: Orientation value (1 - 8)
        :rtype: int or None
        """
        if (not hasattr(self, 'exif')) or self.exif is None:
            return None

        segment = self._get_exif_segment()
        if segment:
            orientation = segment.primary['Orientation']
            if orientation:
                return orientation[0]
        return None

    def reorientate(self, override_exif=True):
        """
        Rotates the image in the buffer so that it is oriented correctly.
        If override_exif is True (default) then the metadata
        orientation is adjusted as well.
        :param override_exif: If the metadata should be adjusted as well.
        :type override_exif: Boolean
        """

        orientation = self.get_orientation()

        if orientation == 2:
            self.flip_horizontally()
        elif orientation == 3:
            self.rotate(180)
        elif orientation == 4:
            self.flip_vertically()
        elif orientation == 5:
            # Horizontal Mirror + Rotation 270 CCW
            self.flip_vertically()
            self.rotate(270)
        elif orientation == 6:
            self.rotate(270)
        elif orientation == 7:
            # Vertical Mirror + Rotation 270 CCW
            self.flip_horizontally()
            self.rotate(270)
        elif orientation == 8:
            self.rotate(90)

        if orientation != 1 and override_exif:
            segment = self._get_exif_segment()
            if segment:
                segment.primary['Orientation'] = [1]
                self.exif = segment.get_data()

    def gen_image(self):
        raise NotImplementedError()

    def create_image(self):
        raise NotImplementedError()

    def crop(self):
        raise NotImplementedError()

    def resize(self):
        raise NotImplementedError()

    def focus(self, points):
        pass

    def flip_horizontally(self):
        raise NotImplementedError()

    def flip_vertically(self):
        raise NotImplementedError()

    def rotate(self, amount):
        """
        Rotates the image the given amount CCW.
        :param amount: Amount to rotate in degrees.
        :type amount: int
        """
        pass

    def read(self, extension, quality):
        raise NotImplementedError()

    def get_image_data(self):
        raise NotImplementedError()

    def set_image_data(self, data):
        raise NotImplementedError()

    def get_image_mode(self):
        """ Possible return values should be: RGB, RBG, GRB, GBR,
            BRG, BGR, RGBA, AGBR, ...  """
        raise NotImplementedError()

    def paste(self):
        raise NotImplementedError()

    def enable_alpha(self):
        raise NotImplementedError()

    def image_data_as_rgb(self, update_image=True):
        raise NotImplementedError()

    def strip_icc(self):
        pass

    def extract_cover(self):
        raise NotImplementedError()
<|endoftext|>


<commit_before># Copyright 2016 The Closure Rules Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Utilities for building JavaScript Protocol Buffers.
"""

load("//closure/compiler:closure_js_library.bzl", "closure_js_library")

def closure_js_proto_library(
    name,
    srcs,
    suppress = [],
    add_require_for_enums = 0,
    testonly = None,
    binary = 1,
    import_style = None,
    protocbin = Label("//third_party/protobuf:protoc"),
    **kwargs):
  cmd = ["$(location %s)" % protocbin]
  js_out_options = ["library=%s,error_on_name_conflict" % name]
  if add_require_for_enums:
    js_out_options += ["add_require_for_enums"]
  if testonly:
    js_out_options += ["testonly"]
  if binary:
    js_out_options += ["binary"]
  if import_style:
    js_out_options += ["import_style=%s" % import_style]
  cmd += ["--js_out=%s:$(@D)" % ",".join(js_out_options)]
  cmd += ["--descriptor_set_out=$(@D)/%s.descriptor" % name]
  cmd += ["$(locations " + src + ")" for src in srcs]

  native.genrule(
      name = name + "_gen",
      srcs = srcs,
      testonly = testonly,
      visibility = ["//visibility:private"],
      message = "Generating JavaScript Protocol Buffer file",
      outs = [name + ".js", name + ".descriptor"],
      tools = [protocbin],
      cmd = " ".join(cmd),
  )

  closure_js_library(
      name = name,
      srcs = [name + ".js"],
      testonly = testonly,
      deps = [
          str(Label("//closure/library")),
          str(Label("//closure/protobuf:jspb")),
      ],
      internal_descriptors = [name + ".descriptor"],
      suppress = suppress + [
          "analyzerChecks",
          "missingProperties",
          "reportUnknownTypes",
      ],
      **kwargs
  )
<commit_msg>Ignore unusedLocalVariables in generated JS proto<commit_after># Copyright 2016 The Closure Rules Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Utilities for building JavaScript Protocol Buffers.
"""

load("//closure/compiler:closure_js_library.bzl", "closure_js_library")

def closure_js_proto_library(
    name,
    srcs,
    suppress = [],
    add_require_for_enums = 0,
    testonly = None,
    binary = 1,
    import_style = None,
    protocbin = Label("//third_party/protobuf:protoc"),
    **kwargs):
  cmd = ["$(location %s)" % protocbin]
  js_out_options = ["library=%s,error_on_name_conflict" % name]
  if add_require_for_enums:
    js_out_options += ["add_require_for_enums"]
  if testonly:
    js_out_options += ["testonly"]
  if binary:
    js_out_options += ["binary"]
  if import_style:
    js_out_options += ["import_style=%s" % import_style]
  cmd += ["--js_out=%s:$(@D)" % ",".join(js_out_options)]
  cmd += ["--descriptor_set_out=$(@D)/%s.descriptor" % name]
  cmd += ["$(locations " + src + ")" for src in srcs]

  native.genrule(
      name = name + "_gen",
      srcs = srcs,
      testonly = testonly,
      visibility = ["//visibility:private"],
      message = "Generating JavaScript Protocol Buffer file",
      outs = [name + ".js", name + ".descriptor"],
      tools = [protocbin],
      cmd = " ".join(cmd),
  )

  closure_js_library(
      name = name,
      srcs = [name + ".js"],
      testonly = testonly,
      deps = [
          str(Label("//closure/library")),
          str(Label("//closure/protobuf:jspb")),
      ],
      internal_descriptors = [name + ".descriptor"],
      suppress = suppress + [
          "analyzerChecks",
          "missingProperties",
          "reportUnknownTypes",
          "unusedLocalVariables",
      ],
      **kwargs
  )
<|endoftext|>


<commit_before>__author__ = 'artanis'

from constants import CONTENT_TYPE_HEADER, AUTHENTICATION_HEADER, DEFAULT_CONTENT_TYPE_HEADER, RULE_ACTION, \
    RULE_CONDITION, RULE_NAME, RULE_CONDITION_DEFAULT, RULE_ACTION_DEFAULT, LONG_NAME, RULE_ID
from errors import FAULT_ELEMENT_ERROR, ERROR_CODE_ERROR
from nose.tools import assert_in, assert_equals
import string
import random

list_deletions = [None, u'null']

def create_header(content_type=DEFAULT_CONTENT_TYPE_HEADER, token=None):

    """Method to create the header required to perform requests to the Policy Manager
    :param content_type:  Content type of the HTTP request. By default application/json
    :param token: Token required to policy manager authentication
    :returns: HTTP header (dict)
    """

    header = {CONTENT_TYPE_HEADER: '', AUTHENTICATION_HEADER: ''}

    if content_type in list_deletions:
        del header[CONTENT_TYPE_HEADER]
    else:
        header[CONTENT_TYPE_HEADER] = content_type

    if token in list_deletions:
        del header[AUTHENTICATION_HEADER]
    else:
        header[AUTHENTICATION_HEADER] = token

    return header

def assert_error_code_error(response, expected_error_code=None, expected_fault_element=None):

    """Method to assert response errors from Policy Manger
    :param response: Response obtained from the API REST request to the Policy Manager
    :param expected_fault_element: Expected Fault element in the JSON response
    :param expected_error_code: Expected Error code in the JSON response
    """

    assert_equals(expected_error_code, str(response.status_code),
                  ERROR_CODE_ERROR.format(expected_error_code, str(response.status_code)))
    response_body = response.json()
    assert_in(expected_fault_element, response_body.keys(),
              FAULT_ELEMENT_ERROR.format(expected_fault_element, response_body))
    assert_equals(str(response_body[expected_fault_element]['code']), expected_error_code,
                  ERROR_CODE_ERROR.format(expected_error_code, response_body[expected_fault_element]['code']))

def create_rule_parameters(name=None, condition=None, action=None):

    """Method to create the rule body
    :param name: The name of the rule to be created
    :param condition: the condition to compare the server context
    :param action: the action to take over the server
    """

    if name == 'long_name':
        name = LONG_NAME
    elif name == 'None':
        name = None
    elif name == 'random':
        name = id_generator()

    if condition == 'default':
        condition = RULE_CONDITION_DEFAULT
    elif condition == 'None':
        condition = None

    if action == 'default':
        action = RULE_ACTION_DEFAULT
    elif action == 'None':
        action = None

    return name, condition, action

def id_generator(size=10, chars=string.ascii_letters + string.digits):

    """Method to create random ids
    :param size: define the string size
    :param chars: the characters to be use to create the string
    return ''.join(random.choice(chars) for x in range(size))
    """

    return ''.join(random.choice(chars) for x in range(size))

def assert_json_format(request):

    try:
        response = request.json()
    except ValueError:
        assert False, "JSON Cannot be decode. Response format not correspond with JSON format"

    return response

def assert_rule_information(response, rule_id, name, condition, action):

    assert_equals(response[RULE_NAME], name)
    assert_equals(response[RULE_CONDITION], condition)
    assert_equals(response[RULE_ACTION], action)
    assert_equals(response[RULE_ID], rule_id)
<commit_msg>add comments from @fla in PR<commit_after>__author__ = 'artanis'

from constants import CONTENT_TYPE_HEADER, AUTHENTICATION_HEADER, DEFAULT_CONTENT_TYPE_HEADER, RULE_ACTION, \
    RULE_CONDITION, RULE_NAME, RULE_CONDITION_DEFAULT, RULE_ACTION_DEFAULT, LONG_NAME, RULE_ID
from errors import FAULT_ELEMENT_ERROR, ERROR_CODE_ERROR
from nose.tools import assert_in, assert_equals
import string
import random

list_deletions = [None, u'null']

def create_header(content_type=DEFAULT_CONTENT_TYPE_HEADER, token=None):

    """Method to create the header required to perform requests to the Policy Manager
    :param content_type:  Content type of the HTTP request. By default application/json
    :param token: Token required to policy manager authentication
    :returns: HTTP header (dict)
    """

    header = {CONTENT_TYPE_HEADER: '', AUTHENTICATION_HEADER: ''}

    if content_type in list_deletions:
        del header[CONTENT_TYPE_HEADER]
    else:
        header[CONTENT_TYPE_HEADER] = content_type

    if token in list_deletions:
        del header[AUTHENTICATION_HEADER]
    else:
        header[AUTHENTICATION_HEADER] = token

    return header

def assert_error_code_error(response, expected_error_code=None, expected_fault_element=None):

    """Method to assert response errors from Policy Manger
    :param response: Response obtained from the API REST request to the Policy Manager
    :param expected_fault_element: Expected Fault element in the JSON response
    :param expected_error_code: Expected Error code in the JSON response
    """

    assert_equals(expected_error_code, str(response.status_code),
                  ERROR_CODE_ERROR.format(expected_error_code, str(response.status_code)))
    response_body = response.json()
    assert_in(expected_fault_element, response_body.keys(),
              FAULT_ELEMENT_ERROR.format(expected_fault_element, response_body))
    assert_equals(str(response_body[expected_fault_element]['code']), expected_error_code,
                  ERROR_CODE_ERROR.format(expected_error_code, response_body[expected_fault_element]['code']))

def create_rule_parameters(name=None, condition=None, action=None):

    """Method to create the rule body
    :param name: The name of the rule to be created
    :param condition: the condition to compare the server context
    :param action: the action to take over the server
    """

    if name == 'long_name':
        name = LONG_NAME
    elif name == 'None':
        name = None
    elif name == 'random':
        name = id_generator()

    if condition == 'default':
        condition = RULE_CONDITION_DEFAULT
    elif condition == 'None':
        condition = None

    if action == 'default':
        action = RULE_ACTION_DEFAULT
    elif action == 'None':
        action = None

    return name, condition, action

def id_generator(size=10, chars=string.ascii_letters + string.digits):

    """Method to create random ids
    :param size: define the string size
    :param chars: the characters to be use to create the string
    return ''.join(random.choice(chars) for x in range(size))
    """

    return ''.join(random.choice(chars) for x in range(size))

def assert_json_format(request):

    """"Method to assert the JSON format
    :param request: Object with the response
    :return respose if is JSON compliance
    """

    try:
        response = request.json()
    except ValueError:
        assert False, "JSON Cannot be decode. Response format not correspond with JSON format"

    return response

def assert_rule_information(response, rule_id, name, condition, action):

    """Method to verify the rule body parameters
    :param response: Response body received from server
    :param rule_id: The expected rule identificacion number
    :param name: The expected rule name
    :param condition: The expected rule condition
    :param action: The expected rule action
    """

    assert_equals(response[RULE_NAME], name)
    assert_equals(response[RULE_CONDITION], condition)
    assert_equals(response[RULE_ACTION], action)
    assert_equals(response[RULE_ID], rule_id)
<|endoftext|>


<commit_before>"""
Performs clustering on all three groups of students (successful, probation,
failed) and then on the merged pass/fail data.  Calculates metrics to
determine the quality of each clustering.
"""

from pml.api import *

import util

def cluster_3_groups(data):
    print "Success/Probation/Fail clustering:"
    clustered = kmeans(data, k=3)
    purity = clustered.calculate_purity()
    rand_index = clustered.calculate_rand_index()

    print "Purity = %.5f, Rand index = %.5f" % (purity, rand_index)

def cluster_2_groups(data):
    print "Pass/Fail clustering:"

    # Count successful and probation students as one group (s)
    data.combine_labels(["s", "p"], "s")

    clustered = kmeans(data, k=2)
    purity = clustered.calculate_purity()
    rand_index = clustered.calculate_rand_index()

    print "Purity = %.5f, Rand index = %.5f" % (purity, rand_index)

def cluster_2_groups_with_pca(data):
    cluster_2_groups(pca(data, 2))

def cluster_3_groups_with_pca(data):
    cluster_3_groups(pca(data, 2))

def main():
    # The original data set.
    data = util.load_data()

    # Fill in missing values with the average for that course.
    data.fill_missing_with_feature_means()

    cluster_3_groups(data.copy())
    cluster_2_groups(data.copy())

    util.print_line_break()

    print "Now with PCA:"
    cluster_3_groups_with_pca(data.copy())
    cluster_2_groups_with_pca(data.copy())

if __name__ == "__main__":
    main()
<commit_msg>Added cluster analysis where probation and failed students are grouped together.<commit_after>"""
Performs clustering on all three groups of students (successful, probation,
failed) and then on the merged pass/fail data.  Calculates metrics to
determine the quality of each clustering.
"""

from pml.api import *

import util

def cluster_3_groups(data):
    print "Success/Probation/Fail clustering:"
    clustered = kmeans(data, k=3)
    purity = clustered.calculate_purity()
    rand_index = clustered.calculate_rand_index()

    print "Purity = %.5f, Rand index = %.5f" % (purity, rand_index)

def cluster_pass_fail(data):
    """
    s and p in one group, f in the other
    """
    print "Pass/Fail clustering:"

    # Count successful and probation students as one group (s)
    data.combine_labels(["s", "p"], "s")

    clustered = kmeans(data, k=2)
    purity = clustered.calculate_purity()
    rand_index = clustered.calculate_rand_index()

    print "Purity = %.5f, Rand index = %.5f" % (purity, rand_index)

def cluster_success_struggle(data):
    """
    s in one group, p and f in the other
    """
    print "Success/Struggle clustering:"

    # Count successful and probation students as one group (s)
    data.combine_labels(["p", "f"], "f")

    clustered = kmeans(data, k=2)
    purity = clustered.calculate_purity()
    rand_index = clustered.calculate_rand_index()

    print "Purity = %.5f, Rand index = %.5f" % (purity, rand_index)

def cluster_pass_fail_with_pca(data):
    cluster_pass_fail(pca(data, 2))

def cluster_success_struggle_with_pca(data):
    cluster_success_struggle(pca(data, 2))

def cluster_3_groups_with_pca(data):
    cluster_3_groups(pca(data, 2))

def main():
    # The original data set.
    data = util.load_data()

    # Fill in missing values with the average for that course.
    data.fill_missing_with_feature_means()

    cluster_3_groups(data.copy())
    cluster_pass_fail(data.copy())
    cluster_success_struggle(data.copy())

    util.print_line_break()

    print "Now with PCA:"
    cluster_3_groups_with_pca(data.copy())
    cluster_pass_fail_with_pca(data.copy())
    cluster_success_struggle_with_pca(data.copy())

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># TmLibrary - TissueMAPS library for distibuted image analysis routines.
# Copyright (C) 2016  Markus D. Herrmann, University of Zurich and Robin Hafen
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
import logging
from collections import defaultdict

import tmlib.models as tm
from tmlib.utils import notimplemented
from tmlib.utils import same_docstring_as
from tmlib.errors import NotSupportedError
from tmlib.errors import JobDescriptionError
from tmlib.workflow.align import registration as reg
from tmlib.workflow.api import ClusterRoutines
from tmlib.workflow import register_step_api

logger = logging.getLogger(__name__)

@register_step_api('align')
class ImageRegistrator(ClusterRoutines):

    '''Class for registeration and alignment of images between multiplexing
    cycles.
    '''

    def __init__(self, experiment_id, verbosity):
        '''
        Parameters
        ----------
        experiment_id: int
            ID of the processed experiment
        verbosity: int
            logging level
        '''
        super(ImageRegistrator, self).__init__(experiment_id, verbosity)

    def create_batches(self, args):
        '''Creates job descriptions for parallel computing.

        Parameters
        ----------
        args: tmlib.align.args.AlignInitArgs
            step-specific arguments

        Returns
        -------
        Dict[str, List[dict] or dict]
            job descriptions

        Raises
        ------
        tmlib.errors.NotSupportedError
            when a plate contains only one cycle
        ValueError
            when `args.ref_wavelength` does not exist across all cycles
        '''
        job_count = 0
        job_descriptions = dict()
        job_descriptions['run'] = list()

        with tm.utils.ExperimentSession(self.experiment_id) as session:

            cycles = session.query(tm.Cycle).all()

            if not(len(cycles) > 1):
                raise NotSupportedError(
                    'Alignment requires more than one cycle.'
                )

            if args.ref_cycle >= len(cycles):
                raise JobDescriptionError(
                    'Cycle index must not exceed total number of cycles.'
                )

            site_ids = session.query(tm.Site.id).\
                order_by(tm.Site.id).\
                all()

            batches = self._create_batches(site_ids, args.batch_size)
            for batch in batches:

                job_count += 1
                input_ids = {
                    'reference_file_ids': list(),
                    'target_file_ids': defaultdict(list)
                }

                for cycle in cycles:

                    n = session.query(tm.ChannelImageFile.id).\
                        join(tm.Cycle).\
                        join(tm.Channel).\
                        filter(tm.Cycle.id == cycle.id).\
                        filter(tm.Channel.wavelength == args.ref_wavelength).\
                        filter(~tm.Site.omitted).\
                        count()

                    if n == 0:
                        raise ValueError(
                            'No image files found for cycle %d and '
                            'wavelength "%s"'
                            % (cycle.id, args.ref_wavelength)
                        )

                    for s in batch:

                        files = session.query(tm.ChannelImageFile.id).\
                            join(tm.Site).\
                            join(tm.Cycle).\
                            join(tm.Channel).\
                            filter(tm.Site.id == s).\
                            filter(tm.Cycle.id == cycle.id).\
                            filter(tm.Channel.wavelength == args.ref_wavelength).\
                            filter(~tm.Site.omitted).\
                            all()

                        if not files:
                            # We don't raise an Execption here, because
                            # there may be situations were an aquisition
                            # failed at a given site in one cycle, but
                            # is present in the other cycles.
                            logger.warning(
                                'no files for site %d and cycle %d',
                                s, cycle.id
                            )
                            continue

                        ids = [f.id for f in files]
                        if cycle.index == args.ref_cycle:
                            input_ids['reference_file_ids'].extend(ids)
                        input_ids['target_file_ids'][cycle.id].extend(ids)

                job_descriptions['run'].append({
                    'id': job_count,
                    'input_ids': input_ids,
                    'illumcorr': args.illumcorr
                })

        return job_descriptions

    @same_docstring_as(ClusterRoutines.delete_previous_job_output)
    def delete_previous_job_output(self):
        logger.info('delete existing site shifts and intersections')
        with tm.utils.ExperimentSession(self.experiment_id) as session:
            session.drop_and_recreate(tm.SiteShift)
            session.drop_and_recreate(tm.SiteIntersection)

    def run_job(self, batch):
        '''Calculates shift and overhang values for the given sites.

        Parameters
        ----------
        batch: dict
            description of the *run* job

        Note
        ----
        If sites contain multiple z-planes, z-stacks are projected to 2D and
        the resulting projections are registered.
        '''
        with tm.utils.ExperimentSession(self.experiment_id) as session:
            reference_file_ids = batch['input_ids']['reference_file_ids']
            target_file_ids = batch['input_ids']['target_file_ids']
            if batch['illumcorr']:
                logger.info('correct images for illumination artifacts')

                rid = reference_file_ids[0]
                reference_file = session.query(tm.ChannelImageFile).get(rid)
                try:
                    logger.debug(
                        'load illumination statistics for channel %d of '
                        'reference cycle %d', reference_file.channel_id,
                        reference_file.cycle_id
                    )
                    illumstats_file = session.query(tm.IllumstatsFile).\
                        filter_by(
                            channel_id=reference_file.channel_id,
                            cycle_id=reference_file.cycle_id
                        ).\
                        one()
                except NoResultFound:
                    raise WorkflowError(
                        'No illumination statistics file found for channel %d'
                        % reference_file.channel_id
                    )
                reference_stats = illumstats_file.get()

                target_stats = dict()
                for cycle_id, tids in target_file_ids.iteritems():
                    target_file = session.query(tm.ChannelImageFile).get(tids[0])
                    try:
                        logger.debug(
                            'load illumination statistics for channel %d of'
                            'target cycle %d', target_file.channel_id,
                            target_file.cycle_id
                        )
                        illumstats_file = session.query(tm.IllumstatsFile).\
                            filter_by(
                                channel_id=target_file.channel_id,
                                cycle_id=target_file.cycle_id
                            ).\
                            one()
                    except NoResultFound:
                        raise WorkflowError(
                            'No illumination statistics file found for '
                            'channel %d'
                            % target_file.channel_id
                        )
                    target_stats[cycle_id] = illumstats_file.get()

            for i, rid in enumerate(reference_file_ids):
                reference_file = session.query(tm.ChannelImageFile).get(rid)
                logger.info('register images at site %d', reference_file.site_id)
                logger.debug('load reference image %d', rid)
                reference_img = reference_file.get()
                if batch['illumcorr']:
                    logger.debug('correct reference image')
                    reference_img = reference_img.correct(reference_stats)
                y_shifts = list()
                x_shifts = list()
                for cycle_id, tids in target_file_ids.iteritems():
                    logger.info('calculate shifts for cycle %s', cycle_id)
                    logger.debug('load target image %d', tids[i])
                    target_file = session.query(tm.ChannelImageFile).get(tids[i])
                    target_img = target_file.get()
                    if batch['illumcorr']:
                        logger.debug('correct target image')
                        target_img = target_img.correct(target_stats[cycle_id])

                    y, x = reg.calculate_shift(
                        target_img.array, reference_img.array
                    )

                    session.get_or_create(
                        tm.SiteShift,
                        x=x, y=y,
                        site_id=target_file.site_id,
                        cycle_id=target_file.cycle_id
                    )

                    y_shifts.append(y)
                    x_shifts.append(x)

                logger.info('calculate intersection of sites across cycles')
                top, bottom, right, left = reg.calculate_overhang(
                    y_shifts, x_shifts
                )

                session.get_or_create(
                    tm.SiteIntersection,
                    upper_overhang=top, lower_overhang=bottom,
                    right_overhang=right, left_overhang=left,
                    site_id=reference_file.site_id
                )

    @notimplemented
    def collect_job_output(self, batch):
        pass
        # TODO: set shifts to zero for sites that were omitted

<commit_msg>Fix bug related to missing import in align step<commit_after># TmLibrary - TissueMAPS library for distibuted image analysis routines.
# Copyright (C) 2016  Markus D. Herrmann, University of Zurich and Robin Hafen
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
import logging
from collections import defaultdict

import tmlib.models as tm
from tmlib.utils import notimplemented
from tmlib.utils import same_docstring_as
from tmlib.errors import NotSupportedError, NoResultFound
from tmlib.errors import JobDescriptionError
from tmlib.workflow.align import registration as reg
from tmlib.workflow.api import ClusterRoutines
from tmlib.workflow import register_step_api

logger = logging.getLogger(__name__)

@register_step_api('align')
class ImageRegistrator(ClusterRoutines):

    '''Class for registeration and alignment of images between multiplexing
    cycles.
    '''

    def __init__(self, experiment_id, verbosity):
        '''
        Parameters
        ----------
        experiment_id: int
            ID of the processed experiment
        verbosity: int
            logging level
        '''
        super(ImageRegistrator, self).__init__(experiment_id, verbosity)

    def create_batches(self, args):
        '''Creates job descriptions for parallel computing.

        Parameters
        ----------
        args: tmlib.align.args.AlignInitArgs
            step-specific arguments

        Returns
        -------
        Dict[str, List[dict] or dict]
            job descriptions

        Raises
        ------
        tmlib.errors.NotSupportedError
            when a plate contains only one cycle
        ValueError
            when `args.ref_wavelength` does not exist across all cycles
        '''
        job_count = 0
        job_descriptions = dict()
        job_descriptions['run'] = list()

        with tm.utils.ExperimentSession(self.experiment_id) as session:

            cycles = session.query(tm.Cycle).all()

            if not(len(cycles) > 1):
                raise NotSupportedError(
                    'Alignment requires more than one cycle.'
                )

            if args.ref_cycle >= len(cycles):
                raise JobDescriptionError(
                    'Cycle index must not exceed total number of cycles.'
                )

            site_ids = session.query(tm.Site.id).\
                order_by(tm.Site.id).\
                all()

            batches = self._create_batches(site_ids, args.batch_size)
            for batch in batches:

                job_count += 1
                input_ids = {
                    'reference_file_ids': list(),
                    'target_file_ids': defaultdict(list)
                }

                for cycle in cycles:

                    n = session.query(tm.ChannelImageFile.id).\
                        join(tm.Cycle).\
                        join(tm.Channel).\
                        filter(tm.Cycle.id == cycle.id).\
                        filter(tm.Channel.wavelength == args.ref_wavelength).\
                        filter(~tm.Site.omitted).\
                        count()

                    if n == 0:
                        raise ValueError(
                            'No image files found for cycle %d and '
                            'wavelength "%s"'
                            % (cycle.id, args.ref_wavelength)
                        )

                    for s in batch:

                        files = session.query(tm.ChannelImageFile.id).\
                            join(tm.Site).\
                            join(tm.Cycle).\
                            join(tm.Channel).\
                            filter(tm.Site.id == s).\
                            filter(tm.Cycle.id == cycle.id).\
                            filter(tm.Channel.wavelength == args.ref_wavelength).\
                            filter(~tm.Site.omitted).\
                            all()

                        if not files:
                            # We don't raise an Execption here, because
                            # there may be situations were an aquisition
                            # failed at a given site in one cycle, but
                            # is present in the other cycles.
                            logger.warning(
                                'no files for site %d and cycle %d',
                                s, cycle.id
                            )
                            continue

                        ids = [f.id for f in files]
                        if cycle.index == args.ref_cycle:
                            input_ids['reference_file_ids'].extend(ids)
                        input_ids['target_file_ids'][cycle.id].extend(ids)

                job_descriptions['run'].append({
                    'id': job_count,
                    'input_ids': input_ids,
                    'illumcorr': args.illumcorr
                })

        return job_descriptions

    @same_docstring_as(ClusterRoutines.delete_previous_job_output)
    def delete_previous_job_output(self):
        logger.info('delete existing site shifts and intersections')
        with tm.utils.ExperimentSession(self.experiment_id) as session:
            session.drop_and_recreate(tm.SiteShift)
            session.drop_and_recreate(tm.SiteIntersection)

    def run_job(self, batch):
        '''Calculates shift and overhang values for the given sites.

        Parameters
        ----------
        batch: dict
            description of the *run* job

        Note
        ----
        If sites contain multiple z-planes, z-stacks are projected to 2D and
        the resulting projections are registered.
        '''
        with tm.utils.ExperimentSession(self.experiment_id) as session:
            reference_file_ids = batch['input_ids']['reference_file_ids']
            target_file_ids = batch['input_ids']['target_file_ids']
            if batch['illumcorr']:
                logger.info('correct images for illumination artifacts')

                rid = reference_file_ids[0]
                reference_file = session.query(tm.ChannelImageFile).get(rid)
                try:
                    logger.debug(
                        'load illumination statistics for channel %d of '
                        'reference cycle %d', reference_file.channel_id,
                        reference_file.cycle_id
                    )
                    illumstats_file = session.query(tm.IllumstatsFile).\
                        filter_by(
                            channel_id=reference_file.channel_id,
                            cycle_id=reference_file.cycle_id
                        ).\
                        one()
                except NoResultFound:
                    raise WorkflowError(
                        'No illumination statistics file found for channel %d'
                        % reference_file.channel_id
                    )
                reference_stats = illumstats_file.get()

                target_stats = dict()
                for cycle_id, tids in target_file_ids.iteritems():
                    target_file = session.query(tm.ChannelImageFile).get(tids[0])
                    try:
                        logger.debug(
                            'load illumination statistics for channel %d of'
                            'target cycle %d', target_file.channel_id,
                            target_file.cycle_id
                        )
                        illumstats_file = session.query(tm.IllumstatsFile).\
                            filter_by(
                                channel_id=target_file.channel_id,
                                cycle_id=target_file.cycle_id
                            ).\
                            one()
                    except NoResultFound:
                        raise WorkflowError(
                            'No illumination statistics file found for '
                            'channel %d'
                            % target_file.channel_id
                        )
                    target_stats[cycle_id] = illumstats_file.get()

            for i, rid in enumerate(reference_file_ids):
                reference_file = session.query(tm.ChannelImageFile).get(rid)
                logger.info('register images at site %d', reference_file.site_id)
                logger.debug('load reference image %d', rid)
                reference_img = reference_file.get()
                if batch['illumcorr']:
                    logger.debug('correct reference image')
                    reference_img = reference_img.correct(reference_stats)
                y_shifts = list()
                x_shifts = list()
                for cycle_id, tids in target_file_ids.iteritems():
                    logger.info('calculate shifts for cycle %s', cycle_id)
                    logger.debug('load target image %d', tids[i])
                    target_file = session.query(tm.ChannelImageFile).get(tids[i])
                    target_img = target_file.get()
                    if batch['illumcorr']:
                        logger.debug('correct target image')
                        target_img = target_img.correct(target_stats[cycle_id])

                    y, x = reg.calculate_shift(
                        target_img.array, reference_img.array
                    )

                    session.get_or_create(
                        tm.SiteShift,
                        x=x, y=y,
                        site_id=target_file.site_id,
                        cycle_id=target_file.cycle_id
                    )

                    y_shifts.append(y)
                    x_shifts.append(x)

                logger.info('calculate intersection of sites across cycles')
                top, bottom, right, left = reg.calculate_overhang(
                    y_shifts, x_shifts
                )

                session.get_or_create(
                    tm.SiteIntersection,
                    upper_overhang=top, lower_overhang=bottom,
                    right_overhang=right, left_overhang=left,
                    site_id=reference_file.site_id
                )

    @notimplemented
    def collect_job_output(self, batch):
        pass
        # TODO: set shifts to zero for sites that were omitted

<|endoftext|>


<commit_before>#!/usr/bin/python
# vim: set ts=4 sw=4 et :

from subprocess import Popen
import sys, os, urllib2

try:
    import json
except ImportError:
    import simplejson as json

GMETRIC = "/usr/bin/gmetric --name=\"%s\" --value=\"%s\" --type=\"int32\" --units=\"%s\""

class ServerStatus:
    ops_tmp_file = os.path.join("/", "tmp", "mongo-prevops")

    def __init__(self):
        self.status = self.getServerStatus()
        # call individual metrics
        for f in ["conns", "btree", "mem", "backgroundFlushing", "repl", "ops", "lock"]:
            getattr(self,f)()

    def getServerStatus(self):
        raw = urllib2.urlopen( "http://127.0.0.1:28017/_status" ).read()
        return json.loads( raw )["serverStatus"]

    def callGmetric(self, d):
        for k,v in d.iteritems():
            cmd = GMETRIC % ("mongodb_" + k, v[0], v[1])
            Popen(cmd, shell=True)

    def conns(self):
        ss = self.status
        self.callGmetric({
            "connections" : (ss["connections"]["current"], "connections")
        })

    def btree(self):
        b = self.status["indexCounters"]["btree"]
        self.callGmetric({
            "btree_accesses" : (b["accesses"], "count"),
            "btree_hits" : (b["hits"], "count"),
            "btree_misses" : (b["misses"], "count"),
            "btree_resets" : (b["resets"], "count"),
            "btree_miss_ratio" : (b["missRatio"], "ratio"),
        })

    def mem(self):
        m = self.status["mem"]
        self.callGmetric({
            "mem_resident" : (m["resident"], "MB"),
            "mem_virtual" : (m["virtual"], "MB"),
            "mem_mapped" : (m["mapped"], "MB"),
        })

    def backgroundFlushing(self):
        f = self.status["backgroundFlushing"]
        self.callGmetric({
            "flush_average" : (f["average_ms"], "ms"),
        })

    def ops(self):
        out = {}
        cur_ops = self.status["opcounters"]
        try:
            f = open(self.ops_tmp_file, "r")
            content = f.read()
            prev_ops = json.loads(content)
            f.close()
        except (ValueError, IOError):
            prev_ops = {}

        for k,v in cur_ops.iteritems():
            if k in prev_ops:
                name = k + "s_per_second"
                if k == "query":
                    name = "queries_per_second"
                out[name] = ((float(v) - float(prev_ops[k]) ) / 60, "ops/s")

        f = open(self.ops_tmp_file, 'w')
        try:
            f.write(json.dumps(cur_ops))
        finally:
            f.close()

        self.callGmetric(out)

    def repl(self):
        ismaster = 0;
        if (self.status["repl"]["ismaster"]):
            ismaster = 1

        self.callGmetric({
            "is_master" : (ismaster, "boolean")
        })

    def lock(self):
        self.callGmetric({
            "lock_ratio" : (self.status["globalLock"]["ratio"], "ratio")
        })

if __name__ == "__main__":
    ServerStatus()
<commit_msg>added lock queue metric for readers and writers<commit_after>#!/usr/bin/python
# vim: set ts=4 sw=4 et :

from subprocess import Popen
import sys, os, urllib2

try:
    import json
except ImportError:
    import simplejson as json

GMETRIC = "/usr/bin/gmetric --name=\"%s\" --value=\"%s\" --type=\"int32\" --units=\"%s\""

class ServerStatus:
    ops_tmp_file = os.path.join("/", "tmp", "mongo-prevops")

    def __init__(self):
        self.status = self.getServerStatus()
        # call individual metrics
        for f in ["conns", "btree", "mem", "backgroundFlushing", "repl", "ops", "lock"]:
            getattr(self,f)()

    def getServerStatus(self):
        raw = urllib2.urlopen( "http://127.0.0.1:28017/_status" ).read()
        return json.loads( raw )["serverStatus"]

    def callGmetric(self, d):
        for k,v in d.iteritems():
            cmd = GMETRIC % ("mongodb_" + k, v[0], v[1])
            Popen(cmd, shell=True)

    def conns(self):
        ss = self.status
        self.callGmetric({
            "connections" : (ss["connections"]["current"], "connections")
        })

    def btree(self):
        b = self.status["indexCounters"]["btree"]
        self.callGmetric({
            "btree_accesses" : (b["accesses"], "count"),
            "btree_hits" : (b["hits"], "count"),
            "btree_misses" : (b["misses"], "count"),
            "btree_resets" : (b["resets"], "count"),
            "btree_miss_ratio" : (b["missRatio"], "ratio"),
        })

    def mem(self):
        m = self.status["mem"]
        self.callGmetric({
            "mem_resident" : (m["resident"], "MB"),
            "mem_virtual" : (m["virtual"], "MB"),
            "mem_mapped" : (m["mapped"], "MB"),
        })

    def backgroundFlushing(self):
        f = self.status["backgroundFlushing"]
        self.callGmetric({
            "flush_average" : (f["average_ms"], "ms"),
        })

    def ops(self):
        out = {}
        cur_ops = self.status["opcounters"]
        try:
            f = open(self.ops_tmp_file, "r")
            content = f.read()
            prev_ops = json.loads(content)
            f.close()
        except (ValueError, IOError):
            prev_ops = {}

        for k,v in cur_ops.iteritems():
            if k in prev_ops:
                name = k + "s_per_second"
                if k == "query":
                    name = "queries_per_second"
                out[name] = ((float(v) - float(prev_ops[k]) ) / 60, "ops/s")

        f = open(self.ops_tmp_file, 'w')
        try:
            f.write(json.dumps(cur_ops))
        finally:
            f.close()

        self.callGmetric(out)

    def repl(self):
        ismaster = 0;
        if (self.status["repl"]["ismaster"]):
            ismaster = 1

        self.callGmetric({
            "is_master" : (ismaster, "boolean")
        })

    def lock(self):
        self.callGmetric({
            "lock_ratio" : (self.status["globalLock"]["ratio"], "ratio"),
            "lock_queue_readers" : (self.status["globalLock"]["currentQueue"]["readers"], "queue size"),
            "lock_queue_writers" : (self.status["globalLock"]["currentQueue"]["writers"], "queue size"),
        })

if __name__ == "__main__":
    ServerStatus()
<|endoftext|>


<commit_before>import numpy as np
import fitsio
from astropy.table import Table, join
import os
from glob import glob
from argparse import ArgumentParser

parser = ArgumentParser()

parser.add_argument(
    'det_catalog',
    type=str,
    help='Filename of a Balrog detection catalog (created with a given matching criterion)'
    )
parser.add_argument(
    'gold_catalog',
    type=str,
    help='Filename of a DES gold catalog to match to as a reference catalog.'
    )
parser.add_argument(
    '--det_bright_col',
    default='true_gap_riz_flux_deredden',
    type=str,
    help='Name of brightness col to compare by in the detection catalog'
    )
parser.add_argument(
    '--gold_bright_col',
    default='AVG_RIZ_GAP_FLUX_DEREDDEN'
    type=str,
    help='Name of brightness col to compare by in the gold catalog'
    )
parser.add_argument(
    '--depth',
    default=14,
    type=int,
    help='depth of HTM matcher'
    )
parser.add_argument(
    '--min_radius',
    default=0.5,
    type=float,
    help='Minimum match radius in arcsec'
    )
parser.add_argument(
    '--max_radius',
    default=2.0,
    type=float,
    help='Maximum match radius in arcsec'
    )
parser.add_argument(
    '--radius_step',
    default=0.25,
    type=float,
    help='Radius step in arcsec'
    )
parser.add_argument(
    '--det_ratag',
    default='true_ra',
    type=str,
    help='Name of detection ra column'
    )
parser.add_argument(
    '--det_dectag',
    default='true_dec',
    type=str,
    help='Name of detection dec column'
    )
parser.add_argument(
    '--gold_ratag',
    default='RA',
    type=str,
    help='Name of gold ra column'
    )
parser.add_argument(
    '--gold_dectag',
    default='DEC',
    type=str,
    help='Name of gold dec column'
    )
# parser.add_argument(
#     '--clobber',
#     default=False,
#     type=bool,
#     help='Set to True if you want to write to same file'
#     )
parser.add_argument(
    '--vb',
    action='store_true',
    default=False,
    help='Set to print out more information'
)

# def compute_flux_factors(fluxes, dereddened_fluxes):
#     '''expects np arrays or astropy tables'''
#     return dereddened_fluxes / flux_factors

def main():
    args = parser.parse_args()
    det_file = args.det_catalog
    gold_file = args.gold_catalog
    min_radius = args.min_radius / 3600. # arcsec2deg
    max_radius = args.max_radius / 3600. # arcsec2deg
    drad = args.radius_step / 3600. # arcsec2deg
    det_ratag = args.det_ratag
    det_dectag = args.det_dectag
    gold_ratag = args.gold_ratag
    gold_dettag = args.gold_dettag
    depth = args.depth
    # clobber = args.clobber
    vb = args.vb

    det_bright_col  = args.det_bright_col
    gold_bright_col = args.gold_bright_col

    outfile = det_file.replace('.fits', '_match_flags.fits')

    gold_cols = [gold_ratag, gold_dectag, gold_bright_col]

    if vb is True:
        print('Loading det...')
    det  = Table(fitsio.read(det_file))
    if vb is True:
        print('Loading gold...')
    gold = Table(fitsio.read(gold_file, columns=gold_cols))

    radii = np.arange(min_radius, max_radius+drad, drad)
    if vb is True:
        print('Matching the following radii: {}'.format(radii))

    for match_radius in radii:
        if vb is True:
            print('Initializing match_flag for radius {}...'.format(match_radius))
        match_flag = np.zeros(len(det), dtype='i4')

        if vb is True:
            print('Matching...')
        h = htm.HTM(depth)
        matcher = htm.Matcher(depth=depth, ra=det[det_ratag], dec=det[det_dectag])
        id_gold, id_det, dist = matcher.match(ra=gold[gold_ratag], dec=gold[gold_dectag],
                                            radius=match_radius)
        assert len(id_det) == len(id_gold)

        if vb is True:
            print('Done Matching')
            print('len(det) = {}'.format(len(det)))
            print('len(id_det) = {}'.format(len(id_det)))
            print('len(match_flag) = {}'.format(len(match_flag)))
            print('len(id_gold) = {}'.format(len(id_gold)))
            print('max(id_det) = {}'.format(np.max(id_det)))
            print('max(id_gold) = {}'.format(np.max(id_gold)))

        match_flag[id_det] = 1
        match_indices = zip(id_det, id_gold)

        Nmatches = len(id_det)

        if vb is True:
            print('There are {} injections near a gold object'.format(Nmatches))
            print('Checking brightness ratios...')
        k = 0
        for idet, igold in match_indices:
            k += 1
            if k%100 == 0:
                if vb is True: print('{} / {}'.format(k, Nmatches))
            if det[det_bright_col][idet] < gold[gold_bright_col][igold]:
               match_flag[idet] = 2

        det['match_flag_{}_asec'] = match_flag

    if vb is True:
        print('Writing to {}...'.format(outfile))
    det.write(outfile)

    return

if __name__ == '__main__':
    main()
<commit_msg>Fixed a few minor bugs, radii output formatting, and imports.<commit_after>import numpy as np
import fitsio
from astropy.table import Table, join
import esutil.htm as htm
import os
from glob import glob
from argparse import ArgumentParser

parser = ArgumentParser()

parser.add_argument(
    'det_catalog',
    type=str,
    help='Filename of a Balrog detection catalog (created with a given matching criterion)'
    )
parser.add_argument(
    '--gold_catalog',
    type=str,
    help='Filename of a DES gold catalog to match to as a reference catalog.'
    )
parser.add_argument(
    '--det_bright_col',
    default='true_gap_riz_flux_deredden',
    type=str,
    help='Name of brightness col to compare by in the detection catalog'
    )
parser.add_argument(
    '--gold_bright_col',
    default='AVG_RIZ_GAP_FLUX_DEREDDEN',
    type=str,
    help='Name of brightness col to compare by in the gold catalog'
    )
parser.add_argument(
    '--depth',
    default=14,
    type=int,
    help='depth of HTM matcher'
    )
parser.add_argument(
    '--min_radius',
    default=0.5,
    type=float,
    help='Minimum match radius in arcsec'
    )
parser.add_argument(
    '--max_radius',
    default=2.0,
    type=float,
    help='Maximum match radius in arcsec'
    )
parser.add_argument(
    '--radius_step',
    default=0.25,
    type=float,
    help='Radius step in arcsec'
    )
parser.add_argument(
    '--det_ratag',
    default='true_ra',
    type=str,
    help='Name of detection ra column'
    )
parser.add_argument(
    '--det_dectag',
    default='true_dec',
    type=str,
    help='Name of detection dec column'
    )
parser.add_argument(
    '--gold_ratag',
    default='RA',
    type=str,
    help='Name of gold ra column'
    )
parser.add_argument(
    '--gold_dectag',
    default='DEC',
    type=str,
    help='Name of gold dec column'
    )
# parser.add_argument(
#     '--clobber',
#     default=False,
#     type=bool,
#     help='Set to True if you want to write to same file'
#     )
parser.add_argument(
    '--vb',
    action='store_true',
    default=False,
    help='Set to print out more information'
)

# def compute_flux_factors(fluxes, dereddened_fluxes):
#     '''expects np arrays or astropy tables'''
#     return dereddened_fluxes / flux_factors

def main():
    args = parser.parse_args()
    det_file = args.det_catalog
    gold_file = args.gold_catalog
    min_radius = args.min_radius / 3600. # arcsec2deg
    max_radius = args.max_radius / 3600. # arcsec2deg
    drad = args.radius_step / 3600. # arcsec2deg
    det_ratag = args.det_ratag
    det_dectag = args.det_dectag
    gold_ratag = args.gold_ratag
    gold_dectag = args.gold_dectag
    depth = args.depth
    # clobber = args.clobber
    vb = args.vb

    det_bright_col  = args.det_bright_col
    gold_bright_col = args.gold_bright_col

    outfile = det_file.replace('.fits', '_match_flags.fits')

    gold_cols = [gold_ratag, gold_dectag, gold_bright_col]

    if vb is True:
        print('Loading det...')
    det  = Table(fitsio.read(det_file))
    if vb is True:
        print('Loading gold...')
    gold = Table(fitsio.read(gold_file, columns=gold_cols))

    radii = np.arange(min_radius, max_radius+drad, drad)
    if vb is True:
        print('Matching the following radii: {:.2f}'.format(3600.*radii))

    for match_radius in radii:
        if vb is True:
            print('Initializing match_flag for radius {:.2f}...'.format(3600.*match_radius))
        match_flag = np.zeros(len(det), dtype='i4')

        if vb is True:
            print('Matching...')
        h = htm.HTM(depth)
        matcher = htm.Matcher(depth=depth, ra=det[det_ratag], dec=det[det_dectag])
        id_gold, id_det, dist = matcher.match(ra=gold[gold_ratag], dec=gold[gold_dectag],
                                            radius=match_radius)
        assert len(id_det) == len(id_gold)

        if vb is True:
            print('Done Matching')
            print('len(det) = {}'.format(len(det)))
            print('len(id_det) = {}'.format(len(id_det)))
            print('len(match_flag) = {}'.format(len(match_flag)))
            print('len(id_gold) = {}'.format(len(id_gold)))
            print('max(id_det) = {}'.format(np.max(id_det)))
            print('max(id_gold) = {}'.format(np.max(id_gold)))

        match_flag[id_det] = 1
        match_indices = zip(id_det, id_gold)

        Nmatches = len(id_det)

        if vb is True:
            print('There are {} injections near a gold object'.format(Nmatches))
            print('Checking brightness ratios...')
        k = 0
        for idet, igold in match_indices:
            k += 1
            if k%100 == 0:
                if vb is True: print('{} / {}'.format(k, Nmatches))
            if det[det_bright_col][idet] < gold[gold_bright_col][igold]:
               match_flag[idet] = 2

        det['match_flag_{}_asec'] = match_flag

    if vb is True:
        print('Writing to {}...'.format(outfile))
    det.write(outfile)

    return

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
class Manager(object):

    # should be re-defined in a subclass
    state_name = None
    object_type = None

    def __init__(self, api):
        self.api = api

    # shortcuts

    @property
    def state(self):
        return self.api.state

    @property
    def queue(self):
        return self.api.queue

class AllMixin(object):
    def all(self, filt=None):
        return list(filter(filt, self.state[self.state_name]))

class GetByIdMixin(object):

    def get_by_id(self, obj_id, only_local=False):
        """
        Finds and returns the object based on its id.
        """
        for obj in self.state[self.state_name]:
            if obj['id'] == obj_id or obj.temp_id == str(obj_id):
                return obj

        if not only_local:
            getter = getattr(self.api, 'get_%s' % self.object_type)
            return getter(obj_id)

        return None

class SyncMixin(object):
    """
    Syncs this specific type of objects.
    """
    def sync(self):
        return self.api.sync()
<commit_msg>Use the remote getter call only on objects with an object_type.<commit_after># -*- coding: utf-8 -*-
class Manager(object):

    # should be re-defined in a subclass
    state_name = None
    object_type = None

    def __init__(self, api):
        self.api = api

    # shortcuts

    @property
    def state(self):
        return self.api.state

    @property
    def queue(self):
        return self.api.queue

class AllMixin(object):
    def all(self, filt=None):
        return list(filter(filt, self.state[self.state_name]))

class GetByIdMixin(object):

    def get_by_id(self, obj_id, only_local=False):
        """
        Finds and returns the object based on its id.
        """
        for obj in self.state[self.state_name]:
            if obj['id'] == obj_id or obj.temp_id == str(obj_id):
                return obj

        if not only_local and self.object_type is not None:
            getter = getattr(self.api, 'get_%s' % self.object_type)
            return getter(obj_id)

        return None

class SyncMixin(object):
    """
    Syncs this specific type of objects.
    """
    def sync(self):
        return self.api.sync()
<|endoftext|>