<commit_before>import mimetypes
import multiprocessing
import os

import flask
import requests

from smqtk.algorithms.descriptor_generator import get_descriptor_generator_impls
from smqtk.algorithms.nn_index import NearestNeighborsIndex, get_nn_index_impls
from smqtk.representation import DescriptorElementFactory
from smqtk.representation.data_element.file_element import DataFileElement
from smqtk.representation.data_element.memory_element import DataMemoryElement
from smqtk.representation.data_element.url_element import DataUrlElement
from smqtk.utils import SimpleTimer
from smqtk.utils import plugin
from smqtk.utils.configuration import merge_configs
from smqtk.web import SmqtkWebApp

MIMETYPES = mimetypes.MimeTypes()

SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))

class NearestNeighborServiceServer (SmqtkWebApp):
    """
    Simple server that takes in a specification of the following form:

        /nn/<path:uri>[?...]

    Computes the nearest neighbor index for the given data and returns a list
    of nearest neighbors in the following format

    Standard return JSON::
    {
        "success": <bool>,
        "neighbors": [ <float>, ... ]
        "message": <string>,
        "reference_uri": <uri>
    }
    """

    @classmethod
    def is_usable(cls):
        return True

    @classmethod
    def get_default_config(cls):
        """
        Generate and return a default configuration dictionary for this class.
        This will be primarily used for generating what the configuration
        dictionary would look like for this class without instantiating it.

        :return: Default configuration dictionary for the class.
        :rtype: dict

        """
        c = super(NearestNeighborServiceServer, cls).get_default_config()
        merge_configs(c, {
            "descriptor_factory": DescriptorElementFactory.get_default_config(),
            "descriptor_generator":
                plugin.make_config(get_descriptor_generator_impls),
            "nn_index": plugin.make_config(get_nn_index_impls),
        })
        return c

    def __init__(self, json_config):
        """
        Initialize application based of supplied JSON configuration

        :param json_config: JSON configuration dictionary
        :type json_config: dict

        """
        super(NearestNeighborServiceServer, self).__init__(json_config)

        # Descriptor factory setup
        self.log.info("Initializing DescriptorElementFactory")
        self.descr_elem_factory = DescriptorElementFactory.from_config(
            self.json_config['descriptor_factory']
        )

        # Descriptor generator configuration labels
        #: :type: dict[str, dict]
        self.generator_config = self.json_config['descriptor_generator']

        #: :type: smqtk.algorithms.NearestNeighborsIndex
        self.nn_index = plugin.from_plugin_config(
            json_config['nn_index'],
            get_nn_index_impls
        )

        #: :type: smqtk.algorithms.DescriptorGenerator
        self.descriptor_generator_inst = plugin.from_plugin_config(
                                            self.generator_config,
                                            get_descriptor_generator_impls)

        @self.route("/nn/<path:uri>")
        @self.route("/nn/n=<int:n>/<path:uri>")
        def compute_nearest_neighbors(uri, n=10):
            """
            Data modes for upload/use::

                - local filepath
                - base64
                - http/s URL

            The following sub-sections detail how different URI's can be used.

            Local Filepath
            --------------

            The URI string must be prefixed with ``file://``, followed by the
            full path to the data file to describe.

            Base 64 data
            ------------

            The URI string must be prefixed with "base64://", followed by the
            base64 encoded string. This mode also requires an additional
            ``?content_type=`` to provide data content type information. This
            mode saves the encoded data to temporary file for processing.

            HTTP/S address
            --------------

            This is the default mode when the URI prefix is none of the above.
            This uses the requests module to locally download a data file
            for processing.

            JSON Return format::
                {
                    "success": <bool>

                    "message": <str>

                    "neighbors": <None|list[float]>

                    "reference_uri": <str>
                }

            :type uri: str

            """
            message = "execution nominal"
            descriptor = None

            de = None
            try:
                self.log.debug("Received URI: %s", uri)
                de = self.resolve_data_element(uri)
            except ValueError, ex:
                message = "URI resolution issue: %s" % str(ex)

            if de:
                try:
                    descriptor = self.descriptor_generator_inst.\
                        compute_descriptor(de, self.descr_elem_factory)
                except RuntimeError, ex:
                    message = "Descriptor extraction failure: %s" % str(ex)
                except ValueError, ex:
                    message = "Data content type issue: %s" % str(ex)

            # fail here if de is None
            # Default is 8
            num_neighbors = flask.request.args.get("num_neighbors", n)

            neighbors = []
            dists = []
            if descriptor is not None:
                try:
                    neighbors, dists = \
                        self.nn_index.nn(descriptor, n=num_neighbors)
                except ValueError, ex:
                    message = "Descriptor or index related issue: %s" % str(ex)

            # TODO: Return the optional descriptor vectors for the neighbors
            return flask.jsonify({
                "success": descriptor is not None,
                "message": message,
                "neighbors": [n.uuid() for n in neighbors],
                "distances": dists,
                "reference_uri": uri
            })

    def get_config(self):
        return self.json_config

    def resolve_data_element(self, uri):
        """
        Given the URI to some data, resolve it down to a DataElement instance.

        :raises ValueError: Issue with the given URI regarding either URI source
            resolution or data resolution.

        :param uri: URI to data
        :type uri: str
        :return: DataElement instance wrapping given URI to data.
        :rtype: smqtk.representation.DataElement

        """
        # Resolve URI into appropriate DataElement instance
        if uri[:7] == "file://":
            self.log.debug("Given local disk filepath")
            filepath = uri[7:]
            if not os.path.isfile(filepath):
                raise ValueError("File URI did not point to an existing file "
                                 "on disk.")
            else:
                de = DataFileElement(filepath)

        elif uri[:9] == "base64://":
            self.log.debug("Given base64 string")
            content_type = flask.request.args.get('content_type', None)
            self.log.debug("Content type: %s", content_type)
            if not content_type:
                raise ValueError("No content-type with given base64 data")
            else:
                b64str = uri[9:]
                de = DataMemoryElement.from_base64(b64str, content_type)

        else:
            self.log.debug("Given URL")
            try:
                de = DataUrlElement(uri)
            except requests.HTTPError, ex:
                raise ValueError("Failed to initialize URL element due to "
                                 "HTTPError: %s" % str(ex))

        return de

APPLICATION_CLASS = NearestNeighborServiceServer
<commit_msg>Adds a fake implementation of pagination to allow other applications to specify limit/offset<commit_after>import mimetypes
import multiprocessing
import os

import flask
import requests

from smqtk.algorithms.descriptor_generator import get_descriptor_generator_impls
from smqtk.algorithms.nn_index import NearestNeighborsIndex, get_nn_index_impls
from smqtk.representation import DescriptorElementFactory
from smqtk.representation.data_element.file_element import DataFileElement
from smqtk.representation.data_element.memory_element import DataMemoryElement
from smqtk.representation.data_element.url_element import DataUrlElement
from smqtk.utils import SimpleTimer
from smqtk.utils import plugin
from smqtk.utils.configuration import merge_configs
from smqtk.web import SmqtkWebApp

MIMETYPES = mimetypes.MimeTypes()

SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))

class NearestNeighborServiceServer (SmqtkWebApp):
    """
    Simple server that takes in a specification of the following form:

        /nn/<path:uri>[?...]

    Computes the nearest neighbor index for the given data and returns a list
    of nearest neighbors in the following format

    Standard return JSON::
    {
        "success": <bool>,
        "neighbors": [ <float>, ... ]
        "message": <string>,
        "reference_uri": <uri>
    }
    """

    @classmethod
    def is_usable(cls):
        return True

    @classmethod
    def get_default_config(cls):
        """
        Generate and return a default configuration dictionary for this class.
        This will be primarily used for generating what the configuration
        dictionary would look like for this class without instantiating it.

        :return: Default configuration dictionary for the class.
        :rtype: dict

        """
        c = super(NearestNeighborServiceServer, cls).get_default_config()
        merge_configs(c, {
            "descriptor_factory": DescriptorElementFactory.get_default_config(),
            "descriptor_generator":
                plugin.make_config(get_descriptor_generator_impls),
            "nn_index": plugin.make_config(get_nn_index_impls),
        })
        return c

    def __init__(self, json_config):
        """
        Initialize application based of supplied JSON configuration

        :param json_config: JSON configuration dictionary
        :type json_config: dict

        """
        super(NearestNeighborServiceServer, self).__init__(json_config)

        # Descriptor factory setup
        self.log.info("Initializing DescriptorElementFactory")
        self.descr_elem_factory = DescriptorElementFactory.from_config(
            self.json_config['descriptor_factory']
        )

        # Descriptor generator configuration labels
        #: :type: dict[str, dict]
        self.generator_config = self.json_config['descriptor_generator']

        #: :type: smqtk.algorithms.NearestNeighborsIndex
        self.nn_index = plugin.from_plugin_config(
            json_config['nn_index'],
            get_nn_index_impls
        )

        #: :type: smqtk.algorithms.DescriptorGenerator
        self.descriptor_generator_inst = plugin.from_plugin_config(
                                            self.generator_config,
                                            get_descriptor_generator_impls)

        @self.route("/nn/<path:uri>")
        @self.route("/nn/n=<int:n>/<path:uri>")
        @self.route("/nn/offset=<int:offset>/n=<int:n>/<path:uri>")
        def compute_nearest_neighbors(uri, offset=0, n=10):
            """
            Data modes for upload/use::

                - local filepath
                - base64
                - http/s URL

            The following sub-sections detail how different URI's can be used.

            Local Filepath
            --------------

            The URI string must be prefixed with ``file://``, followed by the
            full path to the data file to describe.

            Base 64 data
            ------------

            The URI string must be prefixed with "base64://", followed by the
            base64 encoded string. This mode also requires an additional
            ``?content_type=`` to provide data content type information. This
            mode saves the encoded data to temporary file for processing.

            HTTP/S address
            --------------

            This is the default mode when the URI prefix is none of the above.
            This uses the requests module to locally download a data file
            for processing.

            JSON Return format::
                {
                    "success": <bool>

                    "message": <str>

                    "neighbors": <None|list[float]>

                    "reference_uri": <str>
                }

            :type uri: str

            """
            message = "execution nominal"
            descriptor = None

            de = None
            try:
                self.log.debug("Received URI: %s", uri)
                de = self.resolve_data_element(uri)
            except ValueError, ex:
                message = "URI resolution issue: %s" % str(ex)

            if de:
                try:
                    descriptor = self.descriptor_generator_inst.\
                        compute_descriptor(de, self.descr_elem_factory)
                except RuntimeError, ex:
                    message = "Descriptor extraction failure: %s" % str(ex)
                except ValueError, ex:
                    message = "Data content type issue: %s" % str(ex)

            # fail here if de is None
            # Default is 10
            num_neighbors = flask.request.args.get("num_neighbors", n)
            page_slice = slice(offset, num_neighbors + offset)

            neighbors = []
            dists = []
            if descriptor is not None:
                try:
                    # We ask for 1000 as an arbitrarily large number that we can
                    # pretend to paginate through
                    neighbors, dists = \
                        self.nn_index.nn(descriptor, n=1000)
                except ValueError, ex:
                    message = "Descriptor or index related issue: %s" % str(ex)

            # TODO: Return the optional descriptor vectors for the neighbors
            return flask.jsonify({
                "success": descriptor is not None,
                "message": message,
                "neighbors": [n.uuid() for n in neighbors[page_slice]],
                "distances": dists[page_slice],
                "reference_uri": uri
            })

    def get_config(self):
        return self.json_config

    def resolve_data_element(self, uri):
        """
        Given the URI to some data, resolve it down to a DataElement instance.

        :raises ValueError: Issue with the given URI regarding either URI source
            resolution or data resolution.

        :param uri: URI to data
        :type uri: str
        :return: DataElement instance wrapping given URI to data.
        :rtype: smqtk.representation.DataElement

        """
        # Resolve URI into appropriate DataElement instance
        if uri[:7] == "file://":
            self.log.debug("Given local disk filepath")
            filepath = uri[7:]
            if not os.path.isfile(filepath):
                raise ValueError("File URI did not point to an existing file "
                                 "on disk.")
            else:
                de = DataFileElement(filepath)

        elif uri[:9] == "base64://":
            self.log.debug("Given base64 string")
            content_type = flask.request.args.get('content_type', None)
            self.log.debug("Content type: %s", content_type)
            if not content_type:
                raise ValueError("No content-type with given base64 data")
            else:
                b64str = uri[9:]
                de = DataMemoryElement.from_base64(b64str, content_type)

        else:
            self.log.debug("Given URL")
            try:
                de = DataUrlElement(uri)
            except requests.HTTPError, ex:
                raise ValueError("Failed to initialize URL element due to "
                                 "HTTPError: %s" % str(ex))

        return de

APPLICATION_CLASS = NearestNeighborServiceServer
<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -*-

import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt
from scipy.optimize import leastsq, fmin
from rpy import r

def platt(light,etr,ini=None):
    '''
    Adjust a curve of best fit, following the Platt model.

    Parameters
    ----------
    light : arr
        Generally PAR values. Where Photosynthetic Active Radiance
        interfer on Primary Production. 'light' is this parameter.

    etr : arr
        Eletron Transference Rate, by means relative ETR, obtained from
        Rapid Light Curves.

    ini : List
        optional intial values for optimization proccess.
    Returns
    -------
    opts : arr
        Values optimized

    pars : arr
        Curve parameters (alpha, Ek, ETRmax)

    See Also
    --------
    T. Platt, C.L. Gallegos and W.G. Harrison, 1980. Photoinibition of photosynthesis in natural
        assemblages of marine phytoplankton

    '''
    opts = []
    pars = []

    r.assign("x", light[~np.isnan(light)])
    r.assign("y", etr[~np.isnan(etr)])

    if ini == None:
        r.assign('ini', [0.4,1.5,1500])

    else:
        r.assign('ini', np.array(ini))

    min_platt = r("""
    platt<- function(params){
        alpha<-params[1]
        Beta<- params[2]
        Ps<- params[3]
        return( sum( (y-Ps*(1-exp(-alpha*x/Ps))*exp(-Beta*x/Ps))^2))
    } """)
    min_adp = r("""
    min_ad<-function(params){
        alpha<-params[1]
        Beta<-params[2]
        Ps<-params[3]
        return( ( (Ps*(1-exp(-alpha*x/Ps)) *exp(-Beta*x/Ps)) ) )
    }""")
    r('etr_sim<-optim(par=ini, fn=platt)')
    r('p_alpha<-etr_sim$par[1]')
    r('p_Beta<-etr_sim$par[2]')
    r('p_Ps2<-etr_sim$par[3]')
    r('''
        if (p_Beta==0 | p_Beta<0){
            p_etrmax<-p_Ps2
        }else {
            p_etrmax<-p_Ps2*(p_alpha/(p_alpha+p_Beta))*
            (p_Beta/(p_alpha+p_Beta))^(p_Beta/p_alpha)
        }

        p_Ek<-p_etrmax/p_alpha
    ''')

    opts = np.append(opts, r('min_ad(par = etr_sim$par)'))
    cpars = r('as.data.frame(cbind(p_alpha, p_Ek, p_etrmax))')
    pars = [cpars['p_alpha'], cpars['p_Ek'], cpars['p_etrmax']]

    return opts, pars

def platt_opts(light, etr, ini=None, opt=None):
    '''
    Adjust `opt` values of PAR levels following the Platt model.

    Parameters
    ----------
    light : arr
        Generally PAR values. Where Photosynthetic Active Radiance
        interfer on Primary Production. 'light' is this parameter.
    etr : arr
        Eletron Transference Rate, by means relative ETR, obtained from
        Rapid Light Curves.
    ini : List
        optional intial values for optimization proccess.
    opt : arr
        light range to be modeled.

    Returns
    -------
    opts : arr
        Values optimized according to opt list of PAR levels.
    pars : arr
        Curve parameters (alpha, Ek, ETRmax), acording to `platt`
    '''
    opts = []
    pars = []

    r.assign("y", etr[~np.isnan(etr)])
    if opt == None:
        r.assign("opt", light[~np.isnan(light)])
    else:
        r.assign("opt", opt[~np.isnan(opt)])

    if ini == None:
        r.assign('ini', [0.4,1.5,1500])

    else:
        r.assign('ini', np.array(ini))

    op, platt_param = platt(light,etr, ini=ini)
    r.assign('platt_param', platt_param)

    min_opt = r("""
    min_opt<-function(params){
        alpha<-params[1]
        Beta<-params[2]
        Ps<-params[3]
        return( ( (Ps*(1-exp(-alpha*opt/Ps)) *exp(-Beta*opt/Ps)) ) )
    }""")

    opts = np.append(opts, r('min_opt(par = platt_param)'))
    pars = platt_param

    return opts, pars

def eilers_peeters(light,etr,ini=None):
    '''
    Adjust a best fit curve to ExP curves, according to Eilers  & Peters
    Model.

    Parameters
    ----------
    light : arr
        Generally PAR values. Where Photosynthetic Active Radiance
        interfer on Primary Production. 'light' is this parameter.

    etr : arr
        Eletron Transference Rate, by means relative ETR, obtained from
        Rapid Light Curves.

    ini : None
        Initial values values to set the curve.
        To insert initial values, they must be a list
        of values of initial parameters (a,b,c) of Eilers-Peeters models

    Return
    ------
    opts : arr
        Values optimized

    params : arr
        Curve Parameters (alpha, Ek, ETR_max)

    See Also
    --------
    P.H.C. Eilers and J.C.H Peeters. 1988. A model for the relationship
    between the light intensity and the rate of photosynthesis in
    phytoplankton. Ecol. Model. 42:199-215.

    #TODO
    ## Implement minimisation in Python.
    ## It's not very clear how to apply `nls2` in Python.
    ## minimize from a list of initial values.

    ##a = varis[0]
    ##b = varis[1]
    ##c = varis[2]
    #a = mini['a']
    #b = mini['b']
    #c = mini['c']

    #opts = (light/(a*(light**2)+(b*light)+c))
    #ad = fmin(ep_minimize,varis,args=(light,etr))

    #alpha = (1./ad[2])
    #etrmax = 1./(ad[1]+2*(ad[0]*ad[2])**0.5)
    #Eopt = (ad[2]/ad[0])**0.5
    #Ek = etrmax/alpha

    #params = [alpha, Ek, etrmax, Eopt]
    '''
    r('library(nls2)')
    r.assign("x", light[~np.isnan(light)])
    r.assign("y", etr[~np.isnan(etr)])
    r('dat<-as.data.frame(cbind(x,y))')
    r('names(dat)<-c("light","etr")')

    if ini == None:
        r('''grid<-expand.grid(list(a=seq(1e-07,9e-06,by=2e-07),
        b=seq(-0.002,0.006,by=0.002),c=seq(-6,6,by=2)))''')
        mini = r('''
            mini<-coefficients(nls2(etr~light/(a*light^2+b*light+c),
            data=dat, start=grid, algorithm="brute-force"))
            ''')
    else:
        mini = ini
        r.assign("mini", mini)

    r('''ep<-nls(etr~light/(a*light^2+b*light+c),data=dat,
    start=list(a=mini[1],b=mini[2],c=mini[3]),
    lower = list(0,-Inf,-Inf), trace=FALSE,
    algorithm = "port", nls.control("maxiter"=100000, tol=0.15))

    a2<-summary(ep)$coefficients[1]
    b2<-summary(ep)$coefficients[2]
    c2<-summary(ep)$coefficients[3]

    alpha<-(1/c2)
    etrmax<-1/(b2+2*(a2*c2)^0.5)
    Eopt<-(c2/a2)^0.5
    Ek<-etrmax/alpha''')

    alpha = r('alpha')
    Ek = r('Ek')
    etr_max = r('etrmax')
    params = [alpha, Ek, etr_max]
    opts = r('opts<-fitted(ep)')

    return opts, params
<commit_msg>platt_opts modified, now outputs standard algorithm values based on light entry and parameters as alpha, Beta, Pmax<commit_after>#! /usr/bin/env python
# -*- coding: utf-8 -*-

import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt
from scipy.optimize import leastsq, fmin
from rpy import r

def platt(light,etr,ini=None):
    '''
    Adjust a curve of best fit, following the Platt model.

    Parameters
    ----------
    light : arr
        Generally PAR values. Where Photosynthetic Active Radiance
        interfer on Primary Production. 'light' is this parameter.

    etr : arr
        Eletron Transference Rate, by means relative ETR, obtained from
        Rapid Light Curves.

    ini : List
        optional intial values for optimization proccess.
    Returns
    -------
    opts : arr
        Values optimized

    pars : arr
        Curve parameters (alpha, Ek, ETRmax)

    See Also
    --------
    T. Platt, C.L. Gallegos and W.G. Harrison, 1980. Photoinibition of photosynthesis in natural
        assemblages of marine phytoplankton

    '''
    opts = []
    pars = []

    r.assign("x", light[~np.isnan(light)])
    r.assign("y", etr[~np.isnan(etr)])

    if ini == None:
        r.assign('ini', [0.4,1.5,1500])

    else:
        r.assign('ini', np.array(ini))

    min_platt = r("""
    platt<- function(params){
        alpha<-params[1]
        Beta<- params[2]
        Ps<- params[3]
        return( sum( (y-Ps*(1-exp(-alpha*x/Ps))*exp(-Beta*x/Ps))^2))
    } """)
    min_adp = r("""
    min_ad<-function(params){
        alpha<-params[1]
        Beta<-params[2]
        Ps<-params[3]
        return( ( (Ps*(1-exp(-alpha*x/Ps)) *exp(-Beta*x/Ps)) ) )
    }""")
    r('etr_sim<-optim(par=ini, fn=platt)')
    r('p_alpha<-etr_sim$par[1]')
    r('p_Beta<-etr_sim$par[2]')
    r('p_Ps2<-etr_sim$par[3]')
    r('''
        if (p_Beta==0 | p_Beta<0){
            p_etrmax<-p_Ps2
        }else {
            p_etrmax<-p_Ps2*(p_alpha/(p_alpha+p_Beta))*
            (p_Beta/(p_alpha+p_Beta))^(p_Beta/p_alpha)
        }

        p_Ek<-p_etrmax/p_alpha
    ''')

    opts = np.append(opts, r('min_ad(par = etr_sim$par)'))
    cpars = r('as.data.frame(cbind(p_alpha, p_Ek, p_etrmax))')
    pars = [cpars['p_alpha'], cpars['p_Ek'], cpars['p_etrmax']]

    return opts, pars

def platt_opts(light, params):
    '''
    Adjust `opt` values of PAR levels following the Platt model.

    Parameters
    ----------
    light : arr
        Generally PAR values. Where Photosynthetic Active Radiance
        interfer on Primary Production. 'light' is this parameter.
    params: arr
        Containing values of (alpha, Beta, etrmax).

    Returns
    -------
    opts : arr
        Values optimized according to `params`and list of PAR levels.
    '''
    opts = []
    pars = []

    r.assign("light", light[~np.isnan(light)])
    r.assign("params", params)
    #if opt == None:
    #    r.assign("opt", light[~np.isnan(light)])
    #else:
    #    r.assign("opt", opt[~np.isnan(opt)])

    #if ini == None:
    #    r.assign('ini', [0.4,1.5,1500])

    #else:
    #    r.assign('ini', np.array(ini))

    #op, platt_param = platt(light,etr, ini=ini)
    #r.assign('platt_param', platt_param)

    min_opt = r("""
    min_opt<-function(light,params){
        alpha<-params[1]
        Beta<-params[2]
        Ps<-params[3]
        return( ( (Ps*(1-exp(-alpha*light/Ps)) *exp(-Beta*light/Ps)) ) )
    }""")

    opts = np.append(opts, r('min_opt(light, params)'))

    return opts

def eilers_peeters(light,etr,ini=None):
    '''
    Adjust a best fit curve to ExP curves, according to Eilers  & Peters
    Model.

    Parameters
    ----------
    light : arr
        Generally PAR values. Where Photosynthetic Active Radiance
        interfer on Primary Production. 'light' is this parameter.

    etr : arr
        Eletron Transference Rate, by means relative ETR, obtained from
        Rapid Light Curves.

    ini : None
        Initial values values to set the curve.
        To insert initial values, they must be a list
        of values of initial parameters (a,b,c) of Eilers-Peeters models

    Return
    ------
    opts : arr
        Values optimized

    params : arr
        Curve Parameters (alpha, Ek, ETR_max)

    See Also
    --------
    P.H.C. Eilers and J.C.H Peeters. 1988. A model for the relationship
    between the light intensity and the rate of photosynthesis in
    phytoplankton. Ecol. Model. 42:199-215.

    #TODO
    ## Implement minimisation in Python.
    ## It's not very clear how to apply `nls2` in Python.
    ## minimize from a list of initial values.

    ##a = varis[0]
    ##b = varis[1]
    ##c = varis[2]
    #a = mini['a']
    #b = mini['b']
    #c = mini['c']

    #opts = (light/(a*(light**2)+(b*light)+c))
    #ad = fmin(ep_minimize,varis,args=(light,etr))

    #alpha = (1./ad[2])
    #etrmax = 1./(ad[1]+2*(ad[0]*ad[2])**0.5)
    #Eopt = (ad[2]/ad[0])**0.5
    #Ek = etrmax/alpha

    #params = [alpha, Ek, etrmax, Eopt]
    '''
    r('library(nls2)')
    r.assign("x", light[~np.isnan(light)])
    r.assign("y", etr[~np.isnan(etr)])
    r('dat<-as.data.frame(cbind(x,y))')
    r('names(dat)<-c("light","etr")')

    if ini == None:
        r('''grid<-expand.grid(list(a=seq(1e-07,9e-06,by=2e-07),
        b=seq(-0.002,0.006,by=0.002),c=seq(-6,6,by=2)))''')
        mini = r('''
            mini<-coefficients(nls2(etr~light/(a*light^2+b*light+c),
            data=dat, start=grid, algorithm="brute-force"))
            ''')
    else:
        mini = ini
        r.assign("mini", mini)

    r('''ep<-nls(etr~light/(a*light^2+b*light+c),data=dat,
    start=list(a=mini[1],b=mini[2],c=mini[3]),
    lower = list(0,-Inf,-Inf), trace=FALSE,
    algorithm = "port", nls.control("maxiter"=100000, tol=0.15))

    a2<-summary(ep)$coefficients[1]
    b2<-summary(ep)$coefficients[2]
    c2<-summary(ep)$coefficients[3]

    alpha<-(1/c2)
    etrmax<-1/(b2+2*(a2*c2)^0.5)
    Eopt<-(c2/a2)^0.5
    Ek<-etrmax/alpha''')

    alpha = r('alpha')
    Ek = r('Ek')
    etr_max = r('etrmax')
    params = [alpha, Ek, etr_max]
    opts = r('opts<-fitted(ep)')

    return opts, params
<|endoftext|>


<commit_before>#  Copyright 2017-2022 John Snow Labs
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
import os
import unittest

import pytest

from sparknlp.annotator import *
from sparknlp.base import *
from pyspark.sql.functions import col

from test.util import SparkSessionForTest

@pytest.mark.fast
class Wav2Vec2ForCTCTestSpec(unittest.TestCase):
    def setUp(self):
        audio_path = os.getcwd() + "/../src/test/resources/audio/json/audio_floats.json"
        self.data = SparkSessionForTest.spark.read.option("inferSchema", value=True).json(audio_path) \
            .select(col("float_array").cast("array<float>").alias("audio_content"))

    def runTest(self):
        self.data.show()
        audio_assembler = AudioAssembler() \
            .setInputCol("audio_content") \
            .setOutputCol("audio_assembler")

        speech_to_text = Wav2Vec2ForCTC \
            .pretrained()\
            .setInputCols("audio_assembler") \
            .setOutputCol("text")

        pipeline = Pipeline(stages=[
            audio_assembler,
            speech_to_text,
        ])

        model = pipeline.fit(self.data)
        result_df = model.transform(self.data)
        assert result_df.select("text").count() > 0
<commit_msg>Disable Wav2Vec2 unit test in Python<commit_after>#  Copyright 2017-2022 John Snow Labs
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
import os
import unittest

import pytest

from sparknlp.annotator import *
from sparknlp.base import *
from pyspark.sql.functions import col

from test.util import SparkSessionForTest

@pytest.mark.slow
class Wav2Vec2ForCTCTestSpec(unittest.TestCase):
    def setUp(self):
        audio_path = os.getcwd() + "/../src/test/resources/audio/json/audio_floats.json"
        self.data = SparkSessionForTest.spark.read.option("inferSchema", value=True).json(audio_path) \
            .select(col("float_array").cast("array<float>").alias("audio_content"))

    def runTest(self):
        self.data.show()
        audio_assembler = AudioAssembler() \
            .setInputCol("audio_content") \
            .setOutputCol("audio_assembler")

        speech_to_text = Wav2Vec2ForCTC \
            .pretrained()\
            .setInputCols("audio_assembler") \
            .setOutputCol("text")

        pipeline = Pipeline(stages=[
            audio_assembler,
            speech_to_text,
        ])

        model = pipeline.fit(self.data)
        result_df = model.transform(self.data)
        assert result_df.select("text").count() > 0
<|endoftext|>


<commit_before>
import os
import Utils
import Process
import BuildSystem

# Directory where the clReflect executables are located
_InstallLocation = None

def _MakePath(filename):

    if _InstallLocation:
        return os.path.join(_InstallLocation, filename)
    return filename

class CppExportNode(BuildSystem.Node):

    def __init__(self, path, input, map_file):

        super().__init__()
        self.Path = path
        self.Input = input
        self.MapFile = map_file
        self.Dependencies = [ input, map_file ]

    def Build(self, env):

        input_file = self.GetInputFile(env)
        output_file = self.GetOutputFiles(env)[0]
        print("clexport: " + os.path.basename(output_file))

        # Construct the command-line
        cmdline = [ _MakePath("clexport.exe") ]
        cmdline += [ input_file ]
        cmdline += [ "-cpp", output_file ]
        cmdline += [ "-cpp_log", output_file + ".log" ]
        if self.MapFile != None:
            cmdline += [ "-map", self.MapFile.GetOutputFiles(env)[0] ]
        Utils.ShowCmdLine(env, cmdline)

        # Launch the exporter and wait for it to finish
        process = Process.OpenPiped(cmdline)
        output = Process.WaitForPipeOutput(process)
        if not env.NoToolOutput:
            print(output)

        return process.returncode == 0

    def GetInputFile(self, env):

        return self.Input.GetOutputFiles(env)[0]

    def GetOutputFiles(self, env):

        path = os.path.join(env.CurrentConfig.OutputPath, self.Path)
        return [ path ]

    def GetTempOutputFiles(self, env):

        return self.GetOutputFiles(env)

class MergeNode (BuildSystem.Node):

    def __init__(self, path, db_files, cpp_codegen):

        super().__init__()
        self.Path = path
        self.Dependencies = db_files
        self.CppCodeGen = cpp_codegen

    def Build(self, env):

        output_file = self.GetOutputFiles(env)[0]
        print("clmerge: " + os.path.basename(output_file))

        # Construct the command-line
        cmdline = [ _MakePath("clmerge.exe") ]
        cmdline += [ output_file ]
        if self.CppCodeGen != None:
            cmdline += [ "-cpp_codegen", self.CppCodeGen.GetInputFile(env) ]
        cmdline += [ file.GetOutputFiles(env)[0] for file in self.Dependencies ]
        Utils.ShowCmdLine(env, cmdline)

        # Launch the merger and wait for it to finish
        process = Process.OpenPiped(cmdline)
        output = Process.WaitForPipeOutput(process)
        if not env.NoToolOutput:
            print(output)

        return process.returncode == 0

    def GetInputFile(self, env):

        path = os.path.join(env.CurrentConfig.IntermediatePath, self.Path)
        return path

    def GetOutputFiles(self, env):

        output_files = [ os.path.join(env.CurrentConfig.IntermediatePath, self.Path) ]
        if self.CppCodeGen != None:
            output_files += [ self.CppCodeGen.GetInputFile(env) ]
        return output_files

    def GetTempOutputFiles(self, env):

        # Exclude the output C++ file as we don't want that to be deleted
        temp_files = self.GetOutputFiles(env)[:1]
        return temp_files

class CppScanNode (BuildSystem.Node):

    def __init__(self, sys_include_paths, include_paths, cpp_output):

        super().__init__()
        self.SysIncludePaths = sys_include_paths
        self.IncludePaths = include_paths
        self.CppOutput = cpp_output
        self.Dependencies = [ cpp_output ]

    def Build(self, env):

        input_file = self.GetInputFile(env)
        output_files = self.GetOutputFiles(env)
        print("clscan: " + os.path.basename(input_file))

        # Construct the command-line
        cmdline = [ _MakePath("clscan.exe") ]
        cmdline += [ input_file ]
        cmdline += [ "-output", output_files[0] ]
        cmdline += [ "-ast_log", output_files[1] ]
        cmdline += [ "-spec_log", output_files[2] ]
        for path in self.SysIncludePaths:
            cmdline += [ "-isystem", path ]
        for path in self.IncludePaths:
            cmdline += [ "-i", path ]
        Utils.ShowCmdLine(env, cmdline)

        # Launch the scanner and wait for it to finish
        output = Utils.IncludeScanner(env, "Included:", None, lambda line, length: line[length:].lstrip())
        process = Process.OpenPiped(cmdline)
        Process.WaitForPipeOutput(process, output)

        return process.returncode == 0

    def GetInputFile(self, env):

        return self.CppOutput.GetInputFile(env)

    def GetOutputFiles(self, env):

        path = os.path.splitext(self.GetInputFile(env))[0]
        path = os.path.join(env.CurrentConfig.IntermediatePath, path)
        return [ path + ".csv", path + "_astlog.txt", path + "_speclog.txt" ]

    def GetTempOutputFiles(self, env):

        return self.GetOutputFiles(env)

def CppScan(sys_include_paths, include_paths, cpp_output):
    return CppScanNode(sys_include_paths, include_paths, cpp_output)

def Merge(path, db_files, cpp_codegen):
    return MergeNode(path, db_files, cpp_codegen)

def CppExport(path, input, map_file):
    return CppExportNode(path, input, map_file)

def SetInstallLocation(location):
    global _InstallLocation
    _InstallLocation = location<commit_msg>Add defines for clReflect.<commit_after>
import os
import Utils
import Process
import BuildSystem

# Directory where the clReflect executables are located
_InstallLocation = None

def _MakePath(filename):

    if _InstallLocation:
        return os.path.join(_InstallLocation, filename)
    return filename

class CppExportNode(BuildSystem.Node):

    def __init__(self, path, input, map_file):

        super().__init__()
        self.Path = path
        self.Input = input
        self.MapFile = map_file
        self.Dependencies = [ input, map_file ]

    def Build(self, env):

        input_file = self.GetInputFile(env)
        output_file = self.GetOutputFiles(env)[0]
        print("clexport: " + os.path.basename(output_file))

        # Construct the command-line
        cmdline = [ _MakePath("clexport.exe") ]
        cmdline += [ input_file ]
        cmdline += [ "-cpp", output_file ]
        cmdline += [ "-cpp_log", output_file + ".log" ]
        if self.MapFile != None:
            cmdline += [ "-map", self.MapFile.GetOutputFiles(env)[0] ]
        Utils.ShowCmdLine(env, cmdline)

        # Launch the exporter and wait for it to finish
        process = Process.OpenPiped(cmdline)
        output = Process.WaitForPipeOutput(process)
        if not env.NoToolOutput:
            print(output)

        return process.returncode == 0

    def GetInputFile(self, env):

        return self.Input.GetOutputFiles(env)[0]

    def GetOutputFiles(self, env):

        path = os.path.join(env.CurrentConfig.OutputPath, self.Path)
        return [ path ]

    def GetTempOutputFiles(self, env):

        return self.GetOutputFiles(env)

class MergeNode (BuildSystem.Node):

    def __init__(self, path, db_files, cpp_codegen):

        super().__init__()
        self.Path = path
        self.Dependencies = db_files
        self.CppCodeGen = cpp_codegen

    def Build(self, env):

        output_file = self.GetOutputFiles(env)[0]
        print("clmerge: " + os.path.basename(output_file))

        # Construct the command-line
        cmdline = [ _MakePath("clmerge.exe") ]
        cmdline += [ output_file ]
        if self.CppCodeGen != None:
            cmdline += [ "-cpp_codegen", self.CppCodeGen.GetInputFile(env) ]
        cmdline += [ file.GetOutputFiles(env)[0] for file in self.Dependencies ]
        Utils.ShowCmdLine(env, cmdline)

        # Launch the merger and wait for it to finish
        process = Process.OpenPiped(cmdline)
        output = Process.WaitForPipeOutput(process)
        if not env.NoToolOutput:
            print(output)

        return process.returncode == 0

    def GetInputFile(self, env):

        path = os.path.join(env.CurrentConfig.IntermediatePath, self.Path)
        return path

    def GetOutputFiles(self, env):

        output_files = [ os.path.join(env.CurrentConfig.IntermediatePath, self.Path) ]
        if self.CppCodeGen != None:
            output_files += [ self.CppCodeGen.GetInputFile(env) ]
        return output_files

    def GetTempOutputFiles(self, env):

        # Exclude the output C++ file as we don't want that to be deleted
        temp_files = self.GetOutputFiles(env)[:1]
        return temp_files

class CppScanNode (BuildSystem.Node):

    def __init__(self, sys_include_paths, include_paths, defines, cpp_output):

        super().__init__()
        self.SysIncludePaths = sys_include_paths
        self.IncludePaths = include_paths
        self.CppOutput = cpp_output
        self.Defines = defines
        self.Dependencies = [ cpp_output ]

    def Build(self, env):

        input_file = self.GetInputFile(env)
        output_files = self.GetOutputFiles(env)
        print("clscan: " + os.path.basename(input_file))

        # Construct the command-line
        cmdline = [ _MakePath("clscan.exe") ]
        cmdline += [ input_file ]
        cmdline += [ "-output", output_files[0] ]
        cmdline += [ "-ast_log", output_files[1] ]
        cmdline += [ "-spec_log", output_files[2] ]
        for path in self.SysIncludePaths:
            cmdline += [ "-isystem", path ]
        for path in self.IncludePaths:
            cmdline += [ "-i", path ]
        for define in self.Defines:
            cmdline += [ "-D", define ]
        Utils.ShowCmdLine(env, cmdline)

        # Launch the scanner and wait for it to finish
        output = Utils.IncludeScanner(env, "Included:", None, lambda line, length: line[length:].lstrip())
        process = Process.OpenPiped(cmdline)
        Process.WaitForPipeOutput(process, output)

        return process.returncode == 0

    def GetInputFile(self, env):

        return self.CppOutput.GetInputFile(env)

    def GetOutputFiles(self, env):

        path = os.path.splitext(self.GetInputFile(env))[0]
        path = os.path.join(env.CurrentConfig.IntermediatePath, path)
        return [ path + ".csv", path + "_astlog.txt", path + "_speclog.txt" ]

    def GetTempOutputFiles(self, env):

        return self.GetOutputFiles(env)

def CppScan(sys_include_paths, include_paths, defines, cpp_output):
    return CppScanNode(sys_include_paths, include_paths, defines, cpp_output)

def Merge(path, db_files, cpp_codegen):
    return MergeNode(path, db_files, cpp_codegen)

def CppExport(path, input, map_file):
    return CppExportNode(path, input, map_file)

def SetInstallLocation(location):
    global _InstallLocation
    _InstallLocation = location<|endoftext|>


<commit_before># Example call for the isochrone API at Walkalytics
# see <https://dev.walkalytics.com> for details
#
# (c) Ernst Basler + Partner
#
import urllib
import urllib2
import json
import re
import math
import netrc
import sys

# Configuration

# - read x and y coordinates from command line arguments, or just use default coords.
try:
    x,y = sys.argv[1:3]
except ValueError:
    x,y = 828895, 5932832
print("Source location is ({},{}).".format(x,y))

# - set the api_key by reading a netrc file with API key.
try:
    secrets = netrc.netrc("walkalytics-api-key")
except IOError:
    print("[Error] Create a file 'walkalytics-api-key' with your API key from https://dev.walkalytics.com.")
    print("[Error] See 'walkalytics-api-key.sample' for the required netrc-style format.")
    sys.exit(-1)

_, _, api_key = secrets.authenticators("api.walkalytics.com")
# - Alternatively, you can replace the two lines above with explicitly set
#   api_key with your key from https://dev.walkalytics.com/developer

# - set API parameters and set of POIs
params = {
    'x' : x,
    'y' : y,
    "only_pois": 'false', # or "true"
    "outputsize": 512,    # max is currently 720
    # more parameters are possible, see API documentation
    'key': api_key
}

# - POIs may also be an empty dict
pois = { "type": "FeatureCollection",
         "crs": {"type": "EPSG", "properties": {"code": 3857}},
         "features": [
             { "type": "Feature",
               "geometry": {"type": "Point", "coordinates": [828895,5932832]},
               "properties": {"id": "Hello!"}
           },
             { "type": "Feature",
               "geometry": {"type": "Point", "coordinates": [827894,5934204]},
               "properties": {"id": "Is it me you're looking for?"}
           }
         ]
     }

# - define API call URL
url = 'https://api.walkalytics.com/v1/isochrone'

# Now make the actual API call as a POST request
try:
    req = urllib2.Request("{0}?{1}".format(url,urllib.urlencode(params)),
                          json.dumps(pois),
                          {'Content-Type': 'application/json'})
    response = urllib2.urlopen(req)
    result = json.loads(response.read())
except urllib2.HTTPError, e:
    if e.code == 204:
        pass
    else:
        print("Error with request: {0}".format(e))
        sys.exit(-1)

# Check status
if result.get("status") != "success":
    print("API call did not succeed. Details: {}".format(result.get("msg")))
    sys.exit(-1)

# Print out service area in pixel and walking time in minutes for each POI
service_area_km2 = (result['service_area']) / 1000000.0
print("The service area for the source location is {:.2f} square km.".format(service_area_km2))
if result['pois'].has_key('features'):
    for poi in result['pois']['features']:
        id = poi['properties']['id']
        t  = poi['properties']['time']
        if t == -1:
            print("- POI '{0}' is not covered by the service area of the source point.".format(id))
        else:
            print("- POI '{0}' has a walking time of {1} minutes to the source point.".format(id,t/60))
else:
    print("No POIs found in the result.")

# Now save the image as a PNG file with an accompanying world file
if not result.has_key('img'):
    print("No image was requested.")
else:
    basename = "isochrone_{0}_{1}".format(params['x'], params['y'])
    print("Save image {0}.png with an accompanying world file.".format(basename))

    # - do a regexp match for extension, code and actual data (this is slightly
    #   more general than necessary)
    pattern = re.compile("^data:.+\/(.+);(.+),(.*)$")
    match = pattern.search(result['img'])
    # - extract extension, coding and actual data
    extension = match.group(1)
    code = match.group(2)
    data = match.group(3)
    assert(code == "base64") # make sure we have base64
    assert(extension == "png") # make sure we have a PNG file

    # - write image to a file
    with open("{0}.{1}".format(basename,extension),"wb") as f:
        f.write(data.decode(code))

    # Now save the world file . The parameters for the world file are calculated based on
    # the lower-left corner, the number of rows and the cellsize.
    with open("{0}.pgw".format(basename),"wb") as f:
        f.write("{}\n0.0\n0.0\n".format(result["cellsize"]))
        f.write("{}\n{}\n{}\n".format(-result["cellsize"],
                                      result["xllcorner"],
                                      result["yllcorner"]+(result["cellsize"]*result["nrows"])))

<commit_msg>parallelization + download of raw data<commit_after># Example call for the isochrone API at Walkalytics
# see <https://dev.walkalytics.com> for details
#
# (c) Ernst Basler + Partner
#
import urllib
import urllib2
import json
import re
import math
import netrc
import sys
import time

from multiprocessing import Pool, freeze_support
from multiprocessing.dummy import Pool as ThreadPool

# Configuration

# - read x and y coordinates from command line arguments, or just use default coords.
try:
    sources  = []
    l = sys.argv[1:]
    for el in l:
        sources.append(tuple([int(x) for x in el.split(",")]))
        assert(len(sources[-1])==2)
except AssertionError:
    print "Error: Arguments should be of the form : x1,y1 (x2,y2) (x3,y3) ..."
    sys.exit(-1)
except ValueError:
    sources  = [ (828895, 5932832) ]

# - set the api_key by reading a netrc file with API key.
try:
    secrets = netrc.netrc("walkalytics-api-key")
except IOError:
    print("[Error] Create a file 'walkalytics-api-key' with your API key from https://dev.walkalytics.com.")
    print("[Error] See 'walkalytics-api-key.sample' for the required netrc-style format.")
    sys.exit(-1)

_, _, api_key = secrets.authenticators("api.walkalytics.com")
# - Alternatively, you can replace the two lines above with explicitly set
#   api_key with your key from https://dev.walkalytics.com/developer

# - POIs may also be an empty dict
pois = { "type": "FeatureCollection",
         "crs": {"type": "EPSG", "properties": {"code": 3857}},
         "features": [
             { "type": "Feature",
               "geometry": {"type": "Point", "coordinates": [828895,5932832]},
               "properties": {"id": "Hello!"}
           },
             { "type": "Feature",
               "geometry": {"type": "Point", "coordinates": [827894,5934204]},
               "properties": {"id": "Is it me you're looking for?"}
           }
         ]
     }

# - define API call URL
url = 'https://api.walkalytics.com/v1/isochrone'

def parse_base64(s):
    # - do a regexp match for extension, code and actual data (this is slightly
    #   more general than necessary)
    pattern = re.compile("^data:.+\/(.+);(.+),(.*)$")
    match = pattern.search(s)
    # - extract extension, coding and actual data
    extension = match.group(1)
    code = match.group(2)
    data = match.group(3)
    return (extension, code, data)

def call_walkalytics(source,basename="isochrone"):
    x,y =  source

    print("Source location is ({},{}).".format(x,y))

    # - set API parameters and set of POIs
    params = {
        'x' : x,
        'y' : y,
        "only_pois": 'false', # or "true"
        "outputsize": 512,    # max is currently 720
        # more parameters are possible, see API documentation
        'key': api_key,
        'raw_data': 'true'
    }

    # Now make the actual API call as a POST request
    try:
        req = urllib2.Request("{0}?{1}".format(url,urllib.urlencode(params)),
                              json.dumps(pois),
                              {'Content-Type': 'application/json'})
        response = urllib2.urlopen(req)
        result = json.loads(response.read())
    except urllib2.HTTPError, e:
        if e.code == 204:
            pass
        else:
            print("Error with request: {0}".format(e))
            return False

    # Check status
    if result.get("status") != "success":
        print("API call did not succeed. Details: {}".format(result.get("msg")))
        return False

    # Print out service area in pixel and walking time in minutes for each POI
    service_area_km2 = (result['service_area']) / 1000000.0
    print("The service area for the source location is {:.2f} square km.".format(service_area_km2))
    if result['pois'].has_key('features'):
        for poi in result['pois']['features']:
            id = poi['properties']['id']
            t  = poi['properties']['time']
            if t == -1:
                print("- POI '{0}' is not covered by the service area of the source point.".format(id))
            else:
                print("- POI '{0}' has a walking time of {1} minutes to the source point.".format(id,t/60))
    else:
        print("No POIs found in the result.")

    # Now save the image as a PNG file with an accompanying world file
    if result.has_key('img'):
        filename = "{}_{}_{}".format(basename,params['x'], params['y'])
        print("Save image {0}.png with an accompanying world file.".format(filename))

        (extension, code, data) = parse_base64(result['img'])
        assert(code == "base64") # make sure we have base64
        assert(extension == "png") # make sure we have a PNG file

        # - write image to a file
        with open("{0}.{1}".format(filename,extension),"wb") as f:
            f.write(data.decode(code))

        # Now save the world file . The parameters for the world file are calculated based on
        # the lower-left corner, the number of rows and the cellsize.
        with open("{0}.pgw".format(filename),"wb") as f:
            f.write("{}\n0.0\n0.0\n".format(result["cellsize"]))
            f.write("{}\n{}\n{}\n".format(-result["cellsize"],
                                          result["xllcorner"],
                                          result["yllcorner"]+(result["cellsize"]*result["nrows"])))
    elif result.has_key("raw_data"):
        filename = "{}_{}_{}".format(basename,params['x'], params['y'])
        print("Save image {0}.asc.gz.".format(filename))
        (extension, code, data) = parse_base64(result['raw_data'])
        assert(code == "base64") # make sure we have base64
        assert(extension == "gzip") # make sure we have a .gz file
        # - write image to a file
        with open("{0}.asc.gz".format(filename),"wb") as f:
            f.write(data.decode(code))
    else:
        print("No image was requested.")

    return True

if __name__ == '__main__':
    t0 = time.clock()
    freeze_support()
    pool = ThreadPool(4)
    pool.map(call_walkalytics, sources)
    pool.close()
    pool.join()

    t1 = time.clock()

    print("Effective seconds per call: {:.4f}s".format((t1-t0)/len(sources)))

    # isochrone.py 828895,5932832 828895,5932833 828895,5932834 828895,5932835 828895,5932836 828895,5932837 828895,5932838 828895,5932839 828895,5932830 828895,5932831
    # for source in sources:
    #     call_walkalytics(source)

<|endoftext|>


<commit_before># encoding: utf-8
import datetime
from south.db import db
from south.v2 import DataMigration
from django.db import models
from knesset.mks.models import Member

class Migration(DataMigration):

    def forwards(self, orm):
        "Write your forwards methods here."
        for mk in Member.objects.all():
            mk.recalc_bill_statistics()

    def backwards(self, orm):
        "Write your backwards methods here."

    models = {
        'auth.group': {
            'Meta': {'object_name': 'Group'},
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
        },
        'auth.permission': {
            'Meta': {'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
        },
        'auth.user': {
            'Meta': {'object_name': 'User'},
            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'blank': 'True'}),
            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}),
            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}),
            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
        },
        'contenttypes.contenttype': {
            'Meta': {'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
        },
        'mks.correlation': {
            'Meta': {'object_name': 'Correlation'},
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'm1': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'m1'", 'to': "orm['mks.Member']"}),
            'm2': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'m2'", 'to': "orm['mks.Member']"}),
            'normalized_score': ('django.db.models.fields.FloatField', [], {'null': 'True'}),
            'not_same_party': ('django.db.models.fields.NullBooleanField', [], {'null': 'True', 'blank': 'True'}),
            'score': ('django.db.models.fields.IntegerField', [], {'default': '0'})
        },
        'mks.member': {
            'Meta': {'object_name': 'Member'},
            'area_of_residence': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}),
            'bills_stats_approved': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
            'bills_stats_first': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
            'bills_stats_pre': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
            'bills_stats_proposed': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
            'blog': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['planet.Blog']", 'unique': 'True', 'null': 'True', 'blank': 'True'}),
            'current_party': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'members'", 'null': 'True', 'to': "orm['mks.Party']"}),
            'current_role_descriptions': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}),
            'date_of_birth': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
            'date_of_death': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}),
            'end_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
            'family_status': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}),
            'fax': ('django.db.models.fields.CharField', [], {'max_length': '20', 'null': 'True', 'blank': 'True'}),
            'gender': ('django.db.models.fields.CharField', [], {'max_length': '1', 'null': 'True', 'blank': 'True'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'img_url': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'}),
            'is_current': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'blank': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
            'number_of_children': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
            'parties': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'all_members'", 'symmetrical': 'False', 'through': "orm['mks.Membership']", 'to': "orm['mks.Party']"}),
            'phone': ('django.db.models.fields.CharField', [], {'max_length': '20', 'null': 'True', 'blank': 'True'}),
            'place_of_birth': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}),
            'place_of_residence': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}),
            'place_of_residence_lat': ('django.db.models.fields.CharField', [], {'max_length': '16', 'null': 'True', 'blank': 'True'}),
            'place_of_residence_lon': ('django.db.models.fields.CharField', [], {'max_length': '16', 'null': 'True', 'blank': 'True'}),
            'residence_centrality': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
            'residence_economy': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
            'start_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True', 'blank': 'True'}),
            'website': ('django.db.models.fields.URLField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
            'year_of_aliyah': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'})
        },
        'mks.membership': {
            'Meta': {'object_name': 'Membership'},
            'end_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'member': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['mks.Member']"}),
            'party': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['mks.Party']"}),
            'start_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'})
        },
        'mks.party': {
            'Meta': {'object_name': 'Party'},
            'end_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'is_coalition': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
            'number_of_members': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
            'number_of_seats': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
            'start_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'})
        },
        'mks.weeklypresence': {
            'Meta': {'object_name': 'WeeklyPresence'},
            'date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
            'hours': ('django.db.models.fields.FloatField', [], {'blank': 'True'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'member': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['mks.Member']"})
        },
        'planet.blog': {
            'Meta': {'object_name': 'Blog'},
            'date_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'title': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '255', 'blank': 'True'}),
            'url': ('django.db.models.fields.URLField', [], {'unique': 'True', 'max_length': '1024', 'db_index': 'True'})
        }
    }

    complete_apps = ['mks']
<commit_msg>fix migration mks/0016<commit_after># encoding: utf-8
import datetime
from south.db import db
from south.v2 import DataMigration
from django.db import models
from knesset.mks.models import Member

class Migration(DataMigration):

    def forwards(self, orm):
        for mk in Member.objects.all():
            mk.bills_stats_proposed = mk.bills.count()
            mk.bills_stats_pre      = mk.bills.filter(stage__in=['2','3','4','5','6']).count()
            mk.bills_stats_first    = mk.bills.filter(stage__in=['4','5','6']).count()
            mk.bills_stats_approved = mk.bills.filter(stage='6').count()

    def backwards(self, orm):
        "Write your backwards methods here."

    models = {
        'auth.group': {
            'Meta': {'object_name': 'Group'},
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
        },
        'auth.permission': {
            'Meta': {'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
        },
        'auth.user': {
            'Meta': {'object_name': 'User'},
            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'blank': 'True'}),
            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}),
            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}),
            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
        },
        'contenttypes.contenttype': {
            'Meta': {'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
        },
        'mks.correlation': {
            'Meta': {'object_name': 'Correlation'},
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'm1': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'m1'", 'to': "orm['mks.Member']"}),
            'm2': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'m2'", 'to': "orm['mks.Member']"}),
            'normalized_score': ('django.db.models.fields.FloatField', [], {'null': 'True'}),
            'not_same_party': ('django.db.models.fields.NullBooleanField', [], {'null': 'True', 'blank': 'True'}),
            'score': ('django.db.models.fields.IntegerField', [], {'default': '0'})
        },
        'mks.member': {
            'Meta': {'object_name': 'Member'},
            'area_of_residence': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}),
            'bills_stats_approved': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
            'bills_stats_first': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
            'bills_stats_pre': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
            'bills_stats_proposed': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
            'blog': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['planet.Blog']", 'unique': 'True', 'null': 'True', 'blank': 'True'}),
            'current_party': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'members'", 'null': 'True', 'to': "orm['mks.Party']"}),
            'current_role_descriptions': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}),
            'date_of_birth': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
            'date_of_death': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}),
            'end_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
            'family_status': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}),
            'fax': ('django.db.models.fields.CharField', [], {'max_length': '20', 'null': 'True', 'blank': 'True'}),
            'gender': ('django.db.models.fields.CharField', [], {'max_length': '1', 'null': 'True', 'blank': 'True'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'img_url': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'}),
            'is_current': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'blank': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
            'number_of_children': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
            'parties': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'all_members'", 'symmetrical': 'False', 'through': "orm['mks.Membership']", 'to': "orm['mks.Party']"}),
            'phone': ('django.db.models.fields.CharField', [], {'max_length': '20', 'null': 'True', 'blank': 'True'}),
            'place_of_birth': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}),
            'place_of_residence': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}),
            'place_of_residence_lat': ('django.db.models.fields.CharField', [], {'max_length': '16', 'null': 'True', 'blank': 'True'}),
            'place_of_residence_lon': ('django.db.models.fields.CharField', [], {'max_length': '16', 'null': 'True', 'blank': 'True'}),
            'residence_centrality': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
            'residence_economy': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
            'start_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True', 'blank': 'True'}),
            'website': ('django.db.models.fields.URLField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
            'year_of_aliyah': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'})
        },
        'mks.membership': {
            'Meta': {'object_name': 'Membership'},
            'end_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'member': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['mks.Member']"}),
            'party': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['mks.Party']"}),
            'start_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'})
        },
        'mks.party': {
            'Meta': {'object_name': 'Party'},
            'end_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'is_coalition': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
            'number_of_members': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
            'number_of_seats': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
            'start_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'})
        },
        'mks.weeklypresence': {
            'Meta': {'object_name': 'WeeklyPresence'},
            'date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
            'hours': ('django.db.models.fields.FloatField', [], {'blank': 'True'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'member': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['mks.Member']"})
        },
        'planet.blog': {
            'Meta': {'object_name': 'Blog'},
            'date_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'title': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '255', 'blank': 'True'}),
            'url': ('django.db.models.fields.URLField', [], {'unique': 'True', 'max_length': '1024', 'db_index': 'True'})
        }
    }

    complete_apps = ['mks']
<|endoftext|>


<commit_before>#! /usr/bin/env python

import copy
import argparse
import json
import operator
import re
import time

import config as _config
import executor

config = _config.Config()

class Flagger(executor.Executor):

    operators = {'>': operator.gt, '<': operator.lt, '==': operator.eq,
                 '>=': operator.ge, '<=': operator.le}

    def dprint(self, message):
        """
        If we're in debug or verbose mode, print message
        """
        if self.debug or self.verbose:
            print message

    def extract_threshold(self, token):
        """
        accept tokens of the format:
        int
        >=int
        <=int
        ==int
        >int
        <int
        returns [comparator, int] or throws error if invalid
        """
        comparator = re.sub("\d+$", "", token)
        value = int(re.sub("\D*", "", token))
        if comparator == '':  # no comparator specified
            comparator = '>='
        self.dprint("comparator: {} value: {}".format(comparator, token))
        assert comparator in self.operators
        return (comparator, value)

    def initialize_control(self):
        """
        sets up known control configuration based on control channel messages
        """
        channel = config.control_channel
        cid = self.slacker.get_channelid(channel)
        messages = self.slacker.get_messages_in_time_range(0, cid, time.time())
        control = {}
        for message in messages:
            text = message['text']
            tokens = text.split()
            if tokens[0:3] != ['flag', 'content', 'rule']:
                continue
            if len(tokens) < 5:
                self.ds.warning("control message {} has too few tokens".format(text))
                continue
            if len(tokens) == 5 and tokens[4] == 'delete':
                uuid = tokens[3]
                if uuid in control:
                    del(control[uuid])
                    self.dprint("Message {} deletes UUID {}".format(text, uuid))
                    continue
            try:
                tokens = text.split()
                uuid = tokens[3]
                comparator, threshold = self.extract_threshold(tokens[4])
                emoji = tokens[5].replace(":", "")
                output_channel_id = re.sub("[<>]", "", tokens[6])
                if output_channel_id.find("|") != -1:
                    cid, cname = output_channel_id.split("|")
                    output_channel_id = cid
                output_channel_name = self.slacker.replace_id(output_channel_id)
                control[uuid] = {'threshold': threshold, "comparator": comparator,
                                 'emoji': emoji, 'output': output_channel_name}
            except Exception, e:
                m = "Couldn't create flagger rule with text {}: {} {}".format(text, Exception, e)
                self.dprint(m)
                if not self.debug:
                    self.ds.warning(m)
        self.control = control
        self.dprint("control: {}".format(json.dumps(self.control, indent=4)))
        self.emoji = [x['emoji'] for x in self.control.values()]
        self.initialize_emoji_aliases()

    def initialize_emoji_aliases(self):
        """
        In some cases, emojiA might be an alias of emojiB
        The problem is that if we say that 2xemojiB should be
        enough to flag something, then we should accept
        2 x emojiB
        1 x emojiA, 1 x emojiB
        2 x emojiA
        This method grabs the emoji list from the Slack and creates the equivalence
        structure
        """
        self.dprint("Starting emoji alias list")
        emojis_response = self.slacker.get_emojis()
        self.dprint("emojis_response keys are {}".format(emojis_response.keys()))
        emojis = emojis_response['emoji']
        equivalents = {}
        for emoji in emojis:
            target = emojis[emoji]
            target_type, target_value = target.split(":", 1)
            if target_type != "alias":
                continue
            self.dprint("Found emoji alias: {} <-> {}".format(emoji, target_value))
            if emoji not in equivalents:
                equivalents[emoji] = []
            if target_value not in equivalents:
                equivalents[target_value] = []
            equivalents[emoji].append(target_value)
            equivalents[target_value].append(emoji)
        self.emoji_equivalents = equivalents
        self.dprint("equivalents: {}".format(json.dumps(self.emoji_equivalents, indent=4)))
        self.dprint("floppy_disk: {}".format(self.emoji_equivalents['floppy_disk']))

    def message_destination(self, message):
        """
        if interesting, returns channel name[s] in which to announce
        otherwise, returns []
        """
        channels = []
        if message.get("reactions") is None:
            return False
        reactions = message.get("reactions")
        emoji_set = set(self.emoji)
        current_reactions = {}
        t = message.get("text")
        if t.find("SVP") != -1:
            def d(p):
                pass  # print p
        else:
            def d(p):
                pass
        d("reactions: {}".format(reactions))
        d("emoji_equivalents:\n{}".format(json.dumps(self.emoji_equivalents, indent=4)))
        d("floppy_disk: {}".format(self.emoji_equivalents['floppy_disk']))
        for reaction in reactions:
            count = reaction['count']
            current_emoji = reaction['name']
            d("current_emoji: {}".format(current_emoji))
            equivalents = copy.copy(self.emoji_equivalents.get(current_emoji, []))
            d("equivalents = {}".format(equivalents))
            equivalents.append(current_emoji)
            d("equivalents = {}".format(equivalents))
            current_set = set(equivalents)
            i = current_set.intersection(emoji_set)
            if not i:
                continue
            for ce in equivalents:
                current_reactions[ce] = current_reactions.get(ce, 0) + count
            # if we're here, at least one emoji matches (but count may still not be right)
        d("Current reactions: {}".format(current_reactions))
        for uuid in self.control:
            rule = self.control[uuid]
            for ce in current_reactions:
                if ce == rule['emoji']:
                    count = current_reactions[ce]
                    threshold = rule['threshold']
                    comparator = rule['comparator']
                    op = self.operators[comparator]
                    if op(count, threshold):
                        channels.append(rule['output'])
        return channels

    def get_interesting_messages(self):
        """
        returns [[message, [listofchannelstoannounce]]
        """
        now = time.time()
        dayago = now - 86400

        messages = []
        for channel in self.slacker.channels_by_name:
            cid = self.slacker.get_channelid(channel)
            cur_messages = self.slacker.get_messages_in_time_range(dayago, cid, now)
            for message in cur_messages:
                announce = self.message_destination(message)
                if announce:
                    messages.append([message, announce])
        return messages

    def announce_interesting_messages(self):
        messages = self.get_interesting_messages()
        slack_name = _config.SLACK_NAME
        for message, channels in messages:
            ts = message['ts'].replace(".", "")
            channel = message['channel']
            author = message['user']
            author_name = self.slacker.users_by_id[author]
            text = self.slacker.asciify(message['text'])
            text = self.slacker.detokenize(text)
            url = "http://{}.slack.com/archives/{}/p{}".format(slack_name, channel, ts)
            m = "*@{}* said in *#{}* _'{}'_ ({})".format(author_name, channel, text, url)
            for output_channel in channels:
                md = "Saying {} to {}".format(m, output_channel)
                self.dprint(md)
                if not self.debug:
                    self.sb.say(output_channel, m)

    def flag(self):
        self.initialize_control()
        self.announce_interesting_messages()

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Flag interesting Slack messages.')
    parser.add_argument("--debug", action="store_true", default=False)
    parser.add_argument("--verbose", action="store_true", default=False)
    args = parser.parse_args()

    flagger = Flagger(debug=args.debug, verbose=args.verbose)
    flagger.flag()
<commit_msg>improve debug message when we can't parse a rule; also, html.unescape rule tokens<commit_after>#! /usr/bin/env python

import copy
import argparse
import json
import HTMLParser
import operator
import re
import time
import traceback

import config as _config
import executor

config = _config.Config()

class Flagger(executor.Executor):

    operators = {'>': operator.gt, '<': operator.lt, '==': operator.eq,
                 '>=': operator.ge, '<=': operator.le}

    def __init__(self, *args, **kwargs):
        self.htmlparser = HTMLParser.HTMLParser()
        super(Flagger, self).__init__(*args, **kwargs)

    def dprint(self, message):
        """
        If we're in debug or verbose mode, print message
        """
        if self.debug or self.verbose:
            print message

    def extract_threshold(self, token):
        """
        accept tokens of the format:
        int
        >=int
        <=int
        ==int
        >int
        <int
        returns [comparator, int] or throws error if invalid
        """
        comparator = re.sub("\d+$", "", token)
        value = int(re.sub("\D*", "", token))
        if comparator == '':  # no comparator specified
            comparator = '>='

        comparator = self.htmlparser.unescape(comparator)
        self.dprint("token: {} comparator: {} value: {}".format(token, comparator, value))

        assert comparator in self.operators
        return (comparator, value)

    def initialize_control(self):
        """
        sets up known control configuration based on control channel messages
        """
        channel = config.control_channel
        cid = self.slacker.get_channelid(channel)
        messages = self.slacker.get_messages_in_time_range(0, cid, time.time())
        control = {}
        for message in messages:
            text = message['text']
            tokens = text.split()
            if tokens[0:3] != ['flag', 'content', 'rule']:
                continue
            if len(tokens) < 5:
                self.ds.warning("control message {} has too few tokens".format(text))
                continue
            if len(tokens) == 5 and tokens[4] == 'delete':
                uuid = tokens[3]
                if uuid in control:
                    del(control[uuid])
                    self.dprint("Message {} deletes UUID {}".format(text, uuid))
                    continue
            try:
                tokens = text.split()
                uuid = tokens[3]
                comparator, threshold = self.extract_threshold(tokens[4])
                emoji = tokens[5].replace(":", "")
                output_channel_id = re.sub("[<>]", "", tokens[6])
                if output_channel_id.find("|") != -1:
                    cid, cname = output_channel_id.split("|")
                    output_channel_id = cid
                output_channel_name = self.slacker.replace_id(output_channel_id)
                control[uuid] = {'threshold': threshold, "comparator": comparator,
                                 'emoji': emoji, 'output': output_channel_name}
            except Exception, e:
                tb = traceback.format_exc()
                m = "Couldn't create flagger rule with text {}: {} {}".format(text, Exception, e)
                self.dprint(m)
                self.dprint(tb)
                if not self.debug:
                    self.ds.warning(m)
        self.control = control
        self.dprint("control: {}".format(json.dumps(self.control, indent=4)))
        self.emoji = [x['emoji'] for x in self.control.values()]
        self.initialize_emoji_aliases()

    def initialize_emoji_aliases(self):
        """
        In some cases, emojiA might be an alias of emojiB
        The problem is that if we say that 2xemojiB should be
        enough to flag something, then we should accept
        2 x emojiB
        1 x emojiA, 1 x emojiB
        2 x emojiA
        This method grabs the emoji list from the Slack and creates the equivalence
        structure
        """
        self.dprint("Starting emoji alias list")
        emojis_response = self.slacker.get_emojis()
        self.dprint("emojis_response keys are {}".format(emojis_response.keys()))
        emojis = emojis_response['emoji']
        equivalents = {}
        for emoji in emojis:
            target = emojis[emoji]
            target_type, target_value = target.split(":", 1)
            if target_type != "alias":
                continue
            self.dprint("Found emoji alias: {} <-> {}".format(emoji, target_value))
            if emoji not in equivalents:
                equivalents[emoji] = []
            if target_value not in equivalents:
                equivalents[target_value] = []
            equivalents[emoji].append(target_value)
            equivalents[target_value].append(emoji)
        self.emoji_equivalents = equivalents
        self.dprint("equivalents: {}".format(json.dumps(self.emoji_equivalents, indent=4)))
        self.dprint("floppy_disk: {}".format(self.emoji_equivalents['floppy_disk']))

    def message_destination(self, message):
        """
        if interesting, returns channel name[s] in which to announce
        otherwise, returns []
        """
        channels = []
        if message.get("reactions") is None:
            return False
        reactions = message.get("reactions")
        emoji_set = set(self.emoji)
        current_reactions = {}
        t = message.get("text")
        if t.find("SVP") != -1:
            def d(p):
                pass  # print p
        else:
            def d(p):
                pass
        d("reactions: {}".format(reactions))
        d("emoji_equivalents:\n{}".format(json.dumps(self.emoji_equivalents, indent=4)))
        d("floppy_disk: {}".format(self.emoji_equivalents['floppy_disk']))
        for reaction in reactions:
            count = reaction['count']
            current_emoji = reaction['name']
            d("current_emoji: {}".format(current_emoji))
            equivalents = copy.copy(self.emoji_equivalents.get(current_emoji, []))
            d("equivalents = {}".format(equivalents))
            equivalents.append(current_emoji)
            d("equivalents = {}".format(equivalents))
            current_set = set(equivalents)
            i = current_set.intersection(emoji_set)
            if not i:
                continue
            for ce in equivalents:
                current_reactions[ce] = current_reactions.get(ce, 0) + count
            # if we're here, at least one emoji matches (but count may still not be right)
        d("Current reactions: {}".format(current_reactions))
        for uuid in self.control:
            rule = self.control[uuid]
            for ce in current_reactions:
                if ce == rule['emoji']:
                    count = current_reactions[ce]
                    threshold = rule['threshold']
                    comparator = rule['comparator']
                    op = self.operators[comparator]
                    if op(count, threshold):
                        channels.append(rule['output'])
        return channels

    def get_interesting_messages(self):
        """
        returns [[message, [listofchannelstoannounce]]
        """
        now = time.time()
        dayago = now - 86400

        messages = []
        for channel in self.slacker.channels_by_name:
            cid = self.slacker.get_channelid(channel)
            cur_messages = self.slacker.get_messages_in_time_range(dayago, cid, now)
            for message in cur_messages:
                announce = self.message_destination(message)
                if announce:
                    messages.append([message, announce])
        return messages

    def announce_interesting_messages(self):
        messages = self.get_interesting_messages()
        slack_name = _config.SLACK_NAME
        for message, channels in messages:
            ts = message['ts'].replace(".", "")
            channel = message['channel']
            author = message['user']
            author_name = self.slacker.users_by_id[author]
            text = self.slacker.asciify(message['text'])
            text = self.slacker.detokenize(text)
            url = "http://{}.slack.com/archives/{}/p{}".format(slack_name, channel, ts)
            m = "*@{}* said in *#{}* _'{}'_ ({})".format(author_name, channel, text, url)
            for output_channel in channels:
                md = "Saying {} to {}".format(m, output_channel)
                self.dprint(md)
                if not self.debug:
                    self.sb.say(output_channel, m)

    def flag(self):
        self.initialize_control()
        self.announce_interesting_messages()

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Flag interesting Slack messages.')
    parser.add_argument("--debug", action="store_true", default=False)
    parser.add_argument("--verbose", action="store_true", default=False)
    args = parser.parse_args()

    flagger = Flagger(debug=args.debug, verbose=args.verbose)
    flagger.flag()
<|endoftext|>


<commit_before>from contentbase import (
    calculated_property,
    collection,
    load_schema,
)
from pyramid.security import Authenticated
from .base import (
    Item,
    paths_filtered_by_status,
)

class Donor(Item):
    base_types = ['Donor'] + Item.base_types
    embedded = ['organism', 'references']
    name_key = 'accession'
    rev = {
        'characterizations': ('DonorCharacterization', 'characterizes'),
    }

    @calculated_property(schema={
        "title": "Characterizations",
        "type": "array",
        "items": {
            "type": ['string', 'object'],
            "linkFrom": "DonorCharacterization.characterizes",
        },
    })
    def characterizations(self, request, characterizations):
        return paths_filtered_by_status(request, characterizations)

@collection(
    name='mouse-donors',
    acl=[],
    properties={
        'title': 'Mouse donors',
        'description': 'Listing Biosample Donors',
    })
class MouseDonor(Donor):
    item_type = 'mouse_donor'
    schema = load_schema('encoded:schemas/mouse_donor.json')

    def __ac_local_roles__(self):
        # Disallow lab submitter edits
        return {Authenticated: 'role.viewing_group_member'}

@collection(
    name='fly-donors',
    properties={
        'title': 'Fly donors',
        'description': 'Listing Biosample Donors',
    })
class FlyDonor(Donor):
    item_type = 'fly_donor'
    schema = load_schema('encoded:schemas/fly_donor.json')
    embedded = ['organism', 'constructs', 'constructs.target']

@collection(
    name='worm-donors',
    properties={
        'title': 'Worm donors',
        'description': 'Listing Biosample Donors',
    })
class WormDonor(Donor):
    item_type = 'worm_donor'
    schema = load_schema('encoded:schemas/worm_donor.json')
    embedded = ['organism', 'constructs', 'constructs.target']

@collection(
    name='human-donors',
    properties={
        'title': 'Human donors',
        'description': 'Listing Biosample Donors',
    })
class HumanDonor(Donor):
    item_type = 'human_donor'
    schema = load_schema('encoded:schemas/human_donor.json')
<commit_msg>Embed references<commit_after>from contentbase import (
    calculated_property,
    collection,
    load_schema,
)
from pyramid.security import Authenticated
from .base import (
    Item,
    paths_filtered_by_status,
)

class Donor(Item):
    base_types = ['Donor'] + Item.base_types
    embedded = ['organism']
    name_key = 'accession'
    rev = {
        'characterizations': ('DonorCharacterization', 'characterizes'),
    }

    @calculated_property(schema={
        "title": "Characterizations",
        "type": "array",
        "items": {
            "type": ['string', 'object'],
            "linkFrom": "DonorCharacterization.characterizes",
        },
    })
    def characterizations(self, request, characterizations):
        return paths_filtered_by_status(request, characterizations)

@collection(
    name='mouse-donors',
    acl=[],
    properties={
        'title': 'Mouse donors',
        'description': 'Listing Biosample Donors',
    })
class MouseDonor(Donor):
    item_type = 'mouse_donor'
    schema = load_schema('encoded:schemas/mouse_donor.json')
    embedded = Donor.embedded + ['references']

    def __ac_local_roles__(self):
        # Disallow lab submitter edits
        return {Authenticated: 'role.viewing_group_member'}

@collection(
    name='fly-donors',
    properties={
        'title': 'Fly donors',
        'description': 'Listing Biosample Donors',
    })
class FlyDonor(Donor):
    item_type = 'fly_donor'
    schema = load_schema('encoded:schemas/fly_donor.json')
    embedded = ['organism', 'constructs', 'constructs.target']

@collection(
    name='worm-donors',
    properties={
        'title': 'Worm donors',
        'description': 'Listing Biosample Donors',
    })
class WormDonor(Donor):
    item_type = 'worm_donor'
    schema = load_schema('encoded:schemas/worm_donor.json')
    embedded = ['organism', 'constructs', 'constructs.target']

@collection(
    name='human-donors',
    properties={
        'title': 'Human donors',
        'description': 'Listing Biosample Donors',
    })
class HumanDonor(Donor):
    item_type = 'human_donor'
    schema = load_schema('encoded:schemas/human_donor.json')
    embedded = Donor.embedded + ['references']
<|endoftext|>