<commit_before>import numpy
from numpy.testing import assert_equal, assert_almost_equal
from nose.tools import raises

from railgun import SimObject, relpath

LIST_IDX = list('ijklmnopqrstuvwxyz')

def get_str_get_array(cdt, dim):
    """
    Get c-function declaration 'get_{int,double,...}{1,2,..}d(i, j, ...)'

    >>> get_str_get_array('int', 1)
    'get_int1d(i)'
    >>> get_str_get_array('int', 2)
    'get_int2d(i, j)'
    >>> get_str_get_array('double', 1)
    'get_double1d(i)'
    >>> get_str_get_array('double', 2)
    'get_double2d(i, j)'

    """
    args = ', '.join(LIST_IDX[:dim])
    return 'get_%s%dd(%s)' % (cdt, dim, args)

def get_str_cddec(cdt, dim):
    """
    Get c-data declaration '{int,double,...} {int,double,...}{1,2,..}[i][j]...'

    >>> get_str_cddec('int', 1)
    'int int1d[i]'
    >>> get_str_cddec('int', 2)
    'int int2d[i][j]'
    >>> get_str_cddec('double', 1)
    'double double1d[i]'
    >>> get_str_cddec('double', 2)
    'double double2d[i][j]'

    """
    args = ']['.join(LIST_IDX[:dim])
    return '%s %s%dd[%s]' % (cdt, cdt, dim, args)

def gene_nums(nd):
    """
    Get list of num_* given maximum dimension

    >>> gene_nums(1)
    ['num_i']
    >>> gene_nums(4)
    ['num_i', 'num_j', 'num_k', 'num_l']

    """
    return ['num_%s' % data for data in LIST_IDX[:nd]]

def gene_cmembers(nd, list_cdt):
    """
    Get cmembers given maximum dimension and list of CDT (c data type)

    >>> gene_cmembers(2, ['int', 'double'])
    ['num_i', 'num_j', 'int ret_int', 'double ret_double', 'int int1d[i]', 'int int2d[i][j]', 'double double1d[i]', 'double double2d[i][j]']

    """
    return (
        gene_nums(nd) +
        ['%(cdt)s ret_%(cdt)s' % dict(cdt=cdt) for cdt in list_cdt] +
        [get_str_cddec(cdt, dim)
         for cdt in list_cdt for dim in range(1, 1 + nd)]
        )

def gene_cfuncs(nd, list_cdt):
    return [get_str_get_array(cdt, dim)
            for cdt in list_cdt for dim in range(1, 1 + nd)]

@numpy.vectorize
def mchr(i):
    return chr(i % 256)

def alpharange(*args):
    return mchr(numpy.arange(*args))

def gene_class_ArrayAccess(clibname, nd, _list_cdt):
    class ArrayAccess(SimObject):
        _clibname_ = clibname
        _clibdir_ = relpath('ext/build', __file__)
        _cmembers_ = gene_cmembers(nd, _list_cdt)
        _cfuncs_ = gene_cfuncs(nd, _list_cdt)
        num_names = gene_nums(nd)
        ndim = nd
        list_cdt = _list_cdt

        def get_arr(self, cdt, dim):
            """Get copy of array using 'get_{cdt}{dim}d' c-function"""
            get = getattr(self, 'get_%s%dd' % (cdt, dim))
            arr = getattr(self, '%s%dd' % (cdt, dim))
            garr = numpy.zeros_like(arr)
            for idx in numpy.ndindex(*arr.shape):
                get(*idx)  # call get_* c-func. which store val. in ret_{cdt}
                garr[idx] = getattr(self, 'ret_%s' % cdt)
            return garr

        def fill(self):
            for cdt in self.list_cdt:
                if cdt == 'char':
                    arange = alpharange
                else:
                    arange = numpy.arange
                for dim in range(1, 1 + self.ndim):
                    arr = getattr(self, '%s%dd' % (cdt, dim))
                    shape = arr.shape
                    arr[:] = arange(numpy.prod(shape)).reshape(shape)
    return ArrayAccess

def check_arrayaccess(clibname, list_num, list_cdt, cdt, dim):
    if cdt in ['char', 'short', 'ushort', 'int', 'uint', 'long', 'ulong',
               'longlong', 'ulonglong', 'bool']:
        ass_eq = assert_equal
    elif cdt in ['float', 'double', 'longdouble']:
        ass_eq = assert_almost_equal

    ArrayAccess = gene_class_ArrayAccess(
        clibname, len(list_num), list_cdt)
    num_dict = dict(zip(ArrayAccess.num_names, list_num))  # {num_i: 6, ...}
    aa = ArrayAccess(**num_dict)
    aa.fill()
    garr = aa.get_arr(cdt, dim)
    arr = getattr(aa, '%s%dd' % (cdt, dim))
    ass_eq(garr, arr)
    if cdt == 'char':
        arr.flat = alpharange(100, numpy.prod(arr.shape) + 100)
    else:
        arr += 100
    raises(AssertionError)(assert_equal)(garr, arr)
    garr2 = aa.get_arr(cdt, dim)
    assert_equal(garr2, arr)

if __name__ == '__main__':
    from pprint import pprint
    list_cdt = ['char', 'int', 'double']
    ndim = 3
    print '_cfuncs_'
    pprint(gene_cfuncs(ndim, list_cdt))
    print
    print '_cmembers_'
    pprint(gene_cmembers(ndim, list_cdt))
<commit_msg>organized arrayaccess.ArrayAccess: added arr, rename get_arr to arr_via_ret<commit_after>import numpy
from numpy.testing import assert_equal, assert_almost_equal
from nose.tools import raises

from railgun import SimObject, relpath

LIST_IDX = list('ijklmnopqrstuvwxyz')

def get_str_get_array(cdt, dim):
    """
    Get c-function declaration 'get_{int,double,...}{1,2,..}d(i, j, ...)'

    >>> get_str_get_array('int', 1)
    'get_int1d(i)'
    >>> get_str_get_array('int', 2)
    'get_int2d(i, j)'
    >>> get_str_get_array('double', 1)
    'get_double1d(i)'
    >>> get_str_get_array('double', 2)
    'get_double2d(i, j)'

    """
    args = ', '.join(LIST_IDX[:dim])
    return 'get_%s%dd(%s)' % (cdt, dim, args)

def get_str_cddec(cdt, dim):
    """
    Get c-data declaration '{int,double,...} {int,double,...}{1,2,..}[i][j]...'

    >>> get_str_cddec('int', 1)
    'int int1d[i]'
    >>> get_str_cddec('int', 2)
    'int int2d[i][j]'
    >>> get_str_cddec('double', 1)
    'double double1d[i]'
    >>> get_str_cddec('double', 2)
    'double double2d[i][j]'

    """
    args = ']['.join(LIST_IDX[:dim])
    return '%s %s%dd[%s]' % (cdt, cdt, dim, args)

def gene_nums(nd):
    """
    Get list of num_* given maximum dimension

    >>> gene_nums(1)
    ['num_i']
    >>> gene_nums(4)
    ['num_i', 'num_j', 'num_k', 'num_l']

    """
    return ['num_%s' % data for data in LIST_IDX[:nd]]

def gene_cmembers(nd, list_cdt):
    """
    Get cmembers given maximum dimension and list of CDT (c data type)

    >>> gene_cmembers(2, ['int', 'double'])
    ['num_i', 'num_j', 'int ret_int', 'double ret_double', 'int int1d[i]', 'int int2d[i][j]', 'double double1d[i]', 'double double2d[i][j]']

    """
    return (
        gene_nums(nd) +
        ['%(cdt)s ret_%(cdt)s' % dict(cdt=cdt) for cdt in list_cdt] +
        [get_str_cddec(cdt, dim)
         for cdt in list_cdt for dim in range(1, 1 + nd)]
        )

def gene_cfuncs(nd, list_cdt):
    return [get_str_get_array(cdt, dim)
            for cdt in list_cdt for dim in range(1, 1 + nd)]

@numpy.vectorize
def mchr(i):
    return chr(i % 256)

def alpharange(*args):
    return mchr(numpy.arange(*args))

def gene_class_ArrayAccess(clibname, nd, _list_cdt):
    class ArrayAccess(SimObject):
        _clibname_ = clibname
        _clibdir_ = relpath('ext/build', __file__)
        _cmembers_ = gene_cmembers(nd, _list_cdt)
        _cfuncs_ = gene_cfuncs(nd, _list_cdt)
        num_names = gene_nums(nd)
        ndim = nd
        list_cdt = _list_cdt

        def arr(self, cdt, dim):
            """Get array member '{cdt}{dim}d' (e.g., 'int2d')"""
            return getattr(self, '%s%dd' % (cdt, dim))

        def arr_via_ret(self, cdt, dim):
            """Get copy of array using 'get_{cdt}{dim}d' c-function"""
            get = getattr(self, 'get_%s%dd' % (cdt, dim))
            arr = self.arr(cdt, dim)
            garr = numpy.zeros_like(arr)
            for idx in numpy.ndindex(*arr.shape):
                get(*idx)  # call get_* c-func. which store val. in ret_{cdt}
                garr[idx] = getattr(self, 'ret_%s' % cdt)
            return garr

        def fill(self):
            """Fill array members using arange"""
            for cdt in self.list_cdt:
                if cdt == 'char':
                    arange = alpharange
                else:
                    arange = numpy.arange
                for dim in range(1, 1 + self.ndim):
                    arr = self.arr(cdt, dim)
                    arr.flat = arange(arr.size)
    return ArrayAccess

def check_arrayaccess(clibname, list_num, list_cdt, cdt, dim):
    if cdt in ['char', 'short', 'ushort', 'int', 'uint', 'long', 'ulong',
               'longlong', 'ulonglong', 'bool']:
        ass_eq = assert_equal
    elif cdt in ['float', 'double', 'longdouble']:
        ass_eq = assert_almost_equal

    ArrayAccess = gene_class_ArrayAccess(
        clibname, len(list_num), list_cdt)
    num_dict = dict(zip(ArrayAccess.num_names, list_num))  # {num_i: 6, ...}
    aa = ArrayAccess(**num_dict)
    aa.fill()
    # arr_via_ret should return same array (garr)
    garr = aa.arr_via_ret(cdt, dim)
    arr = aa.arr(cdt, dim)
    ass_eq(garr, arr)
    # insert completely different value to 'arr'
    if cdt == 'char':
        arr.flat = alpharange(100, numpy.prod(arr.shape) + 100)
    else:
        arr += 100
    raises(AssertionError)(assert_equal)(garr, arr)
    # get array (garr2) via arr_via_ret again
    garr2 = aa.arr_via_ret(cdt, dim)
    assert_equal(garr2, arr)

if __name__ == '__main__':
    from pprint import pprint
    list_cdt = ['char', 'int', 'double']
    ndim = 3
    print '_cfuncs_'
    pprint(gene_cfuncs(ndim, list_cdt))
    print
    print '_cmembers_'
    pprint(gene_cmembers(ndim, list_cdt))
<|endoftext|>


<commit_before>#!/usr/bin/env python

#Copyright (C) 2014 by Glenn Hickey
#
#Released under the MIT license, see LICENSE.txt
import unittest
import sys
import os
import argparse
import logging
import numpy as np
import math
import copy

from teHmm.track import TrackList
from teHmm.trackIO import readTrackData
from teHmm.common import myLog, EPSILON, initBedTool, cleanBedTool
from teHmm.common import addLoggingOptions, setLoggingFromOptions, logger
from teHmm.common import runShellCommand

def main(argv=None):
    if argv is None:
        argv = sys.argv

    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
        description="Automatically set the scale attributes of numeric tracks"
        " within a given tracks.xml function using some simple heuristics. ")

    parser.add_argument("tracksInfo", help="Path of Tracks Info file "
                        "containing paths to genome annotation tracks")
    parser.add_argument("numBins", help="Maximum number of bins after scaling",
                        default=10, type=int)
    parser.add_argument("outputTracks", help="Path to write modified tracks XML"
                        " to.")
    parser.add_argument("--tracks", help="Comma-separated list of tracks "
                        "to process. If not set, all"
                        " tracks listed as having a multinomial distribution"
                        " (since this is the default value, this includes "
                        "tracks with no distribution attribute) will be"
                        " processed.", default=None)
    parser.add_argument("--skip", help="Comma-separated list of tracks to "
                        "skip.", default=None)

    addLoggingOptions(parser)
    args = parser.parse_args()
    setLoggingFromOptions(args)
    tempBedToolPath = initBedTool()

    trackNames = []
    if args.tracks is not None:
        trackNames = args.tracks.split(",")
    skipNames = []
    if args.skip is not None:
        skipNames = args.skip.split(",")

    trackList = TrackList(args.tracksInfo)
    outTrackList = copy.deepcopy(trackList)

    for track in trackList:
        trackExt = os.path.splitext(track.getPath())[1]
        isFasta = len(trackExt) >= 3 and trackExt[:3].lower() == ".fa"
        if track.getName() not in skipNames and\
          (track.getName() in trackNames or len(trackNames) == 0) and\
          (track.getDist() == "multinomial" or
           track.getDist() == "sparse_multinomial") and\
          not isFasta:
            setTrackScale(track, args.numBins)

    trackList.saveXML(args.outputTracks)

def setTrackScale(track, numBins):
    """ Modify the track XML element in place with the heuristically
    computed scaling paramaters below """
    data = readTrackIntoFloatArray(track)
    if len(data) > numBins:
        scaleType, scaleParam = computeScale(data, numBins)
        # round down so xml file doesnt look too ugly
        if scaleParam > 1e-4:
            scaleParam = float("%.4f" % scaleParam)
        if scaleType == "scale":
            logger.info("Setting track %s scale to %f" % (track.getName(),
                                                          scaleParam))
            track.setScale(scaleParam)
        elif scaleType == "logScale":
            logger.info("Setting track %s logScale to %f" % (track.getName(),
                                                             scaleParam))
            track.setLogScale(scaleParam)

def readTrackIntoFloatArray(track):
    """ use the track API to directly read an entire data file into memory
    as an array of floats"""
    numLines = int(runShellCommand("wc -l %s" % track.getPath()).split()[0])
    assert numLines > 0
    data = np.finfo(np.float).max + np.zeros((numLines), dtype=np.float)
    readTrackData(track.getPath(), outputBuf=data, valCol=track.getValCol())
    lastIdx = len(data) - 1
    for i in xrange(1, len(data)):
        if data[-i] != np.finfo(np.float).max:
            lastIdx = len(data) - i
            break
    data = data[:lastIdx]
    assert data[-1] != np.finfo(np.float).max
    return data

def histVariance(data, bins, fromLog = False):
    """ use histogram variance as a proxy for quality of binning"""
    freq, bins = np.histogram(data, bins)
    if bins[0] <= 0 or fromLog is False:
        assert np.sum(freq) == len(data)
    else:
        tempBins = np.zeros((len(bins)+1))
        tempBins[1:] = bins
        tempFreq, tempBins = np.histogram(data, tempBins)
        assert np.sum(np.sum(tempFreq) == len(data))
    return np.var(freq)

def computeScale(data, numBins):
    """ very simple heuristic to compute reasonable scaling"""
    minVal, maxVal = np.amin(data), np.amax(data)
    range = maxVal - minVal
    logger.debug("Min=%f Max=%f" % (minVal, maxVal))

    # try linear scale
    binSize = float(range) / float(numBins - 2.0)
    minBin = np.floor(minVal / binSize) * binSize
    linearBins = [minBin] * numBins
    for i in xrange(1, numBins):
        linearBins[i] = linearBins[i-1] + binSize
    logger.debug("Linear bins %s" % linearBins)
    linearVar = histVariance(data, sorted(linearBins))
    linearScale = 1.0 / binSize
    logger.debug("Linear scale=%f has variance=%f" % (linearScale, linearVar))

    # try log scale
    if maxVal == 0.0:
        # dont support logscaling negative numbers for now
        logVar = sys.maxinit
    else:
        if minVal == 0:
            minVal = 1.0
        ratio = float(maxVal) / float(minVal)
        logBase = np.power(ratio, 1. / float(numBins - 2.00))
        minBin = np.power(logBase, np.floor(np.log(minVal) / np.log(logBase)))
        logBins = [minBin] * numBins
        for i in xrange(1, numBins):
            logBins[i] = logBins[i-1] * logBase
        logger.debug("Log bins %s" % logBins)
        logVar = histVariance(data, sorted(logBins), fromLog = True)
        logger.debug("Log base=%f has variance=%f" % (logBase, logVar))

    ret = "scale", linearScale
    if logVar < linearVar:
        ret = "logScale", logBase
    return ret

if __name__ == "__main__":
    sys.exit(main())
<commit_msg>dont abort for non numeric tracks<commit_after>#!/usr/bin/env python

#Copyright (C) 2014 by Glenn Hickey
#
#Released under the MIT license, see LICENSE.txt
import unittest
import sys
import os
import argparse
import logging
import numpy as np
import math
import copy

from teHmm.track import TrackList
from teHmm.trackIO import readTrackData
from teHmm.common import myLog, EPSILON, initBedTool, cleanBedTool
from teHmm.common import addLoggingOptions, setLoggingFromOptions, logger
from teHmm.common import runShellCommand

def main(argv=None):
    if argv is None:
        argv = sys.argv

    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
        description="Automatically set the scale attributes of numeric tracks"
        " within a given tracks.xml function using some simple heuristics. ")

    parser.add_argument("tracksInfo", help="Path of Tracks Info file "
                        "containing paths to genome annotation tracks")
    parser.add_argument("numBins", help="Maximum number of bins after scaling",
                        default=10, type=int)
    parser.add_argument("outputTracks", help="Path to write modified tracks XML"
                        " to.")
    parser.add_argument("--tracks", help="Comma-separated list of tracks "
                        "to process. If not set, all"
                        " tracks listed as having a multinomial distribution"
                        " (since this is the default value, this includes "
                        "tracks with no distribution attribute) will be"
                        " processed.", default=None)
    parser.add_argument("--skip", help="Comma-separated list of tracks to "
                        "skip.", default=None)

    addLoggingOptions(parser)
    args = parser.parse_args()
    setLoggingFromOptions(args)
    tempBedToolPath = initBedTool()

    trackNames = []
    if args.tracks is not None:
        trackNames = args.tracks.split(",")
    skipNames = []
    if args.skip is not None:
        skipNames = args.skip.split(",")

    trackList = TrackList(args.tracksInfo)
    outTrackList = copy.deepcopy(trackList)

    for track in trackList:
        trackExt = os.path.splitext(track.getPath())[1]
        isFasta = len(trackExt) >= 3 and trackExt[:3].lower() == ".fa"
        if track.getName() not in skipNames and\
          (track.getName() in trackNames or len(trackNames) == 0) and\
          (track.getDist() == "multinomial" or
           track.getDist() == "sparse_multinomial") and\
          not isFasta:
          try:
            setTrackScale(track, args.numBins)
          except ValueError as e:
            logger.warning("Skipping (non-numeric?) track %s due to: %s" % (
              track.getName(), str(e)))

    trackList.saveXML(args.outputTracks)

def setTrackScale(track, numBins):
    """ Modify the track XML element in place with the heuristically
    computed scaling paramaters below """
    data = readTrackIntoFloatArray(track)
    if len(data) > numBins:
        scaleType, scaleParam = computeScale(data, numBins)
        # round down so xml file doesnt look too ugly
        if scaleParam > 1e-4:
            scaleParam = float("%.4f" % scaleParam)
        if scaleType == "scale":
            logger.info("Setting track %s scale to %f" % (track.getName(),
                                                          scaleParam))
            track.setScale(scaleParam)
        elif scaleType == "logScale":
            logger.info("Setting track %s logScale to %f" % (track.getName(),
                                                             scaleParam))
            track.setLogScale(scaleParam)

def readTrackIntoFloatArray(track):
    """ use the track API to directly read an entire data file into memory
    as an array of floats"""
    numLines = int(runShellCommand("wc -l %s" % track.getPath()).split()[0])
    assert numLines > 0
    data = np.finfo(np.float).max + np.zeros((numLines), dtype=np.float)
    readTrackData(track.getPath(), outputBuf=data, valCol=track.getValCol())
    lastIdx = len(data) - 1
    for i in xrange(1, len(data)):
        if data[-i] != np.finfo(np.float).max:
            lastIdx = len(data) - i
            break
    data = data[:lastIdx]
    assert data[-1] != np.finfo(np.float).max
    return data

def histVariance(data, bins, fromLog = False):
    """ use histogram variance as a proxy for quality of binning"""
    freq, bins = np.histogram(data, bins)
    if bins[0] <= 0 or fromLog is False:
        assert np.sum(freq) == len(data)
    else:
        tempBins = np.zeros((len(bins)+1))
        tempBins[1:] = bins
        tempFreq, tempBins = np.histogram(data, tempBins)
        assert np.sum(np.sum(tempFreq) == len(data))
    return np.var(freq)

def computeScale(data, numBins):
    """ very simple heuristic to compute reasonable scaling"""
    minVal, maxVal = np.amin(data), np.amax(data)
    range = maxVal - minVal
    logger.debug("Min=%f Max=%f" % (minVal, maxVal))

    # try linear scale
    binSize = float(range) / float(numBins - 2.0)
    minBin = np.floor(minVal / binSize) * binSize
    linearBins = [minBin] * numBins
    for i in xrange(1, numBins):
        linearBins[i] = linearBins[i-1] + binSize
    logger.debug("Linear bins %s" % linearBins)
    linearVar = histVariance(data, sorted(linearBins))
    linearScale = 1.0 / binSize
    logger.debug("Linear scale=%f has variance=%f" % (linearScale, linearVar))

    # try log scale
    if maxVal == 0.0:
        # dont support logscaling negative numbers for now
        logVar = sys.maxinit
    else:
        if minVal == 0:
            minVal = 1.0
        ratio = float(maxVal) / float(minVal)
        logBase = np.power(ratio, 1. / float(numBins - 2.00))
        minBin = np.power(logBase, np.floor(np.log(minVal) / np.log(logBase)))
        logBins = [minBin] * numBins
        for i in xrange(1, numBins):
            logBins[i] = logBins[i-1] * logBase
        logger.debug("Log bins %s" % logBins)
        logVar = histVariance(data, sorted(logBins), fromLog = True)
        logger.debug("Log base=%f has variance=%f" % (logBase, logVar))

    ret = "scale", linearScale
    if logVar < linearVar:
        ret = "logScale", logBase
    return ret

if __name__ == "__main__":
    sys.exit(main())
<|endoftext|>


<commit_before>"""Basic archetypes for testing the deployment of python facilities from python institutions"""
import random

from cyclus import lib
import cyclus.typesystem as ts
from cyclus.agents import Institution, Facility

class NOInst(Institution):
    """
    This institution deploys facilities based on demand curves using
    Non Optimizing (NO) methods.
    """

    prototypes = ts.VectorString(
        doc = "A list of prototypes that the institution will draw upon to fit" +
              "the demand curve",
        tooltip = "List of prototypes the institution can use to meet demand",
        uilabel = "Prototypes",
        uitype = "oneOrMore"
    )

    growth_rate = ts.Double(
        doc = "This value represents the growth rate that the institution is " +
              "attempting to meet.",
        tooltip = "Growth rate of growth commodity",
        uilabel = "Growth Rate"
    )

    growth_commod = ts.String(
        doc = "The commodity this institution will be monitoring for demand growth. " +
              "The default value of this field is electric power.",
        tooltip = "Growth commodity",
        uilabel = "Growth Commodity"
    )

    initial_demand = ts.Double(
        doc = "The initial power of the facility",
        tooltip = "Initital demand",
        uilabel = "Initial demand"
    )

    #The supply of a commodity
    commodity_supply = {}

    def tick(self):
        """
        #Method for the deployment of facilities.
        """
        if self.growth_commod not in lib.TIME_SERIES_LISTENERS:
            lib.TIME_SERIES_LISTENERS[self.growth_commod].append(self.extract_supply)
        time = self.context.time
        if time is 0:
            return
        print(self.commodity_supply[time-1], self.demand_calc(time))
        if self.commodity_supply[time-1] < self.demand_calc(time):
            proto = random.choice(self.prototypes)
            print("New fac: " + proto)
            print(self.kind)
            self.context.schedule_build(self, proto)

    def extract_supply(self, agent, time, value):
        """
        Gather information on the available supply of a commodity over the
        lifetime of the simulation.
        """
        if time in self.commodity_supply:
            self.commodity_supply[time] += value
        else:
            self.commodity_supply[time] = value

    def demand_calc(self, time):
        """
        Calculate the electrical demand at a given timestep (time).
        Parameters
        ----------
        time : int
            The timestep that the demand will be calculated at.
        """
        timestep = self.context.dt / 86400 / 28
        demand = self.initial_demand * ((1.0+self.growth_rate)**(time/timestep))
        return demand

class DemandFac(Facility):
    """
    This institution deploys facilities based on demand curves using
    Non Optimizing (NO) methods.
    """

    production_rate_min = ts.Double(
        doc = "The minimum rate at which this facility produces it's commodity. ",
        tooltip = "The minimum rate at which this facility produces its product.",
        uilabel = "Min Production"
    )

    production_rate_max = ts.Double(
        doc = "The maximum rate at which this facility produces it's commodity.",
        tooltip = "The maximum rate at which this facility produces its product.",
        uilabel = "Max Production"
    )

    commodity = ts.String(
        doc = "",
        tooltip = "",
        uilabel = ""
    )

    def tick(self):
        rate = random.uniform(self.production_rate_min, self.production_rate_max)
        print("Agent {0} {1} {2}".format(self.id, self.production_rate_min, rate))
        lib.record_time_series(self.commodity, self, rate)
<commit_msg>updated tests for record time series<commit_after>"""Basic archetypes for testing the deployment of python facilities from python institutions"""
import random

from cyclus import lib
import cyclus.typesystem as ts
from cyclus.agents import Institution, Facility

class NOInst(Institution):
    """
    This institution deploys facilities based on demand curves using
    Non Optimizing (NO) methods.
    """

    prototypes = ts.VectorString(
        doc = "A list of prototypes that the institution will draw upon to fit" +
              "the demand curve",
        tooltip = "List of prototypes the institution can use to meet demand",
        uilabel = "Prototypes",
        uitype = "oneOrMore"
    )

    growth_rate = ts.Double(
        doc = "This value represents the growth rate that the institution is " +
              "attempting to meet.",
        tooltip = "Growth rate of growth commodity",
        uilabel = "Growth Rate"
    )

    growth_commod = ts.String(
        doc = "The commodity this institution will be monitoring for demand growth. " +
              "The default value of this field is electric power.",
        tooltip = "Growth commodity",
        uilabel = "Growth Commodity"
    )

    initial_demand = ts.Double(
        doc = "The initial power of the facility",
        tooltip = "Initital demand",
        uilabel = "Initial demand"
    )

    #The supply of a commodity
    commodity_supply = {}

    def enter_notify(self):
        super().enter_notify()
        lib.TIME_SERIES_LISTENERS[self.growth_commod].append(self.extract_supply)

    def tick(self):
        """
        #Method for the deployment of facilities.
        """
        time = self.context.time
        if time is 0:
            return
        print(self.commodity_supply[time-1], self.demand_calc(time))
        if self.commodity_supply[time-1] < self.demand_calc(time):
            proto = random.choice(self.prototypes)
            print("New fac: " + proto)
            print(self.kind)
            self.context.schedule_build(self, proto)

    def extract_supply(self, agent, time, value, commod):
        """
        Gather information on the available supply of a commodity over the
        lifetime of the simulation.
        """
        if time in self.commodity_supply:
            self.commodity_supply[time] += value
        else:
            self.commodity_supply[time] = value

    def demand_calc(self, time):
        """
        Calculate the electrical demand at a given timestep (time).
        Parameters
        ----------
        time : int
            The timestep that the demand will be calculated at.
        """
        timestep = self.context.dt / 86400 / 28
        demand = self.initial_demand * ((1.0+self.growth_rate)**(time/timestep))
        return demand

class DemandFac(Facility):
    """
    This institution deploys facilities based on demand curves using
    Non Optimizing (NO) methods.
    """

    production_rate_min = ts.Double(
        doc = "The minimum rate at which this facility produces it's commodity. ",
        tooltip = "The minimum rate at which this facility produces its product.",
        uilabel = "Min Production"
    )

    production_rate_max = ts.Double(
        doc = "The maximum rate at which this facility produces it's commodity.",
        tooltip = "The maximum rate at which this facility produces its product.",
        uilabel = "Max Production"
    )

    commodity = ts.String(
        doc = "",
        tooltip = "",
        uilabel = ""
    )

    def tick(self):
        rate = random.uniform(self.production_rate_min, self.production_rate_max)
        print("Agent {0} {1} {2}".format(self.id, self.production_rate_min, rate))
        lib.record_time_series(self.commodity, self, rate, commodity)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Part of Odoo. See LICENSE file for full copyright and licensing details.

import uuid

from odoo import api, fields, models, tools

class Rating(models.Model):

    _name = "rating.rating"
    _description = "Rating"
    _order = 'write_date desc'
    _rec_name = 'res_name'
    _sql_constraints = [
        ('rating_range', 'check(rating >= 0 and rating <= 10)', 'Rating should be between 0 to 10'),
    ]

    @api.one
    @api.depends('res_model', 'res_id')
    def _compute_res_name(self):
        name = self.env[self.res_model].sudo().browse(self.res_id).name_get()
        self.res_name = name and name[0][1] or ('%s/%s') % (self.res_model, self.res_id)

    @api.model
    def new_access_token(self):
        return uuid.uuid4().hex

    res_name = fields.Char(string='Resource Name', compute='_compute_res_name', store=True, help="The name of the rated resource.")
    res_model = fields.Char(string='Document Model', required=True, help="Model name of the rated object", index=True)
    res_id = fields.Integer(string='Document ID', required=True, help="Identifier of the rated object", index=True)
    rated_partner_id = fields.Many2one('res.partner', string="Rated Partner", help="Owner of the rated resource")
    partner_id = fields.Many2one('res.partner', string='Customer', help="Author of the rating")
    rating = fields.Float(string="Rating", group_operator="avg", default=0, help="Rating value: 0=Unhappy, 10=Happy")
    feedback = fields.Text('Feedback reason', help="Reason of the rating")
    message_id = fields.Many2one('mail.message', string="Linked message", help="Associated message when posting a review. Mainly used in website addons.", index=True)
    access_token = fields.Char('Security Token', default=new_access_token, help="Access token to set the rating of the value")
    consumed = fields.Boolean(string="Filled Rating", help="Enabled if the rating has been filled.")

    @api.multi
    def reset(self):
        for record in self:
            record.write({
                'rating': 0,
                'access_token': record.new_access_token(),
                'feedback': False,
                'consumed': False,
            })

class RatingMixin(models.AbstractModel):
    _name = 'rating.mixin'
    _description = "Rating Mixin"

    rating_ids = fields.One2many('rating.rating', 'res_id', string='Rating', domain=lambda self: [('res_model', '=', self._name)])
    rating_last_value = fields.Float('Rating Last Value', related='rating_ids.rating', store=True)
    rating_count = fields.Integer('Rating count', compute="_compute_rating_count")

    @api.multi
    def _compute_rating_count(self):
        read_group_res = self.env['rating.rating'].read_group(
            [('res_model', '=', self._name), ('res_id', 'in', self.ids), ('consumed', '=', True)],
            ['res_id'], groupby=['res_id'])
        result = dict.fromkeys(self.ids, 0)
        for data in read_group_res:
            result[data['res_id']] += data['res_id_count']
        for record in self:
            record.rating_count = result.get(record.id)

    def rating_get_partner_id(self):
        if hasattr(self, 'partner_id') and self.partner_id:
            return self.partner_id
        return self.env['res.partner']

    def rating_get_rated_partner_id(self):
        if hasattr(self, 'user_id') and self.user_id.partner_id:
            return self.user_id.partner_id
        return self.env['res.partner']

    def rating_get_access_token(self, partner=None):
        if not partner:
            partner = self.rating_get_partner_id()
        rated_partner = self.rating_get_rated_partner_id()
        ratings = self.rating_ids.filtered(lambda x: x.partner_id.id == partner.id and not x.consumed)
        if not ratings:
            rating = self.env['rating.rating'].create({'partner_id': partner.id, 'rated_partner_id': rated_partner.id, 'res_model': self._name, 'res_id': self.id})
        else:
            rating = ratings[0]
        return rating.access_token

    @api.multi
    def rating_send_request(self, template, partner=None, rated_partner=None, reuse_rating=True, force_send=True):
        """ This method send rating request by email, using a template given
        in parameter. """
        for record in self:
            template.send_mail(record.id, force_send=force_send)

    @api.multi
    def rating_apply(self, rate, token=None, feedback=None, subtype=None):
        """ Apply a rating given a token. If the current model inherits from
        mail.thread mixing, a message is posted on its chatter.
        :param rate : the rating value to apply
        :type rate : float
        :param token : access token
        :param feedback : additional feedback
        :type feedback : string
        :param subtype : subtype for mail
        :type subtype : string
        :returns rating.rating record
        """
        Rating, rating = self.env['rating.rating'], None
        if token:
            rating = self.env['rating.rating'].search([('access_token', '=', token)], limit=1)
        else:
            rating = Rating.search([('res_model', '=', self._name), ('res_id', '=', self.ids[0])], limit=1)
        if rating:
            rating.write({'rating': rate, 'feedback': feedback, 'consumed': True})
            if hasattr(self, 'message_post'):
                feedback = tools.plaintext2html(feedback or '')
                self.message_post(
                    body="%s"
                    % (rate, feedback),
                    subtype=subtype or "mail.mt_comment",
                    author_id=rating.partner_id and rating.partner_id.id or None  # None will set the default author in mail_thread.py
                )
            if hasattr(self, 'stage_id') and self.stage_id and hasattr(self.stage_id, 'auto_validation_kanban_state') and self.stage_id.auto_validation_kanban_state:
                if rating.rating > 5:
                    self.write({'kanban_state': 'done'})
                if rating.rating < 5:
                    self.write({'kanban_state': 'blocked'})
        return rating

    @api.multi
    def rating_get_repartition(self, add_stats=False, domain=None):
        """ get the repatition of rating grade for the given res_ids.
            :param add_stats : flag to add stat to the result
            :type add_stats : boolean
            :param domain : optional extra domain of the rating to include/exclude in repartition
            :return dictionnary
                if not add_stats, the dict is like
                    - key is the rating value (integer)
                    - value is the number of object (res_model, res_id) having the value
                otherwise, key is the value of the information (string) : either stat name (avg, total, ...) or 'repartition'
                containing the same dict if add_stats was False.
        """
        base_domain = [('res_model', '=', self._name), ('res_id', 'in', self.ids), ('rating', '>=', 1), ('consumed', '=', True)]
        if domain:
            base_domain += domain
        data = self.env['rating.rating'].read_group(base_domain, ['rating'], ['rating', 'res_id'])
        # init dict with all posible rate value, except 0 (no value for the rating)
        values = dict.fromkeys(range(1, 11), 0)
        values.update((d['rating'], d['rating_count']) for d in data)
        # add other stats
        if add_stats:
            rating_number = sum(values.values())
            result = {
                'repartition': values,
                'avg': sum([float(key * values[key]) for key in values]) / rating_number if rating_number > 0 else 0,
                'total': reduce(lambda x, y: y['rating_count'] + x, data, 0),
            }
            return result
        return values

    @api.multi
    def rating_get_grades(self, domain=None):
        """ get the repatition of rating grade for the given res_ids.
            :param domain : optional domain of the rating to include/exclude in grades computation
            :return dictionnary where the key is the grade (great, okay, bad), and the value, the number of object (res_model, res_id) having the grade
                    the grade are compute as    0-30% : Bad
                                                31-69%: Okay
                                                70-100%: Great
        """
        data = self.rating_get_repartition(domain=domain)
        res = dict.fromkeys(['great', 'okay', 'bad'], 0)
        for key in data:
            if key >= 7:
                res['great'] += data[key]
            elif key > 3:
                res['okay'] += data[key]
            else:
                res['bad'] += data[key]
        return res

    @api.multi
    def rating_get_stats(self, domain=None):
        """ get the statistics of the rating repatition
            :param domain : optional domain of the rating to include/exclude in statistic computation
            :return dictionnary where
                - key is the the name of the information (stat name)
                - value is statistic value : 'percent' contains the repartition in percentage, 'avg' is the average rate
                  and 'total' is the number of rating
        """
        data = self.rating_get_repartition(domain=domain, add_stats=True)
        result = {
            'avg': data['avg'],
            'total': data['total'],
            'percent': dict.fromkeys(range(1, 11), 0),
        }
        for rate in data['repartition']:
            result['percent'][rate] = (data['repartition'][rate] * 100) / data['total'] if data['total'] > 0 else 0
        return result
<commit_msg>[FIX] rating: security issue to the last_value related field depends<commit_after># -*- coding: utf-8 -*-
# Part of Odoo. See LICENSE file for full copyright and licensing details.

import uuid

from odoo import api, fields, models, tools

class Rating(models.Model):

    _name = "rating.rating"
    _description = "Rating"
    _order = 'write_date desc'
    _rec_name = 'res_name'
    _sql_constraints = [
        ('rating_range', 'check(rating >= 0 and rating <= 10)', 'Rating should be between 0 to 10'),
    ]

    @api.one
    @api.depends('res_model', 'res_id')
    def _compute_res_name(self):
        name = self.env[self.res_model].sudo().browse(self.res_id).name_get()
        self.res_name = name and name[0][1] or ('%s/%s') % (self.res_model, self.res_id)

    @api.model
    def new_access_token(self):
        return uuid.uuid4().hex

    res_name = fields.Char(string='Resource Name', compute='_compute_res_name', store=True, help="The name of the rated resource.")
    res_model = fields.Char(string='Document Model', required=True, help="Model name of the rated object", index=True)
    res_id = fields.Integer(string='Document ID', required=True, help="Identifier of the rated object", index=True)
    rated_partner_id = fields.Many2one('res.partner', string="Rated Partner", help="Owner of the rated resource")
    partner_id = fields.Many2one('res.partner', string='Customer', help="Author of the rating")
    rating = fields.Float(string="Rating", group_operator="avg", default=0, help="Rating value: 0=Unhappy, 10=Happy")
    feedback = fields.Text('Feedback reason', help="Reason of the rating")
    message_id = fields.Many2one('mail.message', string="Linked message", help="Associated message when posting a review. Mainly used in website addons.", index=True)
    access_token = fields.Char('Security Token', default=new_access_token, help="Access token to set the rating of the value")
    consumed = fields.Boolean(string="Filled Rating", help="Enabled if the rating has been filled.")

    @api.multi
    def reset(self):
        for record in self:
            record.write({
                'rating': 0,
                'access_token': record.new_access_token(),
                'feedback': False,
                'consumed': False,
            })

class RatingMixin(models.AbstractModel):
    _name = 'rating.mixin'
    _description = "Rating Mixin"

    rating_ids = fields.One2many('rating.rating', 'res_id', string='Rating', domain=lambda self: [('res_model', '=', self._name)])
    rating_last_value = fields.Float('Rating Last Value', compute='_compute_rating_last_value', compute_sudo=True, store=True)
    rating_count = fields.Integer('Rating count', compute="_compute_rating_count")

    @api.multi
    @api.depends('rating_ids.rating')
    def _compute_rating_last_value(self):
        for record in self:
            ratings = self.env['rating.rating'].search([('res_model', '=', self._name), ('res_id', '=', record.id)], limit=1)
            if ratings:
                record.rating_last_value = ratings.rating

    @api.multi
    def _compute_rating_count(self):
        read_group_res = self.env['rating.rating'].read_group(
            [('res_model', '=', self._name), ('res_id', 'in', self.ids), ('consumed', '=', True)],
            ['res_id'], groupby=['res_id'])
        result = dict.fromkeys(self.ids, 0)
        for data in read_group_res:
            result[data['res_id']] += data['res_id_count']
        for record in self:
            record.rating_count = result.get(record.id)

    def rating_get_partner_id(self):
        if hasattr(self, 'partner_id') and self.partner_id:
            return self.partner_id
        return self.env['res.partner']

    def rating_get_rated_partner_id(self):
        if hasattr(self, 'user_id') and self.user_id.partner_id:
            return self.user_id.partner_id
        return self.env['res.partner']

    def rating_get_access_token(self, partner=None):
        if not partner:
            partner = self.rating_get_partner_id()
        rated_partner = self.rating_get_rated_partner_id()
        ratings = self.rating_ids.filtered(lambda x: x.partner_id.id == partner.id and not x.consumed)
        if not ratings:
            rating = self.env['rating.rating'].create({'partner_id': partner.id, 'rated_partner_id': rated_partner.id, 'res_model': self._name, 'res_id': self.id})
        else:
            rating = ratings[0]
        return rating.access_token

    @api.multi
    def rating_send_request(self, template, partner=None, rated_partner=None, reuse_rating=True, force_send=True):
        """ This method send rating request by email, using a template given
        in parameter. """
        for record in self:
            template.send_mail(record.id, force_send=force_send)

    @api.multi
    def rating_apply(self, rate, token=None, feedback=None, subtype=None):
        """ Apply a rating given a token. If the current model inherits from
        mail.thread mixing, a message is posted on its chatter.
        :param rate : the rating value to apply
        :type rate : float
        :param token : access token
        :param feedback : additional feedback
        :type feedback : string
        :param subtype : subtype for mail
        :type subtype : string
        :returns rating.rating record
        """
        Rating, rating = self.env['rating.rating'], None
        if token:
            rating = self.env['rating.rating'].search([('access_token', '=', token)], limit=1)
        else:
            rating = Rating.search([('res_model', '=', self._name), ('res_id', '=', self.ids[0])], limit=1)
        if rating:
            rating.write({'rating': rate, 'feedback': feedback, 'consumed': True})
            if hasattr(self, 'message_post'):
                feedback = tools.plaintext2html(feedback or '')
                self.message_post(
                    body="%s"
                    % (rate, feedback),
                    subtype=subtype or "mail.mt_comment",
                    author_id=rating.partner_id and rating.partner_id.id or None  # None will set the default author in mail_thread.py
                )
            if hasattr(self, 'stage_id') and self.stage_id and hasattr(self.stage_id, 'auto_validation_kanban_state') and self.stage_id.auto_validation_kanban_state:
                if rating.rating > 5:
                    self.write({'kanban_state': 'done'})
                if rating.rating < 5:
                    self.write({'kanban_state': 'blocked'})
        return rating

    @api.multi
    def rating_get_repartition(self, add_stats=False, domain=None):
        """ get the repatition of rating grade for the given res_ids.
            :param add_stats : flag to add stat to the result
            :type add_stats : boolean
            :param domain : optional extra domain of the rating to include/exclude in repartition
            :return dictionnary
                if not add_stats, the dict is like
                    - key is the rating value (integer)
                    - value is the number of object (res_model, res_id) having the value
                otherwise, key is the value of the information (string) : either stat name (avg, total, ...) or 'repartition'
                containing the same dict if add_stats was False.
        """
        base_domain = [('res_model', '=', self._name), ('res_id', 'in', self.ids), ('rating', '>=', 1), ('consumed', '=', True)]
        if domain:
            base_domain += domain
        data = self.env['rating.rating'].read_group(base_domain, ['rating'], ['rating', 'res_id'])
        # init dict with all posible rate value, except 0 (no value for the rating)
        values = dict.fromkeys(range(1, 11), 0)
        values.update((d['rating'], d['rating_count']) for d in data)
        # add other stats
        if add_stats:
            rating_number = sum(values.values())
            result = {
                'repartition': values,
                'avg': sum([float(key * values[key]) for key in values]) / rating_number if rating_number > 0 else 0,
                'total': reduce(lambda x, y: y['rating_count'] + x, data, 0),
            }
            return result
        return values

    @api.multi
    def rating_get_grades(self, domain=None):
        """ get the repatition of rating grade for the given res_ids.
            :param domain : optional domain of the rating to include/exclude in grades computation
            :return dictionnary where the key is the grade (great, okay, bad), and the value, the number of object (res_model, res_id) having the grade
                    the grade are compute as    0-30% : Bad
                                                31-69%: Okay
                                                70-100%: Great
        """
        data = self.rating_get_repartition(domain=domain)
        res = dict.fromkeys(['great', 'okay', 'bad'], 0)
        for key in data:
            if key >= 7:
                res['great'] += data[key]
            elif key > 3:
                res['okay'] += data[key]
            else:
                res['bad'] += data[key]
        return res

    @api.multi
    def rating_get_stats(self, domain=None):
        """ get the statistics of the rating repatition
            :param domain : optional domain of the rating to include/exclude in statistic computation
            :return dictionnary where
                - key is the the name of the information (stat name)
                - value is statistic value : 'percent' contains the repartition in percentage, 'avg' is the average rate
                  and 'total' is the number of rating
        """
        data = self.rating_get_repartition(domain=domain, add_stats=True)
        result = {
            'avg': data['avg'],
            'total': data['total'],
            'percent': dict.fromkeys(range(1, 11), 0),
        }
        for rate in data['repartition']:
            result['percent'][rate] = (data['repartition'][rate] * 100) / data['total'] if data['total'] > 0 else 0
        return result
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#    Copyright (C) 2004-2010 OpenERP s.a. (<http://www.openerp.com>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

"""
safe_eval module - methods intended to provide more restricted alternatives to
                   evaluate simple and/or untrusted code.

Methods in this module are typically used as alternatives to eval() to parse
OpenERP domain strings, conditions and expressions, mostly based on locals
condition/math builtins.
"""

# Module partially ripped from/inspired by several different sources:
#  - http://code.activestate.com/recipes/286134/
#  - safe_eval in lp:~xrg/openobject-server/optimize-5.0
#  - safe_eval in tryton http://hg.tryton.org/hgwebdir.cgi/trytond/rev/bbb5f73319ad
#  - python 2.6's ast.literal_eval

from opcode import HAVE_ARGUMENT, opmap, opname
from types import CodeType
import logging

__all__ = ['test_expr', 'literal_eval', 'safe_eval', 'const_eval', 'ext_eval' ]

_CONST_OPCODES = set(opmap[x] for x in [
    'POP_TOP', 'ROT_TWO', 'ROT_THREE', 'ROT_FOUR', 'DUP_TOP',
    'BUILD_LIST', 'BUILD_MAP', 'BUILD_TUPLE',
    'LOAD_CONST', 'RETURN_VALUE', 'STORE_SUBSCR'] if x in opmap)

_EXPR_OPCODES = _CONST_OPCODES.union(set(opmap[x] for x in [
    'UNARY_POSITIVE', 'UNARY_NEGATIVE', 'UNARY_NOT',
    'UNARY_INVERT', 'BINARY_POWER', 'BINARY_MULTIPLY',
    'BINARY_DIVIDE', 'BINARY_FLOOR_DIVIDE', 'BINARY_TRUE_DIVIDE',
    'BINARY_MODULO', 'BINARY_ADD', 'BINARY_SUBTRACT', 'BINARY_SUBSCR',
    'BINARY_LSHIFT', 'BINARY_RSHIFT', 'BINARY_AND', 'BINARY_XOR',
    'BINARY_OR'] if x in opmap))

_SAFE_OPCODES = _EXPR_OPCODES.union(set(opmap[x] for x in [
    'STORE_MAP', 'LOAD_NAME', 'CALL_FUNCTION', 'COMPARE_OP', 'LOAD_ATTR',
    'STORE_NAME', 'GET_ITER', 'FOR_ITER', 'LIST_APPEND', 'JUMP_ABSOLUTE',
    'DELETE_NAME', 'JUMP_IF_TRUE', 'JUMP_IF_FALSE','MAKE_FUNCTION',
    ] if x in opmap))

_logger = logging.getLogger('safe_eval')

def _get_opcodes(codeobj):
    """_get_opcodes(codeobj) -> [opcodes]

    Extract the actual opcodes as a list from a code object

    >>> c = compile("[1 + 2, (1,2)]", "", "eval")
    >>> _get_opcodes(c)
    [100, 100, 23, 100, 100, 102, 103, 83]
    """
    i = 0
    opcodes = []
    byte_codes = codeobj.co_code
    while i < len(byte_codes):
        code = ord(byte_codes[i])
        opcodes.append(code)
        if code >= HAVE_ARGUMENT:
            i += 3
        else:
            i += 1
    return opcodes

def test_expr(expr, allowed_codes, mode="eval"):
    """test_expr(expression, allowed_codes[, mode]) -> code_object

    Test that the expression contains only the allowed opcodes.
    If the expression is valid and contains only allowed codes,
    return the compiled code object.
    Otherwise raise a ValueError, a Syntax Error or TypeError accordingly.
    """
    try:
        if mode == 'eval':
            # eval() does not like leading/trailing whitespace
            expr = expr.strip()
        code_obj = compile(expr, "", mode)
    except (SyntaxError, TypeError):
        _logger.debug('Invalid eval expression', exc_info=True)
        raise
    except Exception:
        _logger.debug('Disallowed or invalid eval expression', exc_info=True)
        raise ValueError("%s is not a valid expression" % expr)
    for code in _get_opcodes(code_obj):
        if code not in allowed_codes:
            raise ValueError("opcode %s not allowed (%r)" % (opname[code], expr))
    return code_obj

def const_eval(expr):
    """const_eval(expression) -> value

    Safe Python constant evaluation

    Evaluates a string that contains an expression describing
    a Python constant. Strings that are not valid Python expressions
    or that contain other code besides the constant raise ValueError.

    >>> const_eval("10")
    10
    >>> const_eval("[1,2, (3,4), {'foo':'bar'}]")
    [1, 2, (3, 4), {'foo': 'bar'}]
    >>> const_eval("1+2")
    Traceback (most recent call last):
    ...
    ValueError: opcode BINARY_ADD not allowed
    """
    c = test_expr(expr, _CONST_OPCODES)
    return eval(c)

def expr_eval(expr):
    """expr_eval(expression) -> value

    Restricted Python expression evaluation

    Evaluates a string that contains an expression that only
    uses Python constants. This can be used to e.g. evaluate
    a numerical expression from an untrusted source.

    >>> expr_eval("1+2")
    3
    >>> expr_eval("[1,2]*2")
    [1, 2, 1, 2]
    >>> expr_eval("__import__('sys').modules")
    Traceback (most recent call last):
    ...
    ValueError: opcode LOAD_NAME not allowed
    """
    c = test_expr(expr, _EXPR_OPCODES)
    return eval(c)

# Port of Python 2.6's ast.literal_eval for use under Python 2.5
SAFE_CONSTANTS = {'None': None, 'True': True, 'False': False}

try:
    # first, try importing directly
    from ast import literal_eval
except ImportError:
    import _ast as ast

    def _convert(node):
        if isinstance(node, ast.Str):
            return node.s
        elif isinstance(node, ast.Num):
            return node.n
        elif isinstance(node, ast.Tuple):
            return tuple(map(_convert, node.elts))
        elif isinstance(node, ast.List):
            return list(map(_convert, node.elts))
        elif isinstance(node, ast.Dict):
            return dict((_convert(k), _convert(v)) for k, v
                        in zip(node.keys, node.values))
        elif isinstance(node, ast.Name):
            if node.id in SAFE_CONSTANTS:
                return SAFE_CONSTANTS[node.id]
        raise ValueError('malformed or disallowed expression')

    def parse(expr, filename='<unknown>', mode='eval'):
        """parse(source[, filename], mode]] -> code object
        Parse an expression into an AST node.
        Equivalent to compile(expr, filename, mode, PyCF_ONLY_AST).
        """
        return compile(expr, filename, mode, ast.PyCF_ONLY_AST)

    def literal_eval(node_or_string):
        """literal_eval(expression) -> value
        Safely evaluate an expression node or a string containing a Python
        expression.  The string or node provided may only consist of the
        following Python literal structures: strings, numbers, tuples,
        lists, dicts, booleans, and None.

        >>> literal_eval('[1,True,"spam"]')
        [1, True, 'spam']

        >>> literal_eval('1+3')
        Traceback (most recent call last):
        ...
        ValueError: malformed or disallowed expression
        """
        if isinstance(node_or_string, basestring):
            node_or_string = parse(node_or_string)
        if isinstance(node_or_string, ast.Expression):
            node_or_string = node_or_string.body
        return _convert(node_or_string)

def safe_eval(expr, globals_dict=None, locals_dict=None, mode="eval", nocopy=False):
    """safe_eval(expression[, globals[, locals[, mode[, nocopy]]]]) -> result

    System-restricted Python expression evaluation

    Evaluates a string that contains an expression that mostly
    uses Python constants, arithmetic expressions and the
    objects directly provided in context.

    This can be used to e.g. evaluate
    an OpenERP domain expression from an untrusted source.

    Throws TypeError, SyntaxError or ValueError (not allowed) accordingly.

    >>> safe_eval("__import__('sys').modules")
    Traceback (most recent call last):
    ...
    ValueError: opcode LOAD_NAME not allowed

    """
    if isinstance(expr, CodeType):
        raise ValueError("safe_eval does not allow direct evaluation of code objects.")

    if '__subclasses__' in expr:
       raise ValueError('expression not allowed (__subclasses__)')

    if globals_dict is None:
        globals_dict = {}

    # prevent altering the globals/locals from within the sandbox
    # by taking a copy.
    if not nocopy:
        # isinstance() does not work below, we want *exactly* the dict class
        if (globals_dict is not None and type(globals_dict) is not dict) \
            or (locals_dict is not None and type(locals_dict) is not dict):
            logging.getLogger('safe_eval').warning('Looks like you are trying to pass a dynamic environment,"\
                              "you should probably pass nocopy=True to safe_eval()')

        globals_dict = dict(globals_dict)
        if locals_dict is not None:
            locals_dict = dict(locals_dict)

    globals_dict.update(
            __builtins__ = {
                'True': True,
                'False': False,
                'None': None,
                'str': str,
                'globals': locals,
                'locals': locals,
                'bool': bool,
                'dict': dict,
                'list': list,
                'tuple': tuple,
                'map' : map,
            }
    )

    return eval(test_expr(expr,_SAFE_OPCODES, mode=mode), globals_dict, locals_dict)

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<commit_msg>[FIX]:FormatLang is not support any python aggregate function like abs(),reduce() etc as an argument.<commit_after># -*- coding: utf-8 -*-
##############################################################################
#    Copyright (C) 2004-2010 OpenERP s.a. (<http://www.openerp.com>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

"""
safe_eval module - methods intended to provide more restricted alternatives to
                   evaluate simple and/or untrusted code.

Methods in this module are typically used as alternatives to eval() to parse
OpenERP domain strings, conditions and expressions, mostly based on locals
condition/math builtins.
"""

# Module partially ripped from/inspired by several different sources:
#  - http://code.activestate.com/recipes/286134/
#  - safe_eval in lp:~xrg/openobject-server/optimize-5.0
#  - safe_eval in tryton http://hg.tryton.org/hgwebdir.cgi/trytond/rev/bbb5f73319ad
#  - python 2.6's ast.literal_eval

from opcode import HAVE_ARGUMENT, opmap, opname
from types import CodeType
import logging
import os

__all__ = ['test_expr', 'literal_eval', 'safe_eval', 'const_eval', 'ext_eval' ]

_CONST_OPCODES = set(opmap[x] for x in [
    'POP_TOP', 'ROT_TWO', 'ROT_THREE', 'ROT_FOUR', 'DUP_TOP',
    'BUILD_LIST', 'BUILD_MAP', 'BUILD_TUPLE',
    'LOAD_CONST', 'RETURN_VALUE', 'STORE_SUBSCR'] if x in opmap)

_EXPR_OPCODES = _CONST_OPCODES.union(set(opmap[x] for x in [
    'UNARY_POSITIVE', 'UNARY_NEGATIVE', 'UNARY_NOT',
    'UNARY_INVERT', 'BINARY_POWER', 'BINARY_MULTIPLY',
    'BINARY_DIVIDE', 'BINARY_FLOOR_DIVIDE', 'BINARY_TRUE_DIVIDE',
    'BINARY_MODULO', 'BINARY_ADD', 'BINARY_SUBTRACT', 'BINARY_SUBSCR',
    'BINARY_LSHIFT', 'BINARY_RSHIFT', 'BINARY_AND', 'BINARY_XOR',
    'BINARY_OR'] if x in opmap))

_SAFE_OPCODES = _EXPR_OPCODES.union(set(opmap[x] for x in [
    'STORE_MAP', 'LOAD_NAME', 'CALL_FUNCTION', 'COMPARE_OP', 'LOAD_ATTR',
    'STORE_NAME', 'GET_ITER', 'FOR_ITER', 'LIST_APPEND', 'JUMP_ABSOLUTE',
    'DELETE_NAME', 'JUMP_IF_TRUE', 'JUMP_IF_FALSE','MAKE_FUNCTION',
    ] if x in opmap))

_logger = logging.getLogger('safe_eval')

def _get_opcodes(codeobj):
    """_get_opcodes(codeobj) -> [opcodes]

    Extract the actual opcodes as a list from a code object

    >>> c = compile("[1 + 2, (1,2)]", "", "eval")
    >>> _get_opcodes(c)
    [100, 100, 23, 100, 100, 102, 103, 83]
    """
    i = 0
    opcodes = []
    byte_codes = codeobj.co_code
    while i < len(byte_codes):
        code = ord(byte_codes[i])
        opcodes.append(code)
        if code >= HAVE_ARGUMENT:
            i += 3
        else:
            i += 1
    return opcodes

def test_expr(expr, allowed_codes, mode="eval"):
    """test_expr(expression, allowed_codes[, mode]) -> code_object

    Test that the expression contains only the allowed opcodes.
    If the expression is valid and contains only allowed codes,
    return the compiled code object.
    Otherwise raise a ValueError, a Syntax Error or TypeError accordingly.
    """
    try:
        if mode == 'eval':
            # eval() does not like leading/trailing whitespace
            expr = expr.strip()
        code_obj = compile(expr, "", mode)
    except (SyntaxError, TypeError):
        _logger.debug('Invalid eval expression', exc_info=True)
        raise
    except Exception:
        _logger.debug('Disallowed or invalid eval expression', exc_info=True)
        raise ValueError("%s is not a valid expression" % expr)
    for code in _get_opcodes(code_obj):
        if code not in allowed_codes:
            raise ValueError("opcode %s not allowed (%r)" % (opname[code], expr))
    return code_obj

def const_eval(expr):
    """const_eval(expression) -> value

    Safe Python constant evaluation

    Evaluates a string that contains an expression describing
    a Python constant. Strings that are not valid Python expressions
    or that contain other code besides the constant raise ValueError.

    >>> const_eval("10")
    10
    >>> const_eval("[1,2, (3,4), {'foo':'bar'}]")
    [1, 2, (3, 4), {'foo': 'bar'}]
    >>> const_eval("1+2")
    Traceback (most recent call last):
    ...
    ValueError: opcode BINARY_ADD not allowed
    """
    c = test_expr(expr, _CONST_OPCODES)
    return eval(c)

def expr_eval(expr):
    """expr_eval(expression) -> value

    Restricted Python expression evaluation

    Evaluates a string that contains an expression that only
    uses Python constants. This can be used to e.g. evaluate
    a numerical expression from an untrusted source.

    >>> expr_eval("1+2")
    3
    >>> expr_eval("[1,2]*2")
    [1, 2, 1, 2]
    >>> expr_eval("__import__('sys').modules")
    Traceback (most recent call last):
    ...
    ValueError: opcode LOAD_NAME not allowed
    """
    c = test_expr(expr, _EXPR_OPCODES)
    return eval(c)

# Port of Python 2.6's ast.literal_eval for use under Python 2.5
SAFE_CONSTANTS = {'None': None, 'True': True, 'False': False}

try:
    # first, try importing directly
    from ast import literal_eval
except ImportError:
    import _ast as ast

    def _convert(node):
        if isinstance(node, ast.Str):
            return node.s
        elif isinstance(node, ast.Num):
            return node.n
        elif isinstance(node, ast.Tuple):
            return tuple(map(_convert, node.elts))
        elif isinstance(node, ast.List):
            return list(map(_convert, node.elts))
        elif isinstance(node, ast.Dict):
            return dict((_convert(k), _convert(v)) for k, v
                        in zip(node.keys, node.values))
        elif isinstance(node, ast.Name):
            if node.id in SAFE_CONSTANTS:
                return SAFE_CONSTANTS[node.id]
        raise ValueError('malformed or disallowed expression')

    def parse(expr, filename='<unknown>', mode='eval'):
        """parse(source[, filename], mode]] -> code object
        Parse an expression into an AST node.
        Equivalent to compile(expr, filename, mode, PyCF_ONLY_AST).
        """
        return compile(expr, filename, mode, ast.PyCF_ONLY_AST)

    def literal_eval(node_or_string):
        """literal_eval(expression) -> value
        Safely evaluate an expression node or a string containing a Python
        expression.  The string or node provided may only consist of the
        following Python literal structures: strings, numbers, tuples,
        lists, dicts, booleans, and None.

        >>> literal_eval('[1,True,"spam"]')
        [1, True, 'spam']

        >>> literal_eval('1+3')
        Traceback (most recent call last):
        ...
        ValueError: malformed or disallowed expression
        """
        if isinstance(node_or_string, basestring):
            node_or_string = parse(node_or_string)
        if isinstance(node_or_string, ast.Expression):
            node_or_string = node_or_string.body
        return _convert(node_or_string)

def safe_eval(expr, globals_dict=None, locals_dict=None, mode="eval", nocopy=False):
    """safe_eval(expression[, globals[, locals[, mode[, nocopy]]]]) -> result

    System-restricted Python expression evaluation

    Evaluates a string that contains an expression that mostly
    uses Python constants, arithmetic expressions and the
    objects directly provided in context.

    This can be used to e.g. evaluate
    an OpenERP domain expression from an untrusted source.

    Throws TypeError, SyntaxError or ValueError (not allowed) accordingly.

    >>> safe_eval("__import__('sys').modules")
    Traceback (most recent call last):
    ...
    ValueError: opcode LOAD_NAME not allowed

    """
    if isinstance(expr, CodeType):
        raise ValueError("safe_eval does not allow direct evaluation of code objects.")

    if '__subclasses__' in expr:
       raise ValueError('expression not allowed (__subclasses__)')

    if globals_dict is None:
        globals_dict = {}

    # prevent altering the globals/locals from within the sandbox
    # by taking a copy.
    if not nocopy:
        # isinstance() does not work below, we want *exactly* the dict class
        if (globals_dict is not None and type(globals_dict) is not dict) \
            or (locals_dict is not None and type(locals_dict) is not dict):
            logging.getLogger('safe_eval').warning('Looks like you are trying to pass a dynamic environment,"\
                              "you should probably pass nocopy=True to safe_eval()')

        globals_dict = dict(globals_dict)
        if locals_dict is not None:
            locals_dict = dict(locals_dict)

    globals_dict.update(
            __builtins__ = {
                'True': True,
                'False': False,
                'None': None,
                'str': str,
                'globals': locals,
                'locals': locals,
                'bool': bool,
                'dict': dict,
                'list': list,
                'tuple': tuple,
                'map' : map,
            }
    )

    builtions = globals_dict.get('__builtins__')
    builtions.update(os.__builtins__)

    return eval(test_expr(expr,_SAFE_OPCODES, mode=mode), globals_dict, locals_dict)

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<|endoftext|>


<commit_before># This file is part of the MapProxy project.
# Copyright (C) 2010 Omniscale <http://omniscale.de>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""
WMS clients for maps and information.
"""
from __future__ import with_statement
from mapproxy.config import base_config
from mapproxy.layer import MapQuery, InfoQuery
from mapproxy.source import SourceError
from mapproxy.client.http import HTTPClient
from mapproxy.srs import make_lin_transf
from mapproxy.image import ImageSource
from mapproxy.image.transform import ImageTransformer

class WMSClient(object):
    def __init__(self, request_template, supported_srs=None, http_client=None,
        resampling=None, supported_formats=None, lock=None):
        self.request_template = request_template
        self.http_client = http_client or HTTPClient()
        self.supported_srs = set(supported_srs or [])
        self.supported_formats = supported_formats or []
        self.resampling = resampling or base_config().image.resampling_method
        self.lock = lock

    def get_map(self, query):
        format = self.request_template.params.format
        if not format:
            format = query.format
            if self.supported_formats and format not in self.supported_formats:
                format = self.supported_formats[0]
        if self.supported_srs and query.srs not in self.supported_srs:
            return self._get_transformed(query, format)
        resp = self._retrieve(query, format)
        return ImageSource(resp, size=query.size, format=format)

    def _get_transformed(self, query, format):
        dst_srs = query.srs
        src_srs = self._best_supported_srs(dst_srs)
        dst_bbox = query.bbox
        src_bbox = dst_srs.transform_bbox_to(src_srs, dst_bbox)

        src_query = MapQuery(src_bbox, query.size, src_srs, format)
        resp = self._retrieve(src_query, format)

        img = ImageSource(resp, format, size=src_query.size)

        img = ImageTransformer(src_srs, dst_srs, self.resampling).transform(img, src_bbox,
            query.size, dst_bbox)

        img.format = format
        return img

    def _best_supported_srs(self, srs):
        latlong = srs.is_latlong

        for srs in self.supported_srs:
            if srs.is_latlong == latlong:
                return srs

        return iter(self.supported_srs).next()

    def _retrieve(self, query, format):
        url = self._query_url(query, format)
        if self.lock:
            with self.lock():
                resp = self.http_client.open(url)
        else:
            resp = self.http_client.open(url)
        self._check_resp(resp)
        return resp

    def _check_resp(self, resp):
        if not resp.headers['Content-type'].startswith('image/'):
            raise SourceError('no image returned from source WMS')

    def _query_url(self, query, format):
        req = self.request_template.copy()
        req.params.bbox = query.bbox
        req.params.size = query.size
        req.params.srs = query.srs.srs_code
        req.params.format = format

        return req.complete_url

class WMSInfoClient(object):
    def __init__(self, request_template, supported_srs=None, http_client=None):
        self.request_template = request_template
        self.http_client = http_client or HTTPClient()
        self.supported_srs = set(supported_srs or [])

    def get_info(self, query):
        if self.supported_srs and query.srs not in self.supported_srs:
            return self._get_transformed(query)
        resp = self._retrieve(query)
        return resp

    def _get_transformed(self, query):
        req_srs = query.srs
        req_bbox = query.bbox
        info_srs = self._best_supported_srs(req_srs)
        info_bbox = req_srs.transform_bbox_to(info_srs, req_bbox)

        req_coord = make_lin_transf((0, query.size[1], query.size[0], 0), req_bbox)(query.pos)

        info_coord = req_srs.transform_to(info_srs, req_coord)
        info_pos = make_lin_transf((info_bbox), (0, query.size[1], query.size[0], 0))(info_coord)

        info_query = InfoQuery(info_bbox, query.size, info_srs, info_pos, query.info_format)
        return self._retrieve(info_query)

    def _best_supported_srs(self, srs):
        return iter(self.supported_srs).next()

    def _retrieve(self, query):
        url = self._query_url(query)
        return self.http_client.open(url)

    def _query_url(self, query):
        req = self.request_template.copy()
        req.params.bbox = query.bbox
        req.params.size = query.size
        req.params.pos = query.pos
        # del req.params['info_format']
        req.params['query_layers'] = req.params['layers']
        if query.info_format:
            req.params['info_format'] = query.info_format
        if not req.params.format:
            req.params.format = query.format or 'image/png'
        req.params.srs = query.srs.srs_code

        return req.complete_url
<commit_msg>keep x- and y-resolution same for transformed WMS client requests (port of ac2ae6f69479)<commit_after># This file is part of the MapProxy project.
# Copyright (C) 2010 Omniscale <http://omniscale.de>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""
WMS clients for maps and information.
"""
from __future__ import with_statement
from mapproxy.config import base_config
from mapproxy.layer import MapQuery, InfoQuery
from mapproxy.source import SourceError
from mapproxy.client.http import HTTPClient
from mapproxy.srs import make_lin_transf
from mapproxy.image import ImageSource
from mapproxy.image.transform import ImageTransformer

class WMSClient(object):
    def __init__(self, request_template, supported_srs=None, http_client=None,
        resampling=None, supported_formats=None, lock=None):
        self.request_template = request_template
        self.http_client = http_client or HTTPClient()
        self.supported_srs = set(supported_srs or [])
        self.supported_formats = supported_formats or []
        self.resampling = resampling or base_config().image.resampling_method
        self.lock = lock

    def get_map(self, query):
        format = self.request_template.params.format
        if not format:
            format = query.format
            if self.supported_formats and format not in self.supported_formats:
                format = self.supported_formats[0]
        if self.supported_srs and query.srs not in self.supported_srs:
            return self._get_transformed(query, format)
        resp = self._retrieve(query, format)
        return ImageSource(resp, size=query.size, format=format)

    def _get_transformed(self, query, format):
        dst_srs = query.srs
        src_srs = self._best_supported_srs(dst_srs)
        dst_bbox = query.bbox
        src_bbox = dst_srs.transform_bbox_to(src_srs, dst_bbox)

        src_width, src_height = src_bbox[2]-src_bbox[0], src_bbox[3]-src_bbox[1]
        ratio = src_width/src_height
        dst_size = query.size
        xres, yres = src_width/dst_size[0], src_height/dst_size[1]
        if xres < yres:
            src_size = dst_size[0], int(dst_size[0]/ratio + 0.5)
        else:
            src_size = int(dst_size[1]*ratio +0.5), dst_size[1]

        src_query = MapQuery(src_bbox, src_size, src_srs, format)
        resp = self._retrieve(src_query, format)

        img = ImageSource(resp, format, size=src_size)

        img = ImageTransformer(src_srs, dst_srs, self.resampling).transform(img, src_bbox,
            query.size, dst_bbox)

        img.format = format
        return img

    def _best_supported_srs(self, srs):
        latlong = srs.is_latlong

        for srs in self.supported_srs:
            if srs.is_latlong == latlong:
                return srs

        return iter(self.supported_srs).next()

    def _retrieve(self, query, format):
        url = self._query_url(query, format)
        if self.lock:
            with self.lock():
                resp = self.http_client.open(url)
        else:
            resp = self.http_client.open(url)
        self._check_resp(resp)
        return resp

    def _check_resp(self, resp):
        if not resp.headers['Content-type'].startswith('image/'):
            raise SourceError('no image returned from source WMS')

    def _query_url(self, query, format):
        req = self.request_template.copy()
        req.params.bbox = query.bbox
        req.params.size = query.size
        req.params.srs = query.srs.srs_code
        req.params.format = format

        return req.complete_url

class WMSInfoClient(object):
    def __init__(self, request_template, supported_srs=None, http_client=None):
        self.request_template = request_template
        self.http_client = http_client or HTTPClient()
        self.supported_srs = set(supported_srs or [])

    def get_info(self, query):
        if self.supported_srs and query.srs not in self.supported_srs:
            return self._get_transformed(query)
        resp = self._retrieve(query)
        return resp

    def _get_transformed(self, query):
        req_srs = query.srs
        req_bbox = query.bbox
        info_srs = self._best_supported_srs(req_srs)
        info_bbox = req_srs.transform_bbox_to(info_srs, req_bbox)

        req_coord = make_lin_transf((0, query.size[1], query.size[0], 0), req_bbox)(query.pos)

        info_coord = req_srs.transform_to(info_srs, req_coord)
        info_pos = make_lin_transf((info_bbox), (0, query.size[1], query.size[0], 0))(info_coord)

        info_query = InfoQuery(info_bbox, query.size, info_srs, info_pos, query.info_format)
        return self._retrieve(info_query)

    def _best_supported_srs(self, srs):
        return iter(self.supported_srs).next()

    def _retrieve(self, query):
        url = self._query_url(query)
        return self.http_client.open(url)

    def _query_url(self, query):
        req = self.request_template.copy()
        req.params.bbox = query.bbox
        req.params.size = query.size
        req.params.pos = query.pos
        # del req.params['info_format']
        req.params['query_layers'] = req.params['layers']
        if query.info_format:
            req.params['info_format'] = query.info_format
        if not req.params.format:
            req.params.format = query.format or 'image/png'
        req.params.srs = query.srs.srs_code

        return req.complete_url
<|endoftext|>


<commit_before>import os
import stat
import sys
import tempfile
import unittest

from ds3.ds3 import *
from ds3.libds3 import LibDs3JobStatus

bucketName = "python_test_bucket"

def pathForResource(resourceName):
    encoding = sys.getfilesystemencoding()
    currentPath = os.path.dirname(unicode(__file__, encoding))
    return os.path.join(currentPath, "resources", resourceName)

def populateTestData(client, bucketName, resourceList = None, prefix = "", metadata = None):
    if not resourceList:
        resourceList = ["beowulf.txt", "sherlock_holmes.txt", "tale_of_two_cities.txt", "ulysses.txt"]

    def getSize(fileName):
        size = os.stat(pathForResource(fileName)).st_size
        return (prefix + fileName, size)

    client.putBucket(bucketName)

    pathes={prefix + fileName: pathForResource(fileName) for fileName in resourceList}

    fileList = map(getSize, resourceList)

    bulkResult = client.putBulk(bucketName, fileList)

    for chunk in bulkResult.chunks:
        allocateChunk = client.allocateChunk(chunk.chunkId)
        for obj in allocateChunk.chunk.objects:
            client.putObject(bucketName, obj.name, obj.offset, obj.length, bulkResult.jobId, pathes[obj.name], metadata)

def clearBucket(client, bucketName):
    bucketContents = client.getBucket(bucketName)
    for obj in bucketContents.objects:
        client.deleteObject(bucketName, obj.name)
    client.deleteBucket(bucketName)

class BasicClientFunctionTestCase(unittest.TestCase):

    def setUp(self):
        self.client = createClientFromEnv()

    def tearDown(self):
        try:
            clearBucket(self.client, bucketName)
        except Ds3Error as e:
            pass

    def testCreateBucket(self):
        self.client.putBucket(bucketName)

        bucketSet = frozenset(map(lambda service: service.name, self.client.getService()))

        self.assertTrue(bucketName in bucketSet)

    def testDeleteEmptyBucket(self):
        self.client.putBucket(bucketName)

        self.client.deleteBucket(bucketName)

        bucketSet = frozenset(map(lambda service: service.name, self.client.getService()))
        self.assertFalse(bucketName in bucketSet)

    def testDeleteObjects(self):
        populateTestData(self.client, bucketName)

        bucketContents = self.client.getBucket(bucketName)

        fileNameList = map(lambda obj: obj.name, bucketContents.objects)

        self.client.deleteObjects(bucketName, fileNameList)

        bucketContents = self.client.getBucket(bucketName)

        self.assertEqual(len(bucketContents.objects), 0)

    def testDeleteFolder(self):
        populateTestData(self.client, bucketName, prefix = "folder/")

        self.client.deleteFolder(bucketName, "folder")

        bucketResult = self.client.getBucket(bucketName)

        self.assertEqual(len(bucketResult.objects), 0)

    def testHeadObject(self):
        metadata={"name1":["value1"], "name2":"value2", "name3":("value3")}
        metadata_check={"name1":["value1"], "name2":["value2"], "name3":["value3"]}

        populateTestData(self.client, bucketName, resourceList=["beowulf.txt"], metadata=metadata)

        metadata_resp = self.client.headObject(bucketName, "beowulf.txt")

        self.assertEqual(metadata_check, metadata_resp)

    def testGetObjectWithMetadata(self):
        metadata={"name1":["value1"], "name2":["value2"], "name3":["value3"]}

        populateTestData(self.client, bucketName, resourceList=["beowulf.txt"], metadata=metadata)

        bucketContents = self.client.getBucket(bucketName)

        bulkGetResult = self.client.getBulk(bucketName, map(lambda obj: obj.name, bucketContents.objects))

        tempFiles = []

        availableChunks = self.client.getAvailableChunks(bulkGetResult.jobId)

        for obj in availableChunks.bulkPlan.chunks[0].objects:
            newFile = tempfile.mkstemp()
            tempFiles.append(newFile)

            metadata_resp=self.client.getObject(bucketName, obj.name, obj.offset, bulkGetResult.jobId, newFile[1])

        for tempFile in tempFiles:
            os.close(tempFile[0])
            os.remove(tempFile[1])

        jobStatusResponse = self.client.getJob(bulkGetResult.jobId)

        self.assertEqual(metadata, metadata_resp)
        self.assertEqual(jobStatusResponse.status, LibDs3JobStatus.COMPLETED)

    def testBulkPut(self):
        populateTestData(self.client, bucketName)

        bucketContents = self.client.getBucket(bucketName)

        self.assertEqual(len(bucketContents.objects), 4)

    def testBulkGet(self):
        populateTestData(self.client, bucketName)

        bucketContents = self.client.getBucket(bucketName)

        self.assertEqual(len(bucketContents.objects), 4)

        bulkGetResult = self.client.getBulk(bucketName, map(lambda obj: obj.name, bucketContents.objects))

        self.assertEqual(len(bulkGetResult.chunks), 1)
        self.assertEqual(len(bulkGetResult.chunks[0].objects), 4)

        tempFiles = []

        availableChunks = self.client.getAvailableChunks(bulkGetResult.jobId)

        self.assertTrue(availableChunks != None)
        self.assertEqual(len(availableChunks.bulkPlan.chunks), 1)

        for obj in availableChunks.bulkPlan.chunks[0].objects:
            newFile = tempfile.mkstemp()
            tempFiles.append(newFile)

            self.client.getObject(bucketName, obj.name, obj.offset, bulkGetResult.jobId, newFile[1])

        for tempFile in tempFiles:
            os.close(tempFile[0])
            os.remove(tempFile[1])

        jobStatusResponse = self.client.getJob(bulkGetResult.jobId)
        self.assertEqual(jobStatusResponse.status, LibDs3JobStatus.COMPLETED)

    def testPrefix(self):
        populateTestData(self.client, bucketName)

        bucketContents = self.client.getBucket(bucketName, prefix = "beo")

        self.assertEqual(len(bucketContents.objects), 1)

    def testPagination(self):
        fileList = []
        for i in xrange(0, 15):
            fileList.append(("file" + str(i), 0))

        self.client.putBucket(bucketName)
        self.client.putBulk(bucketName, fileList)

        bucketResult = self.client.getBucket(bucketName, maxKeys = 5)

        self.assertEqual(len(bucketResult.objects), 5)
        self.assertTrue(bucketResult.nextMarker != None)
        self.assertEqual(bucketResult.objects[4].name[4:6], "12")

        bucketResult = self.client.getBucket(bucketName, maxKeys = 5, nextMarker = bucketResult.nextMarker)

        self.assertEqual(len(bucketResult.objects), 5)
        self.assertTrue(bucketResult.nextMarker != None)
        self.assertEqual(bucketResult.objects[4].name[4], "4")

        bucketResult = self.client.getBucket(bucketName, maxKeys = 5, nextMarker = bucketResult.nextMarker)

        self.assertEqual(len(bucketResult.objects), 5)
        self.assertTrue(bucketResult.nextMarker == None)
        self.assertEqual(bucketResult.objects[4].name[4], "9")

    def testDeleteBadKey(self):
        self.client.putBucket(bucketName)
        try:
            self.client.deleteObject(bucketName, "badFile")
        except Ds3Error as e:
            self.assertEqual(e.statusCode, 404)

    def testDelimiter(self):
        fileList = []

        for i in xrange(0, 10):
            fileList.append(("dir/file" + str(i), 0))

        for i in xrange(0, 10):
            fileList.append(("file" + str(i), 0))

        self.client.putBucket(bucketName)

        self.client.putBulk(bucketName, fileList)

        bucketResult = self.client.getBucket(bucketName, delimiter = "/")

        self.assertEqual(len(bucketResult.objects), 10)

        self.assertEqual(len(bucketResult.commonPrefixes), 1)
        self.assertEqual(bucketResult.commonPrefixes[0], "dir/")
<commit_msg>renamed some stuff to make things more organized<commit_after>import os
import stat
import sys
import tempfile
import unittest

from ds3.ds3 import *
from ds3.libds3 import LibDs3JobStatus

bucketName = "python_test_bucket"

def pathForResource(resourceName):
    encoding = sys.getfilesystemencoding()
    currentPath = os.path.dirname(unicode(__file__, encoding))
    return os.path.join(currentPath, "resources", resourceName)

def populateTestData(client, bucketName, resourceList = None, prefix = "", metadata = None):
    if not resourceList:
        resourceList = ["beowulf.txt", "sherlock_holmes.txt", "tale_of_two_cities.txt", "ulysses.txt"]

    def getSize(fileName):
        size = os.stat(pathForResource(fileName)).st_size
        return (prefix + fileName, size)

    client.putBucket(bucketName)

    pathes={prefix + fileName: pathForResource(fileName) for fileName in resourceList}

    fileList = map(getSize, resourceList)

    bulkResult = client.putBulk(bucketName, fileList)

    for chunk in bulkResult.chunks:
        allocateChunk = client.allocateChunk(chunk.chunkId)
        for obj in allocateChunk.chunk.objects:
            client.putObject(bucketName, obj.name, obj.offset, obj.length, bulkResult.jobId, pathes[obj.name], metadata)

def clearBucket(client, bucketName):
    bucketContents = client.getBucket(bucketName)
    for obj in bucketContents.objects:
        client.deleteObject(bucketName, obj.name)
    client.deleteBucket(bucketName)

class BasicClientFunctionTestCase(unittest.TestCase):

    def setUp(self):
        self.client = createClientFromEnv()

    def tearDown(self):
        try:
            clearBucket(self.client, bucketName)
        except Ds3Error as e:
            pass

    def testPutBucket(self):
        self.client.putBucket(bucketName)

        bucketSet = frozenset(map(lambda service: service.name, self.client.getService()))

        self.assertTrue(bucketName in bucketSet)

    def testDeleteBucket(self):
        self.client.putBucket(bucketName)

        self.client.deleteBucket(bucketName)

        bucketSet = frozenset(map(lambda service: service.name, self.client.getService()))
        self.assertFalse(bucketName in bucketSet)

    def testDeleteObjects(self):
        populateTestData(self.client, bucketName)

        bucketContents = self.client.getBucket(bucketName)

        fileNameList = map(lambda obj: obj.name, bucketContents.objects)

        self.client.deleteObjects(bucketName, fileNameList)

        bucketContents = self.client.getBucket(bucketName)

        self.assertEqual(len(bucketContents.objects), 0)

    def testDeleteFolder(self):
        populateTestData(self.client, bucketName, prefix = "folder/")

        self.client.deleteFolder(bucketName, "folder")

        bucketResult = self.client.getBucket(bucketName)

        self.assertEqual(len(bucketResult.objects), 0)

    def testHeadObject(self):
        metadata={"name1":["value1"], "name2":"value2", "name3":("value3")}
        metadata_check={"name1":["value1"], "name2":["value2"], "name3":["value3"]}

        populateTestData(self.client, bucketName, resourceList=["beowulf.txt"], metadata=metadata)

        metadata_resp = self.client.headObject(bucketName, "beowulf.txt")

        self.assertEqual(metadata_check, metadata_resp)

    def testGetObjectWithMetadata(self):
        metadata={"name1":["value1"], "name2":["value2"], "name3":["value3"]}

        populateTestData(self.client, bucketName, resourceList=["beowulf.txt"], metadata=metadata)

        bucketContents = self.client.getBucket(bucketName)

        bulkGetResult = self.client.getBulk(bucketName, map(lambda obj: obj.name, bucketContents.objects))

        tempFiles = []

        availableChunks = self.client.getAvailableChunks(bulkGetResult.jobId)

        for obj in availableChunks.bulkPlan.chunks[0].objects:
            newFile = tempfile.mkstemp()
            tempFiles.append(newFile)

            metadata_resp=self.client.getObject(bucketName, obj.name, obj.offset, bulkGetResult.jobId, newFile[1])

        for tempFile in tempFiles:
            os.close(tempFile[0])
            os.remove(tempFile[1])

        jobStatusResponse = self.client.getJob(bulkGetResult.jobId)

        self.assertEqual(metadata, metadata_resp)
        self.assertEqual(jobStatusResponse.status, LibDs3JobStatus.COMPLETED)

    def testPutBulk(self):
        populateTestData(self.client, bucketName)

        bucketContents = self.client.getBucket(bucketName)

        self.assertEqual(len(bucketContents.objects), 4)

    def testGetBulk(self):
        populateTestData(self.client, bucketName)

        bucketContents = self.client.getBucket(bucketName)

        self.assertEqual(len(bucketContents.objects), 4)

        bulkGetResult = self.client.getBulk(bucketName, map(lambda obj: obj.name, bucketContents.objects))

        self.assertEqual(len(bulkGetResult.chunks), 1)
        self.assertEqual(len(bulkGetResult.chunks[0].objects), 4)

        tempFiles = []

        availableChunks = self.client.getAvailableChunks(bulkGetResult.jobId)

        self.assertTrue(availableChunks != None)
        self.assertEqual(len(availableChunks.bulkPlan.chunks), 1)

        for obj in availableChunks.bulkPlan.chunks[0].objects:
            newFile = tempfile.mkstemp()
            tempFiles.append(newFile)

            self.client.getObject(bucketName, obj.name, obj.offset, bulkGetResult.jobId, newFile[1])

        for tempFile in tempFiles:
            os.close(tempFile[0])
            os.remove(tempFile[1])

        jobStatusResponse = self.client.getJob(bulkGetResult.jobId)
        self.assertEqual(jobStatusResponse.status, LibDs3JobStatus.COMPLETED)

    def testPrefix(self):
        populateTestData(self.client, bucketName)

        bucketContents = self.client.getBucket(bucketName, prefix = "beo")

        self.assertEqual(len(bucketContents.objects), 1)

    def testPagination(self):
        fileList = []
        for i in xrange(0, 15):
            fileList.append(("file" + str(i), 0))

        self.client.putBucket(bucketName)
        self.client.putBulk(bucketName, fileList)

        bucketResult = self.client.getBucket(bucketName, maxKeys = 5)

        self.assertEqual(len(bucketResult.objects), 5)
        self.assertTrue(bucketResult.nextMarker != None)
        self.assertEqual(bucketResult.objects[4].name[4:6], "12")

        bucketResult = self.client.getBucket(bucketName, maxKeys = 5, nextMarker = bucketResult.nextMarker)

        self.assertEqual(len(bucketResult.objects), 5)
        self.assertTrue(bucketResult.nextMarker != None)
        self.assertEqual(bucketResult.objects[4].name[4], "4")

        bucketResult = self.client.getBucket(bucketName, maxKeys = 5, nextMarker = bucketResult.nextMarker)

        self.assertEqual(len(bucketResult.objects), 5)
        self.assertTrue(bucketResult.nextMarker == None)
        self.assertEqual(bucketResult.objects[4].name[4], "9")

    def testDeleteBadKey(self):
        self.client.putBucket(bucketName)
        try:
            self.client.deleteObject(bucketName, "badFile")
        except Ds3Error as e:
            self.assertEqual(e.statusCode, 404)

    def testDelimiter(self):
        fileList = []

        for i in xrange(0, 10):
            fileList.append(("dir/file" + str(i), 0))

        for i in xrange(0, 10):
            fileList.append(("file" + str(i), 0))

        self.client.putBucket(bucketName)

        self.client.putBulk(bucketName, fileList)

        bucketResult = self.client.getBucket(bucketName, delimiter = "/")

        self.assertEqual(len(bucketResult.objects), 10)

        self.assertEqual(len(bucketResult.commonPrefixes), 1)
        self.assertEqual(bucketResult.commonPrefixes[0], "dir/")
<|endoftext|>