<commit_before>"""
Created on 23/10/2012
@author: Ben Davies
"""

import os, sys
from os.path import join, splitext
from math import modf
from numbers import Number
from tempfile import mkstemp
from itertools import product
from subprocess import check_call

from scipy.stats.stats import nanmean
from numpy import array, where, nan, isnan, mean, float32, zeros

from shared import Ifg, DEM
from roipac import filename_pair, write_roipac_header
from config import parse_namelist
from config import OBS_DIR, IFG_CROP_OPT, IFG_LKSX, IFG_LKSY, IFG_FILE_LIST
from config import IFG_XFIRST, IFG_XLAST, IFG_YFIRST, IFG_YLAST, DEM_FILE
from config import PROJECTION_FLAG
from ifgconstants import X_FIRST, Y_FIRST, X_LAST, Y_LAST, X_STEP, Y_STEP
from ifgconstants import WIDTH, FILE_LENGTH, WAVELENGTH
from ifgconstants import Z_OFFSET, Z_SCALE, PROJECTION, DATUM

# Constants
MINIMUM_CROP = 1
MAXIMUM_CROP = 2
CUSTOM_CROP = 3
CROP_OPTIONS = [MINIMUM_CROP, MAXIMUM_CROP, CUSTOM_CROP]

GRID_TOL = 1e-6

def prepare_ifgs(params, thresh=0.5, use_exceptions=False, verbose=False):
	"""Produces multilooked/resampled data files for PyRate analysis.
	params - dict of named values from pyrate config file
	threshhold - 0.0->1.0 controls NaN handling when resampling to coarser grids,
	             value is proportion above which the number of NaNs in an area is
	             considered invalid. threshhold=0 resamples to NaN if 1 or more
	             contributing cells are NaNs. At 0.25, it resamples to NaN if 1/4
	             or more contributing cells are NaNs. At 1.0, areas are resampled
	             to NaN only if all area cells are NaNs.
	use_exceptions - TODO: PROB REMOVE
	verbose - controls level of gdalwarp output
	"""
	# validate config file settings
	crop_opt = params[IFG_CROP_OPT]
	if crop_opt not in CROP_OPTIONS:
		raise PreprocessingException("Unrecognised crop option: %s" % crop_opt)

	check_looks(params)
	paths = [join(params[OBS_DIR], p) for p in parse_namelist(params[IFG_FILE_LIST])]
	ifgs = [Ifg(p) for p in paths]

	# treat DEM as Ifg as core API is equivalent
	if params.has_key(DEM_FILE):
		ifgs.append(DEM(params[DEM_FILE]))

	check_resolution(ifgs)
	for i in ifgs:
		i.open()

	# calculate crop option for passing to gdalwarp
	if crop_opt == MINIMUM_CROP:
		xmin, ymin, xmax, ymax = min_bounds(ifgs)
	elif crop_opt == MAXIMUM_CROP:
		xmin, ymin, xmax, ymax = max_bounds(ifgs)

	elif crop_opt == CUSTOM_CROP:
		xmin, xmax = params[IFG_XFIRST], params[IFG_XLAST]
		ymin, ymax = params[IFG_YLAST], params[IFG_YFIRST]
		check_crop_coords(ifgs, xmin, xmax, ymin, ymax, use_exceptions)

	# calculate args for gdalwarp reprojection
	extents = [str(s) for s in (xmin, ymin, xmax, ymax) ]
	resolution = None
	if params[IFG_LKSX] > 1 or params[IFG_LKSY] > 1:
		resolution = [params[IFG_LKSX] * i.X_STEP, params[IFG_LKSY] * i.Y_STEP ]

	# Generate gdalwarp args for each ifg, some hackery required to interface with
	# cmd line interface. For resampling, gdalwarp is called 2x, once to subset
	# the source data for Pirate's form of averaging/resampling, the second to
	# generate the final dataset with correct extents/shape/cell count. Without
	# resampling, gdalwarp is only needed to cut out the required segment.
	for i in ifgs:
		xl, yl = params[IFG_LKSX], params[IFG_LKSY]
		warp(i, xl, yl, extents, resolution, thresh, verbose)

		# FIXME: check if there are orbital corrections for resampling

def _file_ext(raster):
	'''Returns file ext string based on type of raster.'''
	if isinstance(raster, Ifg):
		return "tif"
	elif hasattr(raster, DATUM):
		return "dem"
	else:
		raise NotImplementedError("TODO: additional raster types?")

def _resample_ifg(ifg, cmd, x_looks, y_looks, thresh):
	'''Convenience function to resample data from a given Ifg (more coarse).'''

	# HACK: create tmp ifg, extract data array for manual resampling as gdalwarp
	# lacks Pirate's averaging method
	tmp_path = mkstemp()[1]
	check_call(cmd + [ifg.data_path, tmp_path])
	tmp = type(ifg)(tmp_path, ifg.hdr_path) # dynamically handle Ifgs & Rasters
	tmp.open()

	if isinstance(ifg, Ifg):
		# TODO: resample amplitude band too?
		data = tmp.phase_band.ReadAsArray()
		data = where(data == 0, nan, data) # flag incoherent cells as NaNs
	elif isinstance(ifg, DEM):
		data = tmp.height_band.ReadAsArray()
	else:
		raise NotImplementedError("TODO: other raster types to handle?")

	del tmp # manual close
	os.remove(tmp_path)
	return resample(data, x_looks, y_looks, thresh)

def warp(ifg, x_looks, y_looks, extents, resolution, thresh, verbose):
	'''TODO

	resolution - [xres, yres] or None. Sets resolution output Ifg metadata. Use
	             None if raster size is not being changed.
	thresh - see thresh in prepare_ifgs().
	'''

	# dynamically build command for call to gdalwarp
	cmd = ["gdalwarp", "-overwrite", "-srcnodata", "None", "-te"] + extents
	if not verbose: cmd.append("-q")

	# HACK: if resampling, cut segment with gdalwarp & manually average tiles
	data = None
	if resolution:
		data = _resample_ifg(ifg, cmd, x_looks, y_looks, thresh)
		cmd += ["-tr"] + [str(r) for r in resolution] # change res of final output

	# use GDAL to cut (and resample) the final output layers
	s = splitext(ifg.data_path)
	ext = _file_ext(ifg)
	looks_path = s[0] + "_%srlks.%s" % (y_looks, ext)
	cmd += [ifg.data_path, looks_path]
	check_call(cmd)

	# Add missing/updated metadata to resampled ifg/DEM
	new_lyr = type(ifg)(looks_path, ifg.hdr_path)
	new_lyr.open(readonly=False)
	new_hdr_path = filename_pair(looks_path)[1]
	_create_new_roipac_header(ifg, new_lyr)

	# for non-DEMs, phase bands need extra metadata & conversions
	if hasattr(new_lyr, "phase_band"):
		new_lyr.phase_band.SetNoDataValue(nan)

		if data is None: # data wasn't resampled, so flag incoherent cells
			data = new_lyr.phase_band.ReadAsArray()
			data = where(data == 0, nan, data)

		# TODO: projection
		#if params.has_key(PROJECTION_FLAG):
		#	reproject()

		# tricky: write either resampled or the basic cropped data to new layer
		new_lyr.phase_band.WriteArray(data)
		new_lyr.nan_converted = True

def resample(data, xscale, yscale, threshold):
	"""Resamples/averages 'data' to return an array from the averaging of blocks
	of several tiles in 'data'. NB: Assumes incoherent cells are NaNs.

	data - source array to resample to different size
	xscale - number of cells to average along X axis
	yscale - number of Y axis cells to average
	threshold - minimum allowable proportion of NaN cells (range from 0.0-1.0),
	eg. 0.25 = 1/4 or more as NaNs results in a NaN value for the output cell.
	"""
	if threshold < 0 or threshold > 1:
		raise ValueError("threshold must be >= 0 and <= 1")

	# TODO: check scaling factors are ints

	ysize, xsize = data.shape
	xres, yres = (xsize / xscale), (ysize / yscale)
	dest = zeros((yres, xres), dtype=float32) * nan
	tile_cell_count = xscale * yscale

	# calc mean without nans (fractional threshold ignores tiles with excess NaNs)
	for y,x in product(xrange(yres), xrange(xres)):
		tile = data[y * yscale : (y+1) * yscale, x * xscale : (x+1) * xscale]
		non_nans = [ tile[crd] for crd in product(xrange(yscale), xrange(xscale))
		                  if not isnan(tile[crd])]
		nan_fraction = (tile_cell_count - len(non_nans)) / float(tile_cell_count)

		if nan_fraction < threshold or (nan_fraction == 0 and threshold == 0):
			dest[y,x] = mean(non_nans)

	return dest

def reproject():
	raise NotImplementedError("TODO: Reprojection LOS/Horiz/Vert")

def _create_new_roipac_header(src_ifg, new_ifg, dest=None):
	"""Translates an existing ROIPAC Ifg header to new values following cropping
	and resampling."""

	geotrans = new_ifg.dataset.GetGeoTransform()
	newpars = { WIDTH: new_ifg.dataset.RasterXSize,
							FILE_LENGTH: new_ifg.dataset.RasterYSize,
							X_FIRST: geotrans[0],
							X_STEP: geotrans[1],
							Y_FIRST: geotrans[3],
							Y_STEP: geotrans[5] }

	if isinstance(src_ifg, Ifg):
		newpars[WAVELENGTH] = src_ifg.WAVELENGTH
	elif isinstance(src_ifg, DEM):
		for a in [Z_OFFSET, Z_SCALE, PROJECTION, DATUM]:
			newpars[a] = getattr(src_ifg, a)

	if dest is None:
		dest = filename_pair(new_ifg.data_path)[1]
	write_roipac_header(newpars, dest)

def check_resolution(ifgs):
	"""Verifies Ifg resolutions are equal for the given grids"""
	for var in [X_STEP, Y_STEP]:
		values = array([getattr(i, var) for i in ifgs])
		if not (values == values[0]).all():
			msg = "Grid resolution does not match for %s" % var
			raise PreprocessingException(msg)

def check_looks(params):
	"""Verifies looks parameters are valid"""
	xscale = params[IFG_LKSX]
	yscale = params[IFG_LKSY]
	if not (isinstance(xscale, Number) and isinstance(yscale, Number)):
		msg = "Non-numeric looks parameter(s), x: %s, y: %s" % (xscale, yscale)
		raise PreprocessingException(msg)

	if not (xscale > 0 and yscale > 0):
		msg = "Invalid looks parameter(s), x: %s, y: %s" % (xscale, yscale)
		raise PreprocessingException(msg)

def min_bounds(ifgs):
	'''Returns bounds for overlapping area of the given interferograms.'''
	xmin = max([i.X_FIRST for i in ifgs])
	ymax = min([i.Y_FIRST for i in ifgs])
	xmax = min([i.X_LAST for i in ifgs])
	ymin = max([i.Y_LAST for i in ifgs])
	return xmin, ymin, xmax, ymax

def max_bounds(ifgs):
	'''Returns bounds for the total area covered by the given interferograms.'''
	xmin = min([i.X_FIRST for i in ifgs])
	ymax = max([i.Y_FIRST for i in ifgs])
	xmax = max([i.X_LAST for i in ifgs])
	ymin = min([i.Y_LAST for i in ifgs])
	return xmin, ymin, xmax, ymax

def check_crop_coords(ifgs, xmin, xmax, ymin, ymax, use_exceptions=False):
	'''Ensures cropping coords line up with grid system within tolerance.'''
	# NB: assumption is the first Ifg is correct, so only test against it
	i = ifgs[0]
	for par, crop, step in zip([X_FIRST, X_LAST, Y_FIRST, Y_LAST],
														[xmin, xmax, ymax, ymin],
														[i.X_STEP, i.X_STEP, i.Y_STEP, i.Y_STEP]):

		# is diff of the given extent from grid a multiple of X|Y_STEP ?
		param = getattr(i, par)
		diff = abs(crop - param)
		remainder = abs(modf(diff / step)[0])

		# handle cases where division gives remainder near zero, or just < 1
		if remainder > GRID_TOL and remainder < (1 - GRID_TOL):
			msg = "%s crop extent not within %s of grid coordinate" % (par, GRID_TOL)
			if use_exceptions:
				raise PreprocessingException(msg)
			sys.stderr.write("WARN: %s\n" % msg)

class PreprocessingException(Exception):
	pass
<commit_msg>Fixed doco.<commit_after>"""
Created on 23/10/2012
@author: Ben Davies
"""

import os, sys
from os.path import join, splitext
from math import modf
from numbers import Number
from tempfile import mkstemp
from itertools import product
from subprocess import check_call

from scipy.stats.stats import nanmean
from numpy import array, where, nan, isnan, mean, float32, zeros

from shared import Ifg, DEM
from roipac import filename_pair, write_roipac_header
from config import parse_namelist
from config import OBS_DIR, IFG_CROP_OPT, IFG_LKSX, IFG_LKSY, IFG_FILE_LIST
from config import IFG_XFIRST, IFG_XLAST, IFG_YFIRST, IFG_YLAST, DEM_FILE
from config import PROJECTION_FLAG
from ifgconstants import X_FIRST, Y_FIRST, X_LAST, Y_LAST, X_STEP, Y_STEP
from ifgconstants import WIDTH, FILE_LENGTH, WAVELENGTH
from ifgconstants import Z_OFFSET, Z_SCALE, PROJECTION, DATUM

# Constants
MINIMUM_CROP = 1
MAXIMUM_CROP = 2
CUSTOM_CROP = 3
CROP_OPTIONS = [MINIMUM_CROP, MAXIMUM_CROP, CUSTOM_CROP]

GRID_TOL = 1e-6

def prepare_ifgs(params, thresh=0.5, use_exceptions=False, verbose=False):
	"""Produces multilooked/resampled data files for PyRate analysis.
	params - dict of named values from pyrate config file
	threshhold - 0.0->1.0 controls NaN handling when resampling to coarser grids,
	             value is proportion above which the number of NaNs in an area is
	             considered invalid. threshhold=0 resamples to NaN if 1 or more
	             contributing cells are NaNs. At 0.25, it resamples to NaN if 1/4
	             or more contributing cells are NaNs. At 1.0, areas are resampled
	             to NaN only if all area cells are NaNs.
	use_exceptions - TODO: PROB REMOVE
	verbose - controls level of gdalwarp output
	"""
	# validate config file settings
	crop_opt = params[IFG_CROP_OPT]
	if crop_opt not in CROP_OPTIONS:
		raise PreprocessingException("Unrecognised crop option: %s" % crop_opt)

	check_looks(params)
	paths = [join(params[OBS_DIR], p) for p in parse_namelist(params[IFG_FILE_LIST])]
	ifgs = [Ifg(p) for p in paths]

	# treat DEM as Ifg as core API is equivalent
	if params.has_key(DEM_FILE):
		ifgs.append(DEM(params[DEM_FILE]))

	check_resolution(ifgs)
	for i in ifgs:
		i.open()

	# calculate crop option for passing to gdalwarp
	if crop_opt == MINIMUM_CROP:
		xmin, ymin, xmax, ymax = min_bounds(ifgs)
	elif crop_opt == MAXIMUM_CROP:
		xmin, ymin, xmax, ymax = max_bounds(ifgs)

	elif crop_opt == CUSTOM_CROP:
		xmin, xmax = params[IFG_XFIRST], params[IFG_XLAST]
		ymin, ymax = params[IFG_YLAST], params[IFG_YFIRST]
		check_crop_coords(ifgs, xmin, xmax, ymin, ymax, use_exceptions)

	# calculate args for gdalwarp reprojection
	extents = [str(s) for s in (xmin, ymin, xmax, ymax) ]
	resolution = None
	if params[IFG_LKSX] > 1 or params[IFG_LKSY] > 1:
		resolution = [params[IFG_LKSX] * i.X_STEP, params[IFG_LKSY] * i.Y_STEP ]

	# Generate gdalwarp args for each ifg, some hackery required to interface with
	# cmd line interface. For resampling, gdalwarp is called 2x, once to subset
	# the source data for Pirate's form of averaging/resampling, the second to
	# generate the final dataset with correct extents/shape/cell count. Without
	# resampling, gdalwarp is only needed to cut out the required segment.
	for i in ifgs:
		xl, yl = params[IFG_LKSX], params[IFG_LKSY]
		warp(i, xl, yl, extents, resolution, thresh, verbose)

		# FIXME: check if there are orbital corrections for resampling

def _file_ext(raster):
	'''Returns file ext string based on type of raster.'''
	if isinstance(raster, Ifg):
		return "tif"
	elif hasattr(raster, DATUM):
		return "dem"
	else:
		raise NotImplementedError("TODO: additional raster types?")

def _resample_ifg(ifg, cmd, x_looks, y_looks, thresh):
	'''Convenience function to resample data from a given Ifg (more coarse).'''

	# HACK: create tmp ifg, extract data array for manual resampling as gdalwarp
	# lacks Pirate's averaging method
	tmp_path = mkstemp()[1]
	check_call(cmd + [ifg.data_path, tmp_path])
	tmp = type(ifg)(tmp_path, ifg.hdr_path) # dynamically handle Ifgs & Rasters
	tmp.open()

	if isinstance(ifg, Ifg):
		# TODO: resample amplitude band too?
		data = tmp.phase_band.ReadAsArray()
		data = where(data == 0, nan, data) # flag incoherent cells as NaNs
	elif isinstance(ifg, DEM):
		data = tmp.height_band.ReadAsArray()
	else:
		raise NotImplementedError("TODO: other raster types to handle?")

	del tmp # manual close
	os.remove(tmp_path)
	return resample(data, x_looks, y_looks, thresh)

def warp(ifg, x_looks, y_looks, extents, resolution, thresh, verbose):
	'''TODO
	xlooks - integer factor to scale X axis by, 5 is 5x smaller, 1 is no change.
	ylooks - as xlooks, but for Y axis
	extents - TODO
	resolution - [xres, yres] or None. Sets resolution output Ifg metadata. Use
	             None if raster size is not being changed.
	thresh - see thresh in prepare_ifgs().
	verbose - True = print gdalwarp output to stdout
	'''

	# dynamically build command for call to gdalwarp
	cmd = ["gdalwarp", "-overwrite", "-srcnodata", "None", "-te"] + extents
	if not verbose: cmd.append("-q")

	# HACK: if resampling, cut segment with gdalwarp & manually average tiles
	data = None
	if resolution:
		data = _resample_ifg(ifg, cmd, x_looks, y_looks, thresh)
		cmd += ["-tr"] + [str(r) for r in resolution] # change res of final output

	# use GDAL to cut (and resample) the final output layers
	s = splitext(ifg.data_path)
	ext = _file_ext(ifg)
	looks_path = s[0] + "_%srlks.%s" % (y_looks, ext)
	cmd += [ifg.data_path, looks_path]
	check_call(cmd)

	# Add missing/updated metadata to resampled ifg/DEM
	new_lyr = type(ifg)(looks_path, ifg.hdr_path)
	new_lyr.open(readonly=False)
	new_hdr_path = filename_pair(looks_path)[1]
	_create_new_roipac_header(ifg, new_lyr)

	# for non-DEMs, phase bands need extra metadata & conversions
	if hasattr(new_lyr, "phase_band"):
		new_lyr.phase_band.SetNoDataValue(nan)

		if data is None: # data wasn't resampled, so flag incoherent cells
			data = new_lyr.phase_band.ReadAsArray()
			data = where(data == 0, nan, data)

		# TODO: projection
		#if params.has_key(PROJECTION_FLAG):
		#	reproject()

		# tricky: write either resampled or the basic cropped data to new layer
		new_lyr.phase_band.WriteArray(data)
		new_lyr.nan_converted = True

def resample(data, xscale, yscale, threshold):
	"""Resamples/averages 'data' to return an array from the averaging of blocks
	of several tiles in 'data'. NB: Assumes incoherent cells are NaNs.

	data - source array to resample to different size
	xscale - number of cells to average along X axis
	yscale - number of Y axis cells to average
	threshold - minimum allowable proportion of NaN cells (range from 0.0-1.0),
	eg. 0.25 = 1/4 or more as NaNs results in a NaN value for the output cell.
	"""
	if threshold < 0 or threshold > 1:
		raise ValueError("threshold must be >= 0 and <= 1")

	# TODO: check scaling factors are ints

	ysize, xsize = data.shape
	xres, yres = (xsize / xscale), (ysize / yscale)
	dest = zeros((yres, xres), dtype=float32) * nan
	tile_cell_count = xscale * yscale

	# calc mean without nans (fractional threshold ignores tiles with excess NaNs)
	for y,x in product(xrange(yres), xrange(xres)):
		tile = data[y * yscale : (y+1) * yscale, x * xscale : (x+1) * xscale]
		non_nans = [ tile[crd] for crd in product(xrange(yscale), xrange(xscale))
		                  if not isnan(tile[crd])]
		nan_fraction = (tile_cell_count - len(non_nans)) / float(tile_cell_count)

		if nan_fraction < threshold or (nan_fraction == 0 and threshold == 0):
			dest[y,x] = mean(non_nans)

	return dest

def reproject():
	raise NotImplementedError("TODO: Reprojection LOS/Horiz/Vert")

def _create_new_roipac_header(src_ifg, new_ifg, dest=None):
	"""Translates an existing ROIPAC Ifg header to new values following cropping
	and resampling."""

	geotrans = new_ifg.dataset.GetGeoTransform()
	newpars = { WIDTH: new_ifg.dataset.RasterXSize,
							FILE_LENGTH: new_ifg.dataset.RasterYSize,
							X_FIRST: geotrans[0],
							X_STEP: geotrans[1],
							Y_FIRST: geotrans[3],
							Y_STEP: geotrans[5] }

	if isinstance(src_ifg, Ifg):
		newpars[WAVELENGTH] = src_ifg.WAVELENGTH
	elif isinstance(src_ifg, DEM):
		for a in [Z_OFFSET, Z_SCALE, PROJECTION, DATUM]:
			newpars[a] = getattr(src_ifg, a)

	if dest is None:
		dest = filename_pair(new_ifg.data_path)[1]
	write_roipac_header(newpars, dest)

def check_resolution(ifgs):
	"""Verifies Ifg resolutions are equal for the given grids"""
	for var in [X_STEP, Y_STEP]:
		values = array([getattr(i, var) for i in ifgs])
		if not (values == values[0]).all():
			msg = "Grid resolution does not match for %s" % var
			raise PreprocessingException(msg)

def check_looks(params):
	"""Verifies looks parameters are valid"""
	xscale = params[IFG_LKSX]
	yscale = params[IFG_LKSY]
	if not (isinstance(xscale, Number) and isinstance(yscale, Number)):
		msg = "Non-numeric looks parameter(s), x: %s, y: %s" % (xscale, yscale)
		raise PreprocessingException(msg)

	if not (xscale > 0 and yscale > 0):
		msg = "Invalid looks parameter(s), x: %s, y: %s" % (xscale, yscale)
		raise PreprocessingException(msg)

def min_bounds(ifgs):
	'''Returns bounds for overlapping area of the given interferograms.'''
	xmin = max([i.X_FIRST for i in ifgs])
	ymax = min([i.Y_FIRST for i in ifgs])
	xmax = min([i.X_LAST for i in ifgs])
	ymin = max([i.Y_LAST for i in ifgs])
	return xmin, ymin, xmax, ymax

def max_bounds(ifgs):
	'''Returns bounds for the total area covered by the given interferograms.'''
	xmin = min([i.X_FIRST for i in ifgs])
	ymax = max([i.Y_FIRST for i in ifgs])
	xmax = max([i.X_LAST for i in ifgs])
	ymin = min([i.Y_LAST for i in ifgs])
	return xmin, ymin, xmax, ymax

def check_crop_coords(ifgs, xmin, xmax, ymin, ymax, use_exceptions=False):
	'''Ensures cropping coords line up with grid system within tolerance.'''
	# NB: assumption is the first Ifg is correct, so only test against it
	i = ifgs[0]
	for par, crop, step in zip([X_FIRST, X_LAST, Y_FIRST, Y_LAST],
														[xmin, xmax, ymax, ymin],
														[i.X_STEP, i.X_STEP, i.Y_STEP, i.Y_STEP]):

		# is diff of the given extent from grid a multiple of X|Y_STEP ?
		param = getattr(i, par)
		diff = abs(crop - param)
		remainder = abs(modf(diff / step)[0])

		# handle cases where division gives remainder near zero, or just < 1
		if remainder > GRID_TOL and remainder < (1 - GRID_TOL):
			msg = "%s crop extent not within %s of grid coordinate" % (par, GRID_TOL)
			if use_exceptions:
				raise PreprocessingException(msg)
			sys.stderr.write("WARN: %s\n" % msg)

class PreprocessingException(Exception):
	pass
<|endoftext|>


<commit_before># Game python file
import pygame
import button
import random
import time

def update(game):
    pass

# Deze class zorgt ervoor dat het dice systeem werkt
class Dice:
    def __init__(self):
        # zet de begin image van de die naar een lege
        self.image ="assets\img\die0.png"
    def onclick(self,game):
        # TODO: niet display.flip gebruiken
        for x in range(20):
            self.newimg = "assets\img\die{}.png".format(random.randrange(1,7))
            while self.newimg == self.image:
                self.newimg = "assets\img\die{}.png".format(random.randrange(1,7))
            self.image = self.newimg
            self.draw(game)
            pygame.display.flip()
            time.sleep(0.05)
        # dit pakt een random nummer van 1 t/m 6 en slaat het op in game.dice_roll
        game.dice_roll = random.randrange(1, 7)
        # dit zet het plaatje van de die naar hetgeen wat gegooid is
        self.image = "assets\img\die{}.png".format(game.dice_roll)
    def draw(self,game):
        # dit tekent de die
        button.draw_img(game, game.width - 70, game.height - 70, 64, 64, "", 0, self.image, (0,0,0), self.onclick)

dice = Dice()

def draw(game):
    dice.draw(game)
    pygame.draw.rect(game.screen,(255,0,0),(32,32,110,game.height * 0.8))
    pygame.draw.rect(game.screen,(255,239,0),(162,32,110,game.height * 0.8))
    pygame.draw.rect(game.screen,(52,163,253),(292,32,110,game.height * 0.8))
    pygame.draw.rect(game.screen,(24,208,27),(422,32,110,game.height * 0.8))<commit_msg>Changed position dice + categorie start<commit_after># Game python file
import pygame
import button
import random
import time

def update(game):
    pass

# Deze class zorgt ervoor dat het dice systeem werkt
class Dice:
    def __init__(self):
        # zet de begin image van de die naar een lege
        self.image ="assets\img\die0.png"
    def onclick(self,game):
        # TODO: niet display.flip gebruiken
        for x in range(20):
            self.newimg = "assets\img\die{}.png".format(random.randrange(1,7))
            while self.newimg == self.image:
                self.newimg = "assets\img\die{}.png".format(random.randrange(1,7))
            self.image = self.newimg
            self.draw(game)
            pygame.display.flip()
            time.sleep(0.05)
        # dit pakt een random nummer van 1 t/m 6 en slaat het op in game.dice_roll
        game.dice_roll = random.randrange(1, 7)
        # dit zet het plaatje van de die naar hetgeen wat gegooid is
        self.image = "assets\img\die{}.png".format(game.dice_roll)
    def draw(self,game):
        # dit tekent de die
        button.draw_img(game, game.width - 130, game.height - 70, 64, 64, "", 0, self.image, (0,0,0), self.onclick)

dice = Dice()

def draw(game):
    pygame.draw.rect(game.screen,(204,204,204),(600,0,game.width * 0.9,game.height * 1))
    dice.draw(game)
    pygame.draw.rect(game.screen,(255,0,0),(32,32,110,game.height * 0.8))
    pygame.draw.rect(game.screen,(255,239,0),(162,32,110,game.height * 0.8))
    pygame.draw.rect(game.screen,(52,163,253),(292,32,110,game.height * 0.8))
    pygame.draw.rect(game.screen,(24,208,27),(422,32,110,game.height * 0.8))
    font = pygame.font.Font(None, 48)
    label_1 = font.render("Start", 1, (255,255,255))
    size = font.size("Start")
    game.screen.blit(label_1,(45, game.height * 0.9))
    game.screen.blit(label_1,(175, game.height * 0.9))
    game.screen.blit(label_1,(305, game.height * 0.9))
    game.screen.blit(label_1,(435, game.height * 0.9))<|endoftext|>


<commit_before>import re
import time

from gevent.lock import Semaphore
from datetime import datetime
from holster.emitter import Priority

from disco.types.guild import VerificationLevel

from rowboat.plugins import RowboatPlugin as Plugin
from rowboat.redis import rdb
from rowboat.plugins.modlog import Actions
from rowboat.util.leakybucket import LeakyBucket
from rowboat.types.plugin import PluginConfig
from rowboat.types import SlottedModel, DictField, Field
from rowboat.models.user import Infraction

# TODO:
#  - detect mention spam
#  - detect normal spam

INVITE_LINK_RE = re.compile(r'(discord.me|discord.gg)(?:/#)?(?:/invite)?/([a-z0-9\-]+)')
URL_RE = re.compile(r'(https?://[^\s]+)')
BAD_WORDS_RE = re.compile('({})'.format('|'.join(open('data/badwords.txt', 'r').read())))

class SubConfig(SlottedModel):
    max_messages_check = Field(bool, desc='Whether to limit the max number of messages during a time period.', default=False)
    max_messages_count = Field(int, desc='The max number of messages per interval this user can send')
    max_messages_interval = Field(int, desc='The interval (in seconds) for the max messages count')

    max_mentions_check = Field(bool, desc='Whether to limit the max number of mentions during a time period.', default=False)
    max_mentions_count = Field(int, desc='The max number of mentions per interval')
    max_mentions_interval = Field(int, desc='The interval (in seconds) for the max mentions count')

    ban_duration = Field(int, desc='Duration of ban (0 == perma) in seconds', default=604800)

    max_mentions_per_message = Field(int, desc='The max number of mentions a single message can have')

    advanced_heuristics = Field(bool, desc='Enable advanced spam detection', default=False)

    def get_max_messages_bucket(self, guild_id):
        if not self.max_messages_check:
            return None

        if not hasattr(self, '_max_messages_bucket'):
            return LeakyBucket(rdb, 'b:msgs:{}:{}'.format(guild_id, '{}'), self.max_messages_count, self.max_messages_interval * 1000)

        return self._max_messages_bucket

    def get_max_mentions_bucket(self, guild_id):
        if not self.max_mentions_check:
            return None

        if not hasattr(self, '_max_mentions_bucket'):
            return LeakyBucket(rdb, 'b:mnts:{}:{}'.format(guild_id, '{}'), self.max_mentions_count, self.max_mentions_interval * 1000)

        return self._max_mentions_bucket

class SpamConfig(PluginConfig):
    roles = DictField(str, SubConfig)
    levels = DictField(int, SubConfig)

    def compute_relevant_rules(self, member, level):
        if self.roles:
            if '*' in self.roles:
                yield self.roles['*']

            for rid in member.roles.keys():
                if rid in self.roles:
                    yield self.roles[rid]
                rname = member.guild.roles.get(rid)
                if rname and rname.name in self.roles:
                    yield self.roles[rname.name]

        if self.levels:
            for lvl in self.levels.keys():
                if lvl <= level:
                    yield self.levels[lvl]

class Violation(Exception):
    def __init__(self, rule, event, member, label, msg, **info):
        self.rule = rule
        self.event = event
        self.member = member
        self.label = label
        self.msg = msg
        self.info = info

@Plugin.with_config(SpamConfig)
class SpamPlugin(Plugin):
    def load(self, ctx):
        super(SpamPlugin, self).load(ctx)
        self.guild_locks = {}

    def violate(self, violation):
        key = 'lv:{e.member.guild_id}:{e.member.id}'.format(e=violation.event)
        last_violated = int(rdb.get(key) or 0)
        rdb.setex('lv:{e.member.guild_id}:{e.member.id}'.format(e=violation.event), int(time.time()), 60)

        if not last_violated > time.time() - 10:
            self.bot.plugins.get('ModLogPlugin').log_action_ext(Actions.SPAM_DEBUG, violation.event, v=violation)
            self.bot.plugins.get('CorePlugin').send_control_message(
                u'Spam detected by {} ({}) in guild {} ({})'.format(
                    violation.member,
                    violation.member.id,
                    violation.event.guild,
                    violation.event.guild.id))

            if violation.rule.ban_duration == 0:
                Infraction.ban(self, violation.event, violation.member, 'Spam Detected')
            else:
                Infraction.tempban(
                    self,
                    violation.event,
                    violation.member,
                    'Spam Detected',
                    violation.rule.ban_duration)

            # TODO: clean messages

    def check_message_simple(self, event, member, rule):
        # First, check the max messages rules
        bucket = rule.get_max_messages_bucket(event.guild.id)
        if bucket:
            if not bucket.check(event.author.id):
                raise Violation(
                    rule,
                    event,
                    member,
                    'MAX_MESSAGES',
                    'Too Many Messages ({} / {}s)'.format(bucket.count(event.author.id), bucket.size(event.author.id)))

        # Next, check max mentions rules
        if rule.max_mentions_per_message and len(event.mentions) > rule.max_mentions_per_message:
            raise Violation(
                rule,
                event,
                member,
                'MAX_MENTIONS_PER_MESSAGE',
                'Too Many Mentions ({} / {})'.format(len(event.mentions), rule.max_mentions_per_message))

        bucket = rule.get_max_mentions_bucket(event.guild.id)
        if bucket:
            if not bucket.check(event.author.id, len(event.mentions)):
                raise Violation(
                    rule,
                    event,
                    member,
                    'MAX_MENTIONS',
                    'Too Many Mentions ({} / {}s)'.format(bucket.count(event.author.id), bucket.size(event.author.id)))

    def check_message_advanced(self, event, member, rule):
        member = event.guild.get_member(event.author)
        user_age = (datetime.utcnow() - member.joined_at).total_seconds()

        # Calculate a risk rating based on the contents of the messages
        msg_risk = 0

        # Mention count
        msg_risk += (len(event.mentions) * 250)

        # Each bad word is 25 points
        msg_risk += (len(BAD_WORDS_RE.findall(event.content)) * 250)

        # Invite Links
        msg_risk += (len(INVITE_LINK_RE.findall(event.content)) * 1000)

        # Regular Links
        msg_risk += (
            len(URL_RE.findall(INVITE_LINK_RE.sub('', event.content))) * 500
        )

        rdb.rpush('spam:scores:{}'.format(event.author.id), msg_risk)
        rdb.expire('spam:scores:{}'.format(event.author.id), 60 * 10)
        scores = rdb.lrange('spam:scores:{}'.format(event.author.id), 0, -1)

        score_sum = sum(map(int, scores))
        expected_score = 0

        if event.guild.verification_level is VerificationLevel.HIGH:
            user_age -= 60 * 10

        print user_age, score_sum
        user_age = 60 * 6

        # Gauge the risk by age
        if user_age < (60 * 1):
            if score_sum >= 1000:
                expected_score = 1000
        elif user_age < (60 * 5):
            if score_sum >= 5000:
                expected_score = 5000
        else:
            if score_sum >= 10000:
                expected_score = 10000

        if expected_score:
            self.bot.plugins.get('CorePlugin').send_control_message(
                u'User {} triggered avanced spam detection in channel {}\n  score: {}\n  expected: {}\n  scores: {}'.format(
                    member,
                    event.channel.mention,
                    score_sum,
                    expected_score,
                    len(scores)
                ))

    @Plugin.listen('MessageCreate', priority=Priority.AFTER)
    def on_message_create(self, event):
        if event.author.id == self.state.me.id:
            return

        # Lineralize events by guild ID to prevent spamming events
        if event.guild.id not in self.guild_locks:
            self.guild_locks[event.guild.id] = Semaphore()
        self.guild_locks[event.guild.id].acquire()

        try:
            member = event.guild.get_member(event.author)
            level = int(self.bot.get_level(event.author))

            for rule in event.config.compute_relevant_rules(member, level):
                if rule.advanced_heuristics:
                    self.check_message_advanced(event, member, rule)
                self.check_message_simple(event, member, rule)
        except Violation as v:
            self.violate(v)
        finally:
            self.guild_locks[event.guild.id].release()
<commit_msg>add punishment type<commit_after>import re
import time

from gevent.lock import Semaphore
from datetime import datetime
from holster.enum import Enum
from holster.emitter import Priority

from disco.types.guild import VerificationLevel

from rowboat.plugins import RowboatPlugin as Plugin
from rowboat.redis import rdb
from rowboat.plugins.modlog import Actions
from rowboat.util.leakybucket import LeakyBucket
from rowboat.types.plugin import PluginConfig
from rowboat.types import SlottedModel, DictField, Field
from rowboat.models.user import Infraction

# TODO:
#  - detect mention spam
#  - detect normal spam

INVITE_LINK_RE = re.compile(r'(discord.me|discord.gg)(?:/#)?(?:/invite)?/([a-z0-9\-]+)')
URL_RE = re.compile(r'(https?://[^\s]+)')
BAD_WORDS_RE = re.compile('({})'.format('|'.join(open('data/badwords.txt', 'r').read())))

PunishmentType = Enum(
    'NONE',
    'MUTE',
    'KICK',
    'TEMPBAN',
    'BAN'
)

class SubConfig(SlottedModel):
    max_messages_check = Field(bool, desc='Whether to limit the max number of messages during a time period.', default=False)
    max_messages_count = Field(int, desc='The max number of messages per interval this user can send')
    max_messages_interval = Field(int, desc='The interval (in seconds) for the max messages count')

    max_mentions_check = Field(bool, desc='Whether to limit the max number of mentions during a time period.', default=False)
    max_mentions_count = Field(int, desc='The max number of mentions per interval')
    max_mentions_interval = Field(int, desc='The interval (in seconds) for the max mentions count')

    max_mentions_per_message = Field(int, desc='The max number of mentions a single message can have')

    advanced_heuristics = Field(bool, desc='Enable advanced spam detection', default=False)

    punishment = Field(PunishmentType, default=PunishmentType.NONE)
    punishment_duration = Field(int, default=300)

    def get_max_messages_bucket(self, guild_id):
        if not self.max_messages_check:
            return None

        if not hasattr(self, '_max_messages_bucket'):
            return LeakyBucket(rdb, 'b:msgs:{}:{}'.format(guild_id, '{}'), self.max_messages_count, self.max_messages_interval * 1000)

        return self._max_messages_bucket

    def get_max_mentions_bucket(self, guild_id):
        if not self.max_mentions_check:
            return None

        if not hasattr(self, '_max_mentions_bucket'):
            return LeakyBucket(rdb, 'b:mnts:{}:{}'.format(guild_id, '{}'), self.max_mentions_count, self.max_mentions_interval * 1000)

        return self._max_mentions_bucket

class SpamConfig(PluginConfig):
    roles = DictField(str, SubConfig)
    levels = DictField(int, SubConfig)

    def compute_relevant_rules(self, member, level):
        if self.roles:
            if '*' in self.roles:
                yield self.roles['*']

            for rid in member.roles.keys():
                if rid in self.roles:
                    yield self.roles[rid]
                rname = member.guild.roles.get(rid)
                if rname and rname.name in self.roles:
                    yield self.roles[rname.name]

        if self.levels:
            for lvl in self.levels.keys():
                if lvl <= level:
                    yield self.levels[lvl]

class Violation(Exception):
    def __init__(self, rule, event, member, label, msg, **info):
        self.rule = rule
        self.event = event
        self.member = member
        self.label = label
        self.msg = msg
        self.info = info

@Plugin.with_config(SpamConfig)
class SpamPlugin(Plugin):
    def load(self, ctx):
        super(SpamPlugin, self).load(ctx)
        self.guild_locks = {}

    def violate(self, violation):
        key = 'lv:{e.member.guild_id}:{e.member.id}'.format(e=violation.event)
        last_violated = int(rdb.get(key) or 0)
        rdb.setex('lv:{e.member.guild_id}:{e.member.id}'.format(e=violation.event), int(time.time()), 60)

        if not last_violated > time.time() - 10:
            self.bot.plugins.get('ModLogPlugin').log_action_ext(Actions.SPAM_DEBUG, violation.event, v=violation)
            self.bot.plugins.get('CorePlugin').send_control_message(
                u'Spam detected by {} ({}) in guild {} ({})'.format(
                    violation.member,
                    violation.member.id,
                    violation.event.guild,
                    violation.event.guild.id))

            if violation.rule.punishment is PunishmentType.MUTE:
                pass
            elif violation.rule.punishment is PunishmentType.KICK:
                Infraction.kick(
                    self,
                    violation.event,
                    violation.member,
                    'Spam Detected')
            elif violation.rule.punishment is PunishmentType.TEMPBAN:
                Infraction.tempban(
                    self,
                    violation.event,
                    violation.member,
                    'Spam Detected',
                    violation.rule.punishment_duration)
            else:
                Infraction.ban(
                    self,
                    violation.event,
                    violation.member,
                    'Spam Detected')
            # TODO: clean messages

    def check_message_simple(self, event, member, rule):
        # First, check the max messages rules
        bucket = rule.get_max_messages_bucket(event.guild.id)
        if bucket:
            if not bucket.check(event.author.id):
                raise Violation(
                    rule,
                    event,
                    member,
                    'MAX_MESSAGES',
                    'Too Many Messages ({} / {}s)'.format(bucket.count(event.author.id), bucket.size(event.author.id)))

        # Next, check max mentions rules
        if rule.max_mentions_per_message and len(event.mentions) > rule.max_mentions_per_message:
            raise Violation(
                rule,
                event,
                member,
                'MAX_MENTIONS_PER_MESSAGE',
                'Too Many Mentions ({} / {})'.format(len(event.mentions), rule.max_mentions_per_message))

        bucket = rule.get_max_mentions_bucket(event.guild.id)
        if bucket:
            if not bucket.check(event.author.id, len(event.mentions)):
                raise Violation(
                    rule,
                    event,
                    member,
                    'MAX_MENTIONS',
                    'Too Many Mentions ({} / {}s)'.format(bucket.count(event.author.id), bucket.size(event.author.id)))

    def check_message_advanced(self, event, member, rule):
        member = event.guild.get_member(event.author)
        user_age = (datetime.utcnow() - member.joined_at).total_seconds()

        # Calculate a risk rating based on the contents of the messages
        msg_risk = 0

        # Mention count
        msg_risk += (len(event.mentions) * 250)

        # Each bad word is 25 points
        msg_risk += (len(BAD_WORDS_RE.findall(event.content)) * 250)

        # Invite Links
        msg_risk += (len(INVITE_LINK_RE.findall(event.content)) * 1000)

        # Regular Links
        msg_risk += (
            len(URL_RE.findall(INVITE_LINK_RE.sub('', event.content))) * 500
        )

        rdb.rpush('spam:scores:{}'.format(event.author.id), msg_risk)
        rdb.expire('spam:scores:{}'.format(event.author.id), 60 * 10)
        scores = rdb.lrange('spam:scores:{}'.format(event.author.id), 0, -1)

        score_sum = sum(map(int, scores))
        expected_score = 0

        if event.guild.verification_level is VerificationLevel.HIGH:
            user_age -= 60 * 10

        print user_age, score_sum
        user_age = 60 * 6

        # Gauge the risk by age
        if user_age < (60 * 1):
            if score_sum >= 1000:
                expected_score = 1000
        elif user_age < (60 * 5):
            if score_sum >= 5000:
                expected_score = 5000
        else:
            if score_sum >= 10000:
                expected_score = 10000

        if expected_score:
            self.bot.plugins.get('CorePlugin').send_control_message(
                u'User {} triggered avanced spam detection in channel {}\n  score: {}\n  expected: {}\n  scores: {}'.format(
                    member,
                    event.channel.mention,
                    score_sum,
                    expected_score,
                    len(scores)
                ))

    @Plugin.listen('MessageCreate', priority=Priority.AFTER)
    def on_message_create(self, event):
        if event.author.id == self.state.me.id:
            return

        # Lineralize events by guild ID to prevent spamming events
        if event.guild.id not in self.guild_locks:
            self.guild_locks[event.guild.id] = Semaphore()
        self.guild_locks[event.guild.id].acquire()

        try:
            member = event.guild.get_member(event.author)
            level = int(self.bot.get_level(event.author))

            for rule in event.config.compute_relevant_rules(member, level):
                if rule.advanced_heuristics:
                    self.check_message_advanced(event, member, rule)
                self.check_message_simple(event, member, rule)
        except Violation as v:
            self.violate(v)
        finally:
            self.guild_locks[event.guild.id].release()
<|endoftext|>


<commit_before>"""
This file is part of pyS5p

https://github.com/rmvanhees/pys5p.git

Implementation of the Tukey's biweight algorithm

Copyright (c) 2019-2020 SRON - Netherlands Institute for Space Research
   All Rights Reserved

License:  BSD-3-Clause
"""
import warnings

import numpy as np

# ----- local functions -------------------------
def __biweight(data, spread=False):
    """
    Calculate biweight median and spread, for whole dataset
    """
    mask = np.isfinite(data)
    if np.all(~mask):
        if spread:
            return (np.nan, 0.)

        return np.nan

    # calculate biweight median
    data_median = np.nanmedian(data)
    delta = data - data_median
    delta_median = np.nanmedian(np.abs(delta))
    if delta_median == 0.:
        if spread:
            return (data_median, 0.)

        return data_median

    wmx = np.clip(1 - (delta / (6 * delta_median)) ** 2, 0, None) ** 2
    biweight_median = data_median + np.nansum(wmx * delta) / np.nansum(wmx)
    if not spread:
        return biweight_median

    # calculate biweight spread
    umn = np.clip((delta / (9 * delta_median)) ** 2, None, 1)
    buff = np.nansum(delta ** 2 * (1 - umn) ** 4)
    buff /= np.nansum((1 - umn) * (1 - 5 * umn)) ** 2
    biweight_spread = np.sqrt(np.sum(mask) * buff)

    return (biweight_median, biweight_spread)

def __biweight_axis(data, axis, spread=False):
    """
    Calculate only biweight median, along a given axis
    """
    mask = np.isfinite(data)
    all_nan = np.all(np.isnan(data), axis=axis)

    # calculate biweight median
    data_median = np.nanmedian(data, axis=axis, keepdims=True)
    delta = data - data_median
    delta_median = np.nanmedian(np.abs(delta), axis=axis, keepdims=True)
    _mm = delta_median != 0.
    delta_median[~_mm] = np.nan

    wmx = np.clip(1 - (delta / (6 * delta_median)) ** 2, 0, None) ** 2
    biweight_median = np.squeeze(data_median)
    _mm = np.squeeze(_mm) & ~all_nan
    biweight_median[_mm] += \
        np.nansum(wmx * delta, axis=axis)[_mm] / np.nansum(wmx, axis=axis)[_mm]
    if not spread:
        return biweight_median

    # calculate biweight spread
    umn = np.clip((delta / (9 * delta_median)) ** 2, None, 1)
    buff = np.nansum(delta ** 2 * (1 - umn) ** 4, axis=axis)[_mm]
    buff /= np.nansum((1 - umn) * (1 - 5 * umn), axis=axis)[_mm] ** 2
    biweight_spread = np.full(biweight_median.shape, np.nan)
    biweight_spread[_mm] = np.sqrt(np.sum(mask, axis=axis)[_mm] * buff)

    return (biweight_median, biweight_spread)

# ----- main function -------------------------
def biweight(data, axis=None, spread=False):
    """
    Calculate Tukey's biweight.
    Implementation based on Eqn. 7.6 and 7.7 in the SWIR OCAL ATBD.

    Parameters
    ----------
    data   :   array_like
       input array
    axis   :   int, optional
       axis along which the biweight medians are computed.
        - Note that axis will be ignored when data is a 1-D array.
    spread :   bool, optional
       if True, then return also the biweight spread.

    Returns
    -------
    out    :   ndarray
       biweight median and biweight spread if function argument "spread" is True
    """
    if axis is None or data.ndim == 1:
        return __biweight(data, spread)

    # only a single axis!
    if not isinstance(axis, int):
        raise TypeError('axis not an integer')
    if not 0 <= axis < data.ndim:
        raise ValueError('axis out-of-range')

    with warnings.catch_warnings():
        warnings.filterwarnings('ignore', r'All-NaN (slice|axis) encountered')
        return __biweight_axis(data, axis, spread)
<commit_msg>biweight.py: tried to reduce the memory usage of __biweight_axis, seems not to work. However, the code is faster for data without NaNs<commit_after>"""
This file is part of pyS5p

https://github.com/rmvanhees/pys5p.git

Implementation of the Tukey's biweight algorithm

Copyright (c) 2019-2020 SRON - Netherlands Institute for Space Research
   All Rights Reserved

License:  BSD-3-Clause
"""
import warnings

import numpy as np

# from memory_profiler import profile

# ----- local functions -------------------------
def __biweight(data, spread=False):
    """
    Calculate biweight parameters for the whole dataset
    """
    # calculate median and median absolute difference
    if np.isnan(data).any():
        biweight_median = np.nanmedian(data)
        delta = data - biweight_median
        delta_median = np.nanmedian(np.abs(delta))
    else:
        biweight_median = np.median(data)
        delta = data - biweight_median
        delta_median = np.median(np.abs(delta))
    if delta_median == 0.:
        if spread:
            return (biweight_median, 0.)
        return biweight_median

    # calculate biweight median
    wmx = np.clip(1 - (delta / (6 * delta_median)) ** 2, 0, None) ** 2
    biweight_median += np.nansum(wmx * delta) / np.nansum(wmx)
    if not spread:
        return biweight_median

    # calculate biweight variance
    umn = np.clip((delta / (9 * delta_median)) ** 2, None, 1)
    biweight_var = np.nansum(delta ** 2 * (1 - umn) ** 4)
    biweight_var /= np.nansum((1 - umn) * (1 - 5 * umn)) ** 2
    biweight_var *= np.sum(np.isfinite(data))

    return (biweight_median, np.sqrt(biweight_var))

# @profile(precision=1)
def __biweight_axis(data, axis, spread=False):
    """
    Calculate biweight parameters, along a given axis

    Notes
    -----
    Data should not contain any invalid value
    """
    # calculate median and median absolute difference
    if np.isnan(data).any():
        all_nan = np.isnan(data).all(axis=axis)
        biweight_median = np.nanmedian(data, axis=axis, keepdims=True)
        delta = data - biweight_median
        delta_median = np.nanmedian(np.abs(delta), axis=axis, keepdims=True)
        _mm = delta_median != 0.
        delta_median[~_mm] = np.nan
        _mm = np.squeeze(_mm) & ~all_nan
    else:
        biweight_median = np.median(data, axis=axis, keepdims=True)
        delta = data - biweight_median
        delta_median = np.median(np.abs(delta), axis=axis, keepdims=True)
        _mm = delta_median != 0.
        delta_median[~_mm] = np.nan
        _mm = np.squeeze(_mm)
    biweight_median = np.squeeze(biweight_median)

    # calculate biweight median
    wmx = np.clip(1 - (delta / (6 * delta_median)) ** 2, 0, None) ** 2
    biweight_median[_mm] += \
        np.nansum(wmx * delta, axis=axis)[_mm] / np.nansum(wmx, axis=axis)[_mm]
    del wmx
    if not spread:
        return biweight_median

    # calculate biweight variance
    umn = np.clip((delta / (9 * delta_median)) ** 2, None, 1)
    del delta_median
    biweight_var = np.nansum(delta ** 2 * (1 - umn) ** 4, axis=axis)
    del delta
    biweight_var[_mm] /= \
        np.nansum((1 - umn) * (1 - 5 * umn), axis=axis)[_mm] ** 2
    biweight_var[_mm] *= np.sum(np.isfinite(data), axis=axis)[_mm]

    return (biweight_median, np.sqrt(biweight_var))

# ----- main function -------------------------
def biweight2(data, axis=None, spread=False):
    """
    Calculate Tukey's biweight.
    Implementation based on Eqn. 7.6 and 7.7 in the SWIR OCAL ATBD.

    Parameters
    ----------
    data   :   array_like
       input array
    axis   :   int, optional
       axis along which the biweight medians are computed.
        - Note that axis will be ignored when data is a 1-D array.
    spread :   bool, optional
       if True, then return also the biweight spread.

    Returns
    -------
    out    :   ndarray or tuple
       biweight median and biweight spread if parameter "spread" is True
    """
    if axis is None or data.ndim == 1:
        mask = np.isfinite(data)
        if np.all(~mask):
            if spread:
                return (np.nan, 0.)
            return np.nan
        return __biweight(data, spread)

    # only a single axis!
    if not isinstance(axis, int):
        raise TypeError('axis not an integer')
    if not 0 <= axis < data.ndim:
        raise ValueError('axis out-of-range')

    with warnings.catch_warnings():
        warnings.filterwarnings('ignore', r'All-NaN (slice|axis) encountered')
        return __biweight_axis(data, axis, spread)
<|endoftext|>


<commit_before>from __future__ import division
from __future__ import print_function

import plotly.graph_objs as go

import cea.plots.optimization
from cea.plots.variable_naming import NAMING, COLOR

__author__ = "Daren Thomas"
__copyright__ = "Copyright 2019, Architecture and Building Systems - ETH Zurich"
__credits__ = ["Jimeno A. Fonseca", "Daren Thomas"]
__license__ = "MIT"
__version__ = "0.1"
__maintainer__ = "Daren Thomas"
__email__ = "cea@arch.ethz.ch"
__status__ = "Production"

class AnnualCostsPlot(cea.plots.optimization.GenerationPlotBase):
    """Implement the "CAPEX vs. OPEX of centralized system in generation X" plot"""
    name = "Annual costs"
    expected_parameters = {
        'generation': 'plots-optimization:generation',
        'scenario-name': 'general:scenario-name',
    }

    def __init__(self, project, parameters, cache):
        super(AnnualCostsPlot, self).__init__(project, parameters, cache)
        self.analysis_fields = ["Capex_a_sys_connected_USD",
                                "Capex_a_sys_disconnected_USD",
                                "Opex_a_sys_connected_USD",
                                "Opex_a_sys_disconnected_USD"
                                ]
        self.input_files = [(self.locator.get_optimization_generation_total_performance, [self.generation])]

    @property
    def title(self):
        return "Annual Costs for generation #%s" % self.generation

    @property
    def output_path(self):
        return self.locator.get_timeseries_plots_file(
            'gen{generation}_annualized_costs'.format(generation=self.generation),
            self.category_name)

    @property
    def layout(self):
        return go.Layout(barmode='relative',
                         yaxis=dict(title='Annualized cost [USD$(2015)/year]'))

    def calc_graph(self):
        self.multi_criteria = False  # TODO: add capabilities to plot muticriteria in this plot too
        data = self.process_generation_total_performance()
        graph = []
        for field in self.analysis_fields:
            y = data[field].values
            flag_for_unused_technologies = all(v == 0 for v in y)
            if not flag_for_unused_technologies:
                trace = go.Bar(x=data['individual_name'], y=y, name=NAMING[field],
                               marker=dict(color=COLOR[field]))
                graph.append(trace)

        return graph

def main():
    """Test this plot"""
    import cea.config
    import cea.plots.cache
    config = cea.config.Configuration()
    cache = cea.plots.cache.NullPlotCache()
    locator = cea.inputlocator.InputLocator(config.scenario)
    # cache = cea.plots.cache.PlotCache(config.project)
    AnnualCostsPlot(config.project,
                    {'buildings': None,
                     'scenario-name': config.scenario_name,
                     'generation': config.plots_optimization.generation},
                    cache).plot(auto_open=True)

if __name__ == '__main__':
    main()
<commit_msg>Update annual_costs.py<commit_after>from __future__ import division
from __future__ import print_function

import plotly.graph_objs as go

import cea.plots.optimization
from cea.plots.variable_naming import NAMING, COLOR

__author__ = "Daren Thomas"
__copyright__ = "Copyright 2019, Architecture and Building Systems - ETH Zurich"
__credits__ = ["Jimeno A. Fonseca", "Daren Thomas"]
__license__ = "MIT"
__version__ = "0.1"
__maintainer__ = "Daren Thomas"
__email__ = "cea@arch.ethz.ch"
__status__ = "Production"

class AnnualCostsPlot(cea.plots.optimization.GenerationPlotBase):
    """Implement the "CAPEX vs. OPEX of centralized system in generation X" plot"""
    name = "Annual costs"
    expected_parameters = {
        'generation': 'plots-optimization:generation',
        'scenario-name': 'general:scenario-name',
    }

    def __init__(self, project, parameters, cache):
        super(AnnualCostsPlot, self).__init__(project, parameters, cache)
        self.analysis_fields = ["Capex_a_sys_connected_USD",
                                "Capex_a_sys_disconnected_USD",
                                "Opex_a_sys_connected_USD",
                                "Opex_a_sys_disconnected_USD"
                                ]
        self.input_files = [(self.locator.get_optimization_generation_total_performance, [self.generation])]

    @property
    def title(self):
        return "Annual Costs for generation #%s" % self.generation

    @property
    def output_path(self):
        return self.locator.get_timeseries_plots_file(
            'gen{generation}_annualized_costs'.format(generation=self.generation),
            self.category_name)

    @property
    def layout(self):
        return go.Layout(barmode='relative',
                         yaxis=dict(title='Annualized cost [USD$(2015)/year]'))

    def calc_graph(self):
        self.multi_criteria = False  # TODO: add capabilities to plot muticriteria in this plot too
        data = self.process_generation_total_performance_pareto()
        graph = []
        for field in self.analysis_fields:
            y = data[field].values
            flag_for_unused_technologies = all(v == 0 for v in y)
            if not flag_for_unused_technologies:
                trace = go.Bar(x=data['individual_name'], y=y, name=NAMING[field],
                               marker=dict(color=COLOR[field]))
                graph.append(trace)

        return graph

def main():
    """Test this plot"""
    import cea.config
    import cea.plots.cache
    config = cea.config.Configuration()
    cache = cea.plots.cache.NullPlotCache()
    locator = cea.inputlocator.InputLocator(config.scenario)
    # cache = cea.plots.cache.PlotCache(config.project)
    AnnualCostsPlot(config.project,
                    {'buildings': None,
                     'scenario-name': config.scenario_name,
                     'generation': config.plots_optimization.generation},
                    cache).plot(auto_open=True)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Copyright 2017 Ricequant, Inc
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import sys
import six
import os
import yaml
import simplejson as json
import datetime
import logbook
import locale
import codecs
import shutil
from pprint import pformat

from . import RqAttrDict, logger
from .exception import patch_user_exc
from .logger import user_log, user_system_log, system_log, std_log, user_std_handler
from ..const import RUN_TYPE, PERSIST_MODE
from ..utils.i18n import gettext as _, localization
from ..utils.dict_func import deep_update
from ..utils.py2 import to_utf8
from ..mod.utils import mod_config_value_parse

rqalpha_path = "~/.rqalpha"

def load_config(config_path, loader=yaml.Loader):
    if config_path is None:
        return {}
    if not os.path.exists(config_path):
        raise RuntimeError(_(u"config.yml not found in {config_path}").format(config_path))
    if ".json" in config_path:
        with codecs.open(config_path, encoding="utf-8") as f:
            json_config = f.read()
        config = json.loads(json_config)
    else:
        with codecs.open(config_path, encoding="utf-8") as stream:
            config = yaml.load(stream, loader)
    return config

def dump_config(config_path, config, dumper=yaml.Dumper):
    with codecs.open(config_path, mode='w', encoding='utf-8') as stream:
        stream.write(to_utf8(yaml.dump(config, Dumper=dumper)))

def load_mod_config(config_path, loader=yaml.Loader):
    mod_config = load_config(config_path, loader)
    if mod_config is None or "mod" not in mod_config:
        import os
        os.remove(config_path)
        config_path = get_mod_config_path()
        return load_mod_config(config_path, loader)
    else:
        # FIXME 因为增加了 sys_accounts 所以临时增加了对应 mod_config.yml 的更新
        # TODO 这里应该更改为 mod_config.yml 和 mod_config_template.yml 的同步检查。
        if 'sys_accounts' not in mod_config['mod']:
            mod_config['mod']['sys_accounts'] = {
                'enabled': True
            }
            dump_config(config_path, mod_config)
        return mod_config

def get_mod_config_path(generate=False):
    mod_config_path = os.path.abspath(os.path.expanduser(os.path.join(rqalpha_path, "mod_config.yml")))
    mod_template_path = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../mod_config_template.yml"))
    if not os.path.exists(mod_config_path):
        if generate:
            dir_path = os.path.dirname(mod_config_path)
            if not os.path.exists(dir_path):
                os.makedirs(dir_path)
            shutil.copy(mod_template_path, mod_config_path)
            return mod_config_path
        else:
            return mod_template_path
    return mod_config_path

def get_user_config_path(config_path=None):
    if config_path is None:
        if os.path.exists(os.path.abspath(os.path.join(os.getcwd(), "config.yml"))):
            return os.path.abspath(os.path.join(os.getcwd(), "config.yml"))
        if os.path.exists(os.path.abspath(os.path.join(os.getcwd(), "config.json"))):
            return os.path.abspath(os.path.join(os.getcwd(), "config.json"))
        return None
    else:
        if not os.path.exists(config_path):
            system_log.error(_("config path: {config_path} does not exist.").format(config_path=config_path))
            return None
        return config_path

# def config_version_verify(config, config_path):
#     default_config_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../default_config.yml")
#     default_config = load_config(default_config_path, verify_version=False)
#     config_version = config.get("version", None)
#     if config_version != default_config["version"]:
#         back_config_file_path = config_path + "." + datetime.datetime.now().date().strftime("%Y%m%d") + ".bak"
#         shutil.move(config_path, back_config_file_path)
#         shutil.copy(default_config_path, config_path)
#
#         system_log.warning(_(u"""
# Your current config file {config_file_path} is too old and may cause RQAlpha running error.
# RQAlpha has replaced the config file with the newest one.
# the backup config file has been saved in {back_config_file_path}.
#         """).format(config_file_path=config_path, back_config_file_path=back_config_file_path))
#         config = default_config
#     return config

def set_locale(lc):
    # FIXME: It should depends on the system and locale config
    try:
        locale.setlocale(locale.LC_ALL, "en_US.UTF-8")
        locale.setlocale(locale.LC_CTYPE, "en_US.UTF-8")
        os.environ['TZ'] = 'Asia/Shanghai'
    except Exception as e:
        if os.name != 'nt':
            raise
    localization.set_locale([lc])

def parse_config(config_args, config_path=None, click_type=False, source_code=None, user_funcs=None, verify_config=True):
    mod_configs = config_args.pop("mod_configs", [])
    for cfg, value in mod_configs:
        key = "mod__{}".format(cfg.replace(".", "__"))
        config_args[key] = mod_config_value_parse(value)

    set_locale(config_args.get("extra__locale", None))

    user_config_path = get_user_config_path(config_path)
    mod_config_path = get_mod_config_path()

    # load default config from rqalpha
    config = load_config(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../default_config.yml"))
    # load mod config
    mod_config = load_mod_config(mod_config_path)
    deep_update(mod_config, config)
    # load user config
    user_config = load_config(user_config_path)
    deep_update(user_config, config)

    # use config_args to extend config
    if click_type:
        for key, value in six.iteritems(config_args):
            if config_args[key] is not None:
                keys = key.split("__")
                keys.reverse()
                sub_config = config[keys.pop()]
                while True:
                    if len(keys) == 0:
                        break
                    k = keys.pop()
                    if len(keys) == 0:
                        sub_config[k] = value
                    else:
                        sub_config = sub_config[k]
    else:
        deep_update(config_args, config)

    # config from user code
    if user_funcs is None:
        config = parse_user_config_from_code(config, source_code)
    config = RqAttrDict(config)

    base_config = config.base
    extra_config = config.extra

    set_locale(extra_config.locale)

    if isinstance(base_config.start_date, six.string_types):
        base_config.start_date = datetime.datetime.strptime(base_config.start_date, "%Y-%m-%d")
    if isinstance(base_config.start_date, datetime.datetime):
        base_config.start_date = base_config.start_date.date()
    if isinstance(base_config.end_date, six.string_types):
        base_config.end_date = datetime.datetime.strptime(base_config.end_date, "%Y-%m-%d")
    if isinstance(base_config.end_date, datetime.datetime):
        base_config.end_date = base_config.end_date.date()

    if base_config.data_bundle_path is None:
        base_config.data_bundle_path = os.path.join(os.path.expanduser(rqalpha_path), "bundle")

    base_config.data_bundle_path = os.path.abspath(base_config.data_bundle_path)

    if not os.path.exists(base_config.data_bundle_path):
        raise RuntimeError(
            _(u"data bundle not found in {bundle_path}. Run `rqalpha update_bundle` to download data bundle.").format(
                bundle_path=base_config.data_bundle_path))

    base_config.run_type = parse_run_type(base_config.run_type)
    base_config.accounts = parse_accounts(base_config.accounts)
    base_config.persist_mode = parse_persist_mode(base_config.persist_mode)

    if extra_config.context_vars:
        if isinstance(extra_config.context_vars, six.string_types):
            extra_config.context_vars = json.loads(to_utf8(extra_config.context_vars))

    system_log.level = getattr(logbook, extra_config.log_level.upper(), logbook.NOTSET)
    std_log.level = getattr(logbook, extra_config.log_level.upper(), logbook.NOTSET)
    user_log.level = getattr(logbook, extra_config.log_level.upper(), logbook.NOTSET)
    user_system_log.level = getattr(logbook, extra_config.log_level.upper(), logbook.NOTSET)

    if base_config.frequency == "1d":
        logger.DATETIME_FORMAT = "%Y-%m-%d"

    system_log.debug("\n" + pformat(config.convert_to_dict()))

    return config

def parse_user_config_from_code(config, source_code=None):
    try:
        if source_code is None:
            with codecs.open(config["base"]["strategy_file"], encoding="utf-8") as f:
                source_code = f.read()

        scope = {}

        code = compile(source_code, config["base"]["strategy_file"], 'exec')
        six.exec_(code, scope)

        __config__ = scope.get("__config__", {})

        for sub_key, sub_dict in six.iteritems(__config__):
            if sub_key not in config["whitelist"]:
                continue
            deep_update(sub_dict, config[sub_key])
    except Exception as e:
        system_log.error(_(u"in parse_user_config, exception: {e}").format(e=e))
    finally:
        return config

def parse_accounts(accounts):
    a = {}
    for account_type, starting_cash in six.iteritems(accounts):
        if starting_cash is None:
            continue
        starting_cash = float(starting_cash)
        a[account_type.upper()] = starting_cash
    if len(a) == 0:
        raise RuntimeError(_(u"None account type has been selected."))
    return a

def parse_run_type(rt_str):
    assert isinstance(rt_str, six.string_types)
    if rt_str == "b":
        return RUN_TYPE.BACKTEST
    elif rt_str == "p":
        return RUN_TYPE.PAPER_TRADING
    elif rt_str == 'r':
        return RUN_TYPE.LIVE_TRADING
    else:
        raise NotImplementedError

def parse_persist_mode(persist_mode):
    assert isinstance(persist_mode, six.string_types)
    if persist_mode == 'real_time':
        return PERSIST_MODE.REAL_TIME
    elif persist_mode == 'on_crash':
        return PERSIST_MODE.ON_CRASH
    else:
        raise RuntimeError(_(u"unknown persist mode: {persist_mode}").format(persist_mode=persist_mode))
<commit_msg>bug fixed<commit_after># -*- coding: utf-8 -*-
#
# Copyright 2017 Ricequant, Inc
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import sys
import six
import os
import yaml
import simplejson as json
import datetime
import logbook
import locale
import codecs
import shutil
from pprint import pformat

from . import RqAttrDict, logger
from .exception import patch_user_exc
from .logger import user_log, user_system_log, system_log, std_log, user_std_handler
from ..const import RUN_TYPE, PERSIST_MODE
from ..utils.i18n import gettext as _, localization
from ..utils.dict_func import deep_update
from ..utils.py2 import to_utf8
from ..mod.utils import mod_config_value_parse

rqalpha_path = "~/.rqalpha"

def load_config(config_path, loader=yaml.Loader):
    if config_path is None:
        return {}
    if not os.path.exists(config_path):
        raise RuntimeError(_(u"config.yml not found in {config_path}").format(config_path))
    if ".json" in config_path:
        with codecs.open(config_path, encoding="utf-8") as f:
            json_config = f.read()
        config = json.loads(json_config)
    else:
        with codecs.open(config_path, encoding="utf-8") as stream:
            config = yaml.load(stream, loader)
    return config

def dump_config(config_path, config, dumper=yaml.Dumper):
    with codecs.open(config_path, mode='w', encoding='utf-8') as stream:
        stream.write(to_utf8(yaml.dump(config, Dumper=dumper)))

def load_mod_config(config_path, loader=yaml.Loader):
    mod_config = load_config(config_path, loader)
    if mod_config is None or "mod" not in mod_config:
        import os
        os.remove(config_path)
        config_path = get_mod_config_path()
        return load_mod_config(config_path, loader)
    else:
        # FIXME 因为增加了 sys_accounts 所以临时增加了对应 mod_config.yml 的更新
        # TODO 这里应该更改为 mod_config.yml 和 mod_config_template.yml 的同步检查。
        if 'sys_accounts' not in mod_config['mod']:
            mod_config['mod']['sys_accounts'] = {
                'enabled': True
            }
            dump_config(config_path, mod_config)
        return mod_config

def get_mod_config_path(generate=False):
    mod_config_path = os.path.abspath(os.path.expanduser(os.path.join(rqalpha_path, "mod_config.yml")))
    mod_template_path = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../mod_config_template.yml"))
    if not os.path.exists(mod_config_path):
        if generate:
            dir_path = os.path.dirname(mod_config_path)
            if not os.path.exists(dir_path):
                os.makedirs(dir_path)
            shutil.copy(mod_template_path, mod_config_path)
            return mod_config_path
        else:
            return mod_template_path
    return mod_config_path

def get_user_config_path(config_path=None):
    if config_path is None:
        if os.path.exists(os.path.abspath(os.path.join(os.getcwd(), "config.yml"))):
            return os.path.abspath(os.path.join(os.getcwd(), "config.yml"))
        if os.path.exists(os.path.abspath(os.path.join(os.getcwd(), "config.json"))):
            return os.path.abspath(os.path.join(os.getcwd(), "config.json"))
        return None
    else:
        if not os.path.exists(config_path):
            system_log.error(_("config path: {config_path} does not exist.").format(config_path=config_path))
            return None
        return config_path

# def config_version_verify(config, config_path):
#     default_config_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../default_config.yml")
#     default_config = load_config(default_config_path, verify_version=False)
#     config_version = config.get("version", None)
#     if config_version != default_config["version"]:
#         back_config_file_path = config_path + "." + datetime.datetime.now().date().strftime("%Y%m%d") + ".bak"
#         shutil.move(config_path, back_config_file_path)
#         shutil.copy(default_config_path, config_path)
#
#         system_log.warning(_(u"""
# Your current config file {config_file_path} is too old and may cause RQAlpha running error.
# RQAlpha has replaced the config file with the newest one.
# the backup config file has been saved in {back_config_file_path}.
#         """).format(config_file_path=config_path, back_config_file_path=back_config_file_path))
#         config = default_config
#     return config

def set_locale(lc):
    # FIXME: It should depends on the system and locale config
    try:
        locale.setlocale(locale.LC_ALL, "en_US.UTF-8")
        locale.setlocale(locale.LC_CTYPE, "en_US.UTF-8")
        os.environ['TZ'] = 'Asia/Shanghai'
    except Exception as e:
        if os.name != 'nt':
            raise
    localization.set_locale([lc])

def parse_config(config_args, config_path=None, click_type=False, source_code=None, user_funcs=None):
    mod_configs = config_args.pop("mod_configs", [])
    for cfg, value in mod_configs:
        key = "mod__{}".format(cfg.replace(".", "__"))
        config_args[key] = mod_config_value_parse(value)

    set_locale(config_args.get("extra__locale", None))

    user_config_path = get_user_config_path(config_path)
    mod_config_path = get_mod_config_path()

    # load default config from rqalpha
    config = load_config(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../default_config.yml"))
    # load mod config
    mod_config = load_mod_config(mod_config_path)
    deep_update(mod_config, config)
    # load user config
    user_config = load_config(user_config_path)
    deep_update(user_config, config)

    # use config_args to extend config
    if click_type:
        for key, value in six.iteritems(config_args):
            if config_args[key] is not None:
                keys = key.split("__")
                keys.reverse()
                sub_config = config[keys.pop()]
                while True:
                    if len(keys) == 0:
                        break
                    k = keys.pop()
                    if len(keys) == 0:
                        sub_config[k] = value
                    else:
                        sub_config = sub_config[k]
    else:
        deep_update(config_args, config)

    # config from user code
    if user_funcs is not None:
        config = parse_user_config_from_code(config, source_code)
    config = RqAttrDict(config)

    base_config = config.base
    extra_config = config.extra

    set_locale(extra_config.locale)

    if isinstance(base_config.start_date, six.string_types):
        base_config.start_date = datetime.datetime.strptime(base_config.start_date, "%Y-%m-%d")
    if isinstance(base_config.start_date, datetime.datetime):
        base_config.start_date = base_config.start_date.date()
    if isinstance(base_config.end_date, six.string_types):
        base_config.end_date = datetime.datetime.strptime(base_config.end_date, "%Y-%m-%d")
    if isinstance(base_config.end_date, datetime.datetime):
        base_config.end_date = base_config.end_date.date()

    if base_config.data_bundle_path is None:
        base_config.data_bundle_path = os.path.join(os.path.expanduser(rqalpha_path), "bundle")

    base_config.data_bundle_path = os.path.abspath(base_config.data_bundle_path)

    if not os.path.exists(base_config.data_bundle_path):
        raise RuntimeError(
            _(u"data bundle not found in {bundle_path}. Run `rqalpha update_bundle` to download data bundle.").format(
                bundle_path=base_config.data_bundle_path))

    base_config.run_type = parse_run_type(base_config.run_type)
    base_config.accounts = parse_accounts(base_config.accounts)
    base_config.persist_mode = parse_persist_mode(base_config.persist_mode)

    if extra_config.context_vars:
        if isinstance(extra_config.context_vars, six.string_types):
            extra_config.context_vars = json.loads(to_utf8(extra_config.context_vars))

    system_log.level = getattr(logbook, extra_config.log_level.upper(), logbook.NOTSET)
    std_log.level = getattr(logbook, extra_config.log_level.upper(), logbook.NOTSET)
    user_log.level = getattr(logbook, extra_config.log_level.upper(), logbook.NOTSET)
    user_system_log.level = getattr(logbook, extra_config.log_level.upper(), logbook.NOTSET)

    if base_config.frequency == "1d":
        logger.DATETIME_FORMAT = "%Y-%m-%d"

    system_log.debug("\n" + pformat(config.convert_to_dict()))

    return config

def parse_user_config_from_code(config, source_code=None):
    try:
        if source_code is None:
            with codecs.open(config["base"]["strategy_file"], encoding="utf-8") as f:
                source_code = f.read()

        scope = {}

        code = compile(source_code, config["base"]["strategy_file"], 'exec')
        six.exec_(code, scope)

        __config__ = scope.get("__config__", {})

        for sub_key, sub_dict in six.iteritems(__config__):
            if sub_key not in config["whitelist"]:
                continue
            deep_update(sub_dict, config[sub_key])
    except Exception as e:
        system_log.error(_(u"in parse_user_config, exception: {e}").format(e=e))
    finally:
        return config

def parse_accounts(accounts):
    a = {}
    for account_type, starting_cash in six.iteritems(accounts):
        if starting_cash is None:
            continue
        starting_cash = float(starting_cash)
        a[account_type.upper()] = starting_cash
    if len(a) == 0:
        raise RuntimeError(_(u"None account type has been selected."))
    return a

def parse_run_type(rt_str):
    assert isinstance(rt_str, six.string_types)
    if rt_str == "b":
        return RUN_TYPE.BACKTEST
    elif rt_str == "p":
        return RUN_TYPE.PAPER_TRADING
    elif rt_str == 'r':
        return RUN_TYPE.LIVE_TRADING
    else:
        raise NotImplementedError

def parse_persist_mode(persist_mode):
    assert isinstance(persist_mode, six.string_types)
    if persist_mode == 'real_time':
        return PERSIST_MODE.REAL_TIME
    elif persist_mode == 'on_crash':
        return PERSIST_MODE.ON_CRASH
    else:
        raise RuntimeError(_(u"unknown persist mode: {persist_mode}").format(persist_mode=persist_mode))
<|endoftext|>


<commit_before>"""
Show a Pareto curve plot for individuals in a given generation.
"""
from __future__ import division
from __future__ import print_function

import pandas as pd
import plotly.graph_objs as go

import cea.plots.optimization
from cea.plots.variable_naming import NAMING

__author__ = "Bhargava Srepathi"
__copyright__ = "Copyright 2018, Architecture and Building Systems - ETH Zurich"
__credits__ = ["Bhargava Srepathi", "Daren Thomas"]
__license__ = "MIT"
__version__ = "0.1"
__maintainer__ = "Daren Thomas"
__email__ = "cea@arch.ethz.ch"
__status__ = "Production"

class ParetoCurveForOneGenerationPlot(cea.plots.optimization.GenerationPlotBase):
    """Show a pareto curve for a single generation"""
    name = "Pareto curve of costs, emissions and primary energy"
    expected_parameters = {
        'generation': 'plots-optimization:generation',
        'normalization': 'plots-optimization:normalization',
        'multicriteria': 'plots-optimization:multicriteria',
        'scenario-name': 'general:scenario-name',
    }

    def __init__(self, project, parameters, cache):
        super(ParetoCurveForOneGenerationPlot, self).__init__(project, parameters, cache)
        self.analysis_fields = ['individual_name',
                                'TAC_sys_USD',
                                'GHG_sys_tonCO2',
                                'PEN_sys_MJoil',
                                'Capex_total_sys_USD',
                                'Opex_a_sys_USD']
        self.objectives = ['TAC_sys_USD', 'GHG_sys_tonCO2', 'PEN_sys_MJoil']
        self.normalization = self.parameters['normalization']
        self.input_files = [(self.locator.get_optimization_generation_total_performance, [self.generation])]
        self.multi_criteria = self.parameters['multicriteria']
        self.titlex, self.titley, self.titlez = self.calc_titles()

    def calc_titles(self):
        if self.normalization == "gross floor area":
            titlex = 'Total annualized costs [USD$(2015)/m2.yr]'
            titley = 'GHG emissions [ton CO2-eq/m2.yr]'
            titlez = 'Primary Energy
[MJ Oil-eq/m2.yr]'
        elif self.normalization == "net floor area":
            titlex = 'Total annualized costs [USD$(2015)/m2.yr]'
            titley = 'GHG emissions [ton CO2-eq/m2.yr]'
            titlez = 'Primary Energy
[MJ Oil-eq/m2.yr]'
        elif self.normalization == "air conditioned floor area":
            titlex = 'Total annualized costs [USD$(2015)/m2.yr]'
            titley = 'GHG emissions [ton CO2-eq/m2.yr]'
            titlez = 'Primary Energy
[MJ Oil-eq/m2.yr]'
        elif self.normalization == "building occupancy":
            titlex = 'Total annualized costs [USD$(2015)/pax.yr]'
            titley = 'GHG emissions [ton CO2-eq/pax.yr]'
            titlez = 'Primary Energy
[MJ Oil-eq/pax.yr]'
        else:
            titlex = 'Total annualized costs [USD$(2015)/yr]'
            titley = 'GHG emissions [ton CO2-eq/yr]'
            titlez = 'Primary Energy
[MJ Oil-eq/yr]'

        return titlex, titley, titlez

    @property
    def layout(self):
        return go.Layout(legend=dict(orientation="v", x=0.8, y=0.95),
                         xaxis=dict(title=self.titlex),
                         yaxis=dict(title=self.titley))

    @property
    def title(self):
        if self.normalization != "none":
            return "Pareto curve for generation {generation} normalized to {normalized}".format(generation=self.generation, normalized=self.normalization)
        else:
            return "Pareto curve for generation {generation}".format(generation=self.generation)

    @property
    def output_path(self):
        return self.locator.get_timeseries_plots_file('gen{generation}_pareto_curve'.format(generation=self.generation),
                                                      self.category_name)

    def calc_graph(self):
        graph = []
        # #PUT THE HALL OF FAME INSIDE
        # data_HOF = self.process_generation_total_performance_halloffame()
        # data_HOF = self.normalize_data(data_HOF, self.normalization, self.objectives)
        # xs_HOF = data_HOF[self.objectives[0]].values
        # ys_HOF = data_HOF[self.objectives[1]].values
        # individual_names = data_HOF['individual_name'].values
        # trace = go.Scattergl(x=xs_HOF, y=ys_HOF, mode='markers', name='Hall of fame', text=individual_names,
        #                    marker=dict(size='12', color='grey_light'))
        # graph.append(trace)

        # PUT THE PARETO CURVE INSIDE
        data = self.process_generation_total_performance_pareto()
        data = self.normalize_data(data, self.normalization, self.objectives)
        xs = data[self.objectives[0]].values
        ys = data[self.objectives[1]].values
        zs = data[self.objectives[2]].values

        individual_names = data['individual_name'].values

        trace = go.Scattergl(x=xs, y=ys, mode='markers', name='Pareto curve', text=individual_names,
                           marker=dict(size='12', color=zs,
                                       colorbar=go.ColorBar(title=self.titlez, titleside='bottom'),
                                       colorscale='Jet', showscale=True, opacity=0.8))
        graph.append(trace)

        # This includes the points of the multicriteria assessment in here
        if self.multi_criteria:
            # Insert scatter points of MCDA assessment.
            final_dataframe = calc_final_dataframe(data)
            xs = final_dataframe[self.objectives[0]].values
            ys = final_dataframe[self.objectives[1]].values
            name = final_dataframe["Attribute"].values
            trace = go.Scattergl(x=xs, y=ys, mode='markers', name="Selected by Multi-criteria", text=name,
                               marker=dict(size='20', color='white', line=dict(
                                   color='black',
                                   width=2)))
            graph.append(trace)
        return graph

    # def calc_table(self):
    #     final_dataframe = calc_final_dataframe(self.process_generation_total_performance())
    #
    #     # transform data into currency
    #     for column in final_dataframe.columns:
    #         if '_USD' in column or '_MJoil' in column or '_tonCO2' in column:
    #             final_dataframe[column] = final_dataframe[column].apply(lambda x: '{:20,.2f}'.format(x))
    #
    #     columns = ["Attribute"] + self.analysis_fields
    #     values = []
    #     for field in columns:
    #         if field in ["Attribute", "individual_name"]:
    #             values.append(final_dataframe[field].values)
    #         else:
    #             values.append(final_dataframe[field].values)
    #
    #     columns = ["Attribute"] + [NAMING[field] for field in self.analysis_fields]
    #     table_df = pd.DataFrame({cn: cv for cn, cv in zip(columns, values)}, columns=columns)
    #     return table_df

def calc_final_dataframe(individual_data):
    least_annualized_cost = individual_data.loc[
        individual_data["TAC_rank"] < 2]  # less than two because in the case there are two individuals MCDA calculates 1.5
    least_emissions = individual_data.loc[individual_data["GHG_rank"] < 2]
    least_primaryenergy = individual_data.loc[individual_data["PEN_rank"] < 2]
    user_defined_mcda = individual_data.loc[individual_data["user_MCDA_rank"] < 2]
    # do a check in the case more individuals had the same ranking.
    if least_annualized_cost.shape[0] > 1:
        individual = str(least_annualized_cost["individual_name"].values)
        least_annualized_cost = least_annualized_cost.reset_index(drop=True)
        least_annualized_cost = least_annualized_cost[:1]
        least_annualized_cost["System option"] = individual
    if least_emissions.shape[0] > 1:
        individual = str(least_emissions["individual_name"].values)
        least_emissions = least_emissions.reset_index(drop=True)
        least_emissions = least_emissions.iloc[0].T
        least_emissions["System option"] = individual
    if least_primaryenergy.shape[0] > 1:
        individual = str(least_primaryenergy["individual_name"].values)
        least_primaryenergy = least_primaryenergy.reset_index(drop=True)
        least_primaryenergy = least_primaryenergy.iloc[0].T
        least_primaryenergy["System option"] = individual
    if user_defined_mcda.shape[0] > 1:
        individual = str(user_defined_mcda["individual_name"].values)
        user_defined_mcda = user_defined_mcda.reset_index(drop=True)
        user_defined_mcda = user_defined_mcda.iloc[0].T
        user_defined_mcda["System option"] = individual
    # Now extend all dataframes
    final_dataframe = least_annualized_cost.append(least_emissions).append(least_primaryenergy).append(
        user_defined_mcda)
    final_dataframe.reset_index(drop=True, inplace=True)
    final_dataframe["Attribute"] = ["least annualized costs", "least emissions", "least primary energy",
                                    "user defined MCDA"]
    return final_dataframe

def main():
    """Test this plot"""
    import cea.config
    import cea.plots.cache
    config = cea.config.Configuration()
    cache = cea.plots.cache.NullPlotCache()
    ParetoCurveForOneGenerationPlot(config.project,
                                    {'scenario-name': config.scenario_name,
                                     'generation': config.plots_optimization.generation,
                                     'multicriteria': config.plots_optimization.multicriteria,
                                     'normalization': config.plots_optimization.normalization},
                                    cache).plot(auto_open=True)
if __name__ == '__main__':
    main()
<commit_msg>Update pareto_curve.py<commit_after>"""
Show a Pareto curve plot for individuals in a given generation.
"""
from __future__ import division
from __future__ import print_function

import pandas as pd
import plotly.graph_objs as go

import cea.plots.optimization
from cea.plots.variable_naming import NAMING

__author__ = "Bhargava Srepathi"
__copyright__ = "Copyright 2018, Architecture and Building Systems - ETH Zurich"
__credits__ = ["Bhargava Srepathi", "Daren Thomas"]
__license__ = "MIT"
__version__ = "0.1"
__maintainer__ = "Daren Thomas"
__email__ = "cea@arch.ethz.ch"
__status__ = "Production"

class ParetoCurveForOneGenerationPlot(cea.plots.optimization.GenerationPlotBase):
    """Show a pareto curve for a single generation"""
    name = "Pareto curve of costs, emissions and primary energy"
    expected_parameters = {
        'generation': 'plots-optimization:generation',
        'normalization': 'plots-optimization:normalization',
        'multicriteria': 'plots-optimization:multicriteria',
        'scenario-name': 'general:scenario-name',
    }

    def __init__(self, project, parameters, cache):
        super(ParetoCurveForOneGenerationPlot, self).__init__(project, parameters, cache)
        self.analysis_fields = ['individual_name',
                                'TAC_sys_USD',
                                'GHG_sys_tonCO2',
                                'PEN_sys_MJoil',
                                'Capex_total_sys_USD',
                                'Opex_a_sys_USD']
        self.objectives = ['TAC_sys_USD', 'GHG_sys_tonCO2', 'PEN_sys_MJoil']
        self.normalization = self.parameters['normalization']
        self.input_files = [(self.locator.get_optimization_generation_total_performance_pareto, [self.generation])]
        self.multi_criteria = self.parameters['multicriteria']
        self.titlex, self.titley, self.titlez = self.calc_titles()

    def calc_titles(self):
        if self.normalization == "gross floor area":
            titlex = 'Total annualized costs [USD$(2015)/m2.yr]'
            titley = 'GHG emissions [ton CO2-eq/m2.yr]'
            titlez = 'Primary Energy
[MJ Oil-eq/m2.yr]'
        elif self.normalization == "net floor area":
            titlex = 'Total annualized costs [USD$(2015)/m2.yr]'
            titley = 'GHG emissions [ton CO2-eq/m2.yr]'
            titlez = 'Primary Energy
[MJ Oil-eq/m2.yr]'
        elif self.normalization == "air conditioned floor area":
            titlex = 'Total annualized costs [USD$(2015)/m2.yr]'
            titley = 'GHG emissions [ton CO2-eq/m2.yr]'
            titlez = 'Primary Energy
[MJ Oil-eq/m2.yr]'
        elif self.normalization == "building occupancy":
            titlex = 'Total annualized costs [USD$(2015)/pax.yr]'
            titley = 'GHG emissions [ton CO2-eq/pax.yr]'
            titlez = 'Primary Energy
[MJ Oil-eq/pax.yr]'
        else:
            titlex = 'Total annualized costs [USD$(2015)/yr]'
            titley = 'GHG emissions [ton CO2-eq/yr]'
            titlez = 'Primary Energy
[MJ Oil-eq/yr]'

        return titlex, titley, titlez

    @property
    def layout(self):
        return go.Layout(legend=dict(orientation="v", x=0.8, y=0.95),
                         xaxis=dict(title=self.titlex),
                         yaxis=dict(title=self.titley))

    @property
    def title(self):
        if self.normalization != "none":
            return "Pareto curve for generation {generation} normalized to {normalized}".format(generation=self.generation, normalized=self.normalization)
        else:
            return "Pareto curve for generation {generation}".format(generation=self.generation)

    @property
    def output_path(self):
        return self.locator.get_timeseries_plots_file('gen{generation}_pareto_curve'.format(generation=self.generation),
                                                      self.category_name)

    def calc_graph(self):
        graph = []
        # #PUT THE HALL OF FAME INSIDE
        # data_HOF = self.process_generation_total_performance_halloffame()
        # data_HOF = self.normalize_data(data_HOF, self.normalization, self.objectives)
        # xs_HOF = data_HOF[self.objectives[0]].values
        # ys_HOF = data_HOF[self.objectives[1]].values
        # individual_names = data_HOF['individual_name'].values
        # trace = go.Scattergl(x=xs_HOF, y=ys_HOF, mode='markers', name='Hall of fame', text=individual_names,
        #                    marker=dict(size='12', color='grey_light'))
        # graph.append(trace)

        # PUT THE PARETO CURVE INSIDE
        data = self.process_generation_total_performance_pareto()
        data = self.normalize_data(data, self.normalization, self.objectives)
        xs = data[self.objectives[0]].values
        ys = data[self.objectives[1]].values
        zs = data[self.objectives[2]].values

        individual_names = data['individual_name'].values

        trace = go.Scattergl(x=xs, y=ys, mode='markers', name='Pareto curve', text=individual_names,
                           marker=dict(size='12', color=zs,
                                       colorbar=go.ColorBar(title=self.titlez, titleside='bottom'),
                                       colorscale='Jet', showscale=True, opacity=0.8))
        graph.append(trace)

        # This includes the points of the multicriteria assessment in here
        if self.multi_criteria:
            # Insert scatter points of MCDA assessment.
            final_dataframe = calc_final_dataframe(data)
            xs = final_dataframe[self.objectives[0]].values
            ys = final_dataframe[self.objectives[1]].values
            name = final_dataframe["Attribute"].values
            trace = go.Scattergl(x=xs, y=ys, mode='markers', name="Selected by Multi-criteria", text=name,
                               marker=dict(size='20', color='white', line=dict(
                                   color='black',
                                   width=2)))
            graph.append(trace)
        return graph

    # def calc_table(self):
    #     final_dataframe = calc_final_dataframe(self.process_generation_total_performance())
    #
    #     # transform data into currency
    #     for column in final_dataframe.columns:
    #         if '_USD' in column or '_MJoil' in column or '_tonCO2' in column:
    #             final_dataframe[column] = final_dataframe[column].apply(lambda x: '{:20,.2f}'.format(x))
    #
    #     columns = ["Attribute"] + self.analysis_fields
    #     values = []
    #     for field in columns:
    #         if field in ["Attribute", "individual_name"]:
    #             values.append(final_dataframe[field].values)
    #         else:
    #             values.append(final_dataframe[field].values)
    #
    #     columns = ["Attribute"] + [NAMING[field] for field in self.analysis_fields]
    #     table_df = pd.DataFrame({cn: cv for cn, cv in zip(columns, values)}, columns=columns)
    #     return table_df

def calc_final_dataframe(individual_data):
    least_annualized_cost = individual_data.loc[
        individual_data["TAC_rank"] < 2]  # less than two because in the case there are two individuals MCDA calculates 1.5
    least_emissions = individual_data.loc[individual_data["GHG_rank"] < 2]
    least_primaryenergy = individual_data.loc[individual_data["PEN_rank"] < 2]
    user_defined_mcda = individual_data.loc[individual_data["user_MCDA_rank"] < 2]
    # do a check in the case more individuals had the same ranking.
    if least_annualized_cost.shape[0] > 1:
        individual = str(least_annualized_cost["individual_name"].values)
        least_annualized_cost = least_annualized_cost.reset_index(drop=True)
        least_annualized_cost = least_annualized_cost[:1]
        least_annualized_cost["System option"] = individual
    if least_emissions.shape[0] > 1:
        individual = str(least_emissions["individual_name"].values)
        least_emissions = least_emissions.reset_index(drop=True)
        least_emissions = least_emissions.iloc[0].T
        least_emissions["System option"] = individual
    if least_primaryenergy.shape[0] > 1:
        individual = str(least_primaryenergy["individual_name"].values)
        least_primaryenergy = least_primaryenergy.reset_index(drop=True)
        least_primaryenergy = least_primaryenergy.iloc[0].T
        least_primaryenergy["System option"] = individual
    if user_defined_mcda.shape[0] > 1:
        individual = str(user_defined_mcda["individual_name"].values)
        user_defined_mcda = user_defined_mcda.reset_index(drop=True)
        user_defined_mcda = user_defined_mcda.iloc[0].T
        user_defined_mcda["System option"] = individual
    # Now extend all dataframes
    final_dataframe = least_annualized_cost.append(least_emissions).append(least_primaryenergy).append(
        user_defined_mcda)
    final_dataframe.reset_index(drop=True, inplace=True)
    final_dataframe["Attribute"] = ["least annualized costs", "least emissions", "least primary energy",
                                    "user defined MCDA"]
    return final_dataframe

def main():
    """Test this plot"""
    import cea.config
    import cea.plots.cache
    config = cea.config.Configuration()
    cache = cea.plots.cache.NullPlotCache()
    ParetoCurveForOneGenerationPlot(config.project,
                                    {'scenario-name': config.scenario_name,
                                     'generation': config.plots_optimization.generation,
                                     'multicriteria': config.plots_optimization.multicriteria,
                                     'normalization': config.plots_optimization.normalization},
                                    cache).plot(auto_open=True)
if __name__ == '__main__':
    main()
<|endoftext|>