<commit_before>import os
import re
import sys
import json
import socket
import asyncio
from aiohttp import web, WSMsgType

app = web.Application()
rooms = {}

class Room:
	# Color names taken from https://en.wikipedia.org/wiki/Template:Monopoly_board_layout
	# including the variant that the cheapest ones are Indigo, not SaddleBrown
	property_data = """
	Indigo/White: 60/60 Old Kent, Whitechapel
	SkyBlue: 100/120 Angel Islington, Euston, Pentonville
	DarkOrchid/White: 140/160 Pall Mall, Whitehall, Northumberland
	Orange: 180/200 Bow, Marlborough, Vine
	Red: 220/240 Strand, Fleet, Trafalgar
	Yellow: 260/280 Leicester, Coventry, Piccadilly
	Green/White: 300/320 Regent, Oxford, Bond
	Blue/White: 350/400 Park Lane, Mayfair
	Black/White: 200/200 King's Cross, Marylebone, Fenchurch, Liverpool
	White: 150/150 Electric, Water Works
	"""

	def __init__(self, id):
		if os.environ.get("WS_KEEPALIVE"):
			asyncio.ensure_future(self.keepalive())
		self.clients = []
		self.proporder = []
		self.funds = 1500 # Everyone's initial spendable money
		self.id = id; rooms[self.id] = self # floop
		print("Creating new room %s [%d rooms]" % (self.id, len(rooms)))
		self.dying = None # Set to true when we run out of clients
		self.all_done = False

		# Preprocess the property data into a more useful form.
		self.properties = {}
		for group in self.property_data.splitlines():
			group = group.strip()
			if not group: continue
			color, price1, price2, names = re.match("([A-Za-z/]+): ([0-9]+)/([0-9]+) (.*)", group).groups()
			names = names.split(", ")
			if "/" in color: color, fg = color.split("/")
			else: fg = "Black"
			for name in names:
				self.proporder.append(name)
				self.properties[name] = {"facevalue": int(price1), "color": color, "fg": fg}
			# Alter the price of the last one (the top one of the group)
			self.properties[name]["facevalue"] = int(price2)

	def send_users(self):
		"""Notify all clients of updated public user data"""
		users = {ws.username: self.funds for ws in self.clients if ws.username}
		for prop in self.properties.values():
			if "bidder" in prop:
				users[prop["bidder"]] -= prop["highbid"]
		info = {"type": "users",
			"done_count": sum(ws.done for ws in self.clients if ws.username),
			"all_done": self.all_done,
			"users": sorted(users.items()),
		}
		for ws in self.clients:
			ws.funds = info["funds"] = users.get(ws.username, self.funds)
			info["done"] = ws.done
			ws.send_json(info)

	async def ws_login(self, ws, name, **xtra):
		if ws.username: return None
		ws.username = str(name)[:32]
		ws.send_json({"type": "login", "name": ws.username})
		self.send_users()

	async def ws_done(self, ws, **xtra):
		ws.done = True
		for cli in self.clients:
			if not ws.done: break
		else:
			# Everyone's done. Mode switch!
			self.all_done = True
		self.send_users()

	async def ws_bid(self, ws, name, value, **xtra):
		if self.all_done: return None
		prop = self.properties[name]
		value = int(value)
		minbid = prop["facevalue"] if "bidder" not in prop else prop["highbid"] + 10
		if value < minbid: return None
		if value > ws.funds: return None
		prop["highbid"] = value
		prop["bidder"] = ws.username
		for cli in self.clients: cli.done = False
		self.send_users()
		return {"type": "property", "name": name, "data": prop}

	async def keepalive(self):
		"""Keep the websockets alive

		In some environments, we lose any inactive websockets. So keep telling
		them about users - that's safe, at least.
		"""
		while True:
			await asyncio.sleep(30)
			self.send_users()

	async def websocket(self, ws, login_data):
		ws.username = None; ws.done = False
		self.dying = None # Whenever anyone joins, even if they disconnect fast, reset the death timer.
		self.clients.append(ws)
		await self.ws_login(ws, **login_data)
		print("New socket in %s (now %d)" % (self.id, len(self.clients)))

		ws.send_json({"type": "properties", "data": self.properties, "order": self.proporder});
		async for msg in ws:
			# Ignore non-JSON messages
			if msg.type != WSMsgType.TEXT: continue
			try: msg = json.loads(msg.data)
			except ValueError: continue
			print("MESSAGE", msg)
			if "type" not in msg or "data" not in msg: continue
			f = getattr(self, "ws_" + msg["type"], None)
			if not f: continue
			try:
				resp = await f(ws, **msg["data"])
			except Exception as e:
				print("Exception in ws handler:")
				print(e)
				continue
			if resp is None: continue
			for client in self.clients:
				client.send_json(resp)

		self.clients.remove(ws)
		await ws.close()
		print("Socket gone from %s (%d left)" % (self.id, len(self.clients)))
		if not self.clients:
			asyncio.ensure_future(self.die())
		return ws

	async def die(self):
		"""Destroy this room after a revive delay"""
		sentinel = object()
		self.dying = sentinel
		print("Room %s dying" % self.id)
		await asyncio.sleep(60)
		if self.dying is sentinel:
			# If it's not sentinel, we got revived. Maybe the
			# other connection is in dying mode, maybe not;
			# either way, we aren't in charge of death.
			assert not self.clients
			del rooms[self.id]
			print("Room %s dead - %d rooms left" % (self.id, len(rooms)))
		else:
			if self.dying:
				print("Room %s revived-but-still-dying" % self.id)
			else:
				print("Room %s revived" % self.id)

def route(url):
	def deco(f):
		app.router.add_get(url, f)
		return f
	return deco

@route("/")
async def home(req):
	with open("build/index.html") as f:
		return web.Response(text=f.read(), content_type="text/html")

@route("/ws")
async def websocket(req):
	ws = web.WebSocketResponse()
	await ws.prepare(req)
	async for msg in ws:
		if msg.type != WSMsgType.TEXT: continue
		try:
			msg = json.loads(msg.data)
			if msg["type"] != "login": continue
			room = msg["data"]["room"][:32]
			if room: break
		except (ValueError, KeyError, TypeError):
			# Any parsing error, just wait for another message
			continue
	else:
		# Something went wrong with the handshake. Kick
		# the client and let them reconnect.
		await ws.close()
		return ws
	if room not in rooms: Room(room)
	return await rooms[room].websocket(ws, msg["data"])

# After all the custom routes, handle everything else by loading static files.
app.router.add_static("/", path="build", name="static")

# Lifted from appension
async def serve_http(loop, port, sock=None):
	if sock:
		srv = await loop.create_server(app.make_handler(), sock=sock)
	else:
		srv = await loop.create_server(app.make_handler(), "0.0.0.0", port)
		sock = srv.sockets[0]
	print("Listening on %s:%s" % sock.getsockname(), file=sys.stderr)

def run(port=8080, sock=None):
	loop = asyncio.get_event_loop()
	loop.run_until_complete(serve_http(loop, port, sock))
	# TODO: Announce that we're "ready" in whatever way
	try: loop.run_forever()
	except KeyboardInterrupt: pass

if __name__ == '__main__':
	# Look for a socket provided by systemd
	sock = None
	try:
		pid = int(os.environ.get("LISTEN_PID", ""))
		fd_count = int(os.environ.get("LISTEN_FDS", ""))
	except ValueError:
		pid = fd_count = 0
	if pid == os.getpid() and fd_count >= 1:
		# The PID matches - we've been given at least one socket.
		# The sd_listen_fds docs say that they should start at FD 3.
		sock = socket.socket(fileno=3)
		print("Got %d socket(s)" % fd_count, file=sys.stderr)
	run(port=int(os.environ.get("PORT", "8080")), sock=sock)
<commit_msg>Reorder properties to group them by winner<commit_after>import os
import re
import sys
import json
import socket
import asyncio
from aiohttp import web, WSMsgType

app = web.Application()
rooms = {}

class Room:
	# Color names taken from https://en.wikipedia.org/wiki/Template:Monopoly_board_layout
	# including the variant that the cheapest ones are Indigo, not SaddleBrown
	property_data = """
	Indigo/White: 60/60 Old Kent, Whitechapel
	SkyBlue: 100/120 Angel Islington, Euston, Pentonville
	DarkOrchid/White: 140/160 Pall Mall, Whitehall, Northumberland
	Orange: 180/200 Bow, Marlborough, Vine
	Red: 220/240 Strand, Fleet, Trafalgar
	Yellow: 260/280 Leicester, Coventry, Piccadilly
	Green/White: 300/320 Regent, Oxford, Bond
	Blue/White: 350/400 Park Lane, Mayfair
	Black/White: 200/200 King's Cross, Marylebone, Fenchurch, Liverpool
	White: 150/150 Electric, Water Works
	"""

	def __init__(self, id):
		if os.environ.get("WS_KEEPALIVE"):
			asyncio.ensure_future(self.keepalive())
		self.clients = []
		self.proporder = []
		self.funds = 1500 # Everyone's initial spendable money
		self.id = id; rooms[self.id] = self # floop
		print("Creating new room %s [%d rooms]" % (self.id, len(rooms)))
		self.dying = None # Set to true when we run out of clients
		self.all_done = False

		# Preprocess the property data into a more useful form.
		self.properties = {}
		for group in self.property_data.splitlines():
			group = group.strip()
			if not group: continue
			color, price1, price2, names = re.match("([A-Za-z/]+): ([0-9]+)/([0-9]+) (.*)", group).groups()
			names = names.split(", ")
			if "/" in color: color, fg = color.split("/")
			else: fg = "Black"
			for name in names:
				self.proporder.append(name)
				self.properties[name] = {"facevalue": int(price1), "color": color, "fg": fg}
			# Alter the price of the last one (the top one of the group)
			self.properties[name]["facevalue"] = int(price2)

	def send_users(self):
		"""Notify all clients of updated public user data"""
		users = {ws.username: self.funds for ws in self.clients if ws.username}
		for prop in self.properties.values():
			if "bidder" in prop:
				users[prop["bidder"]] -= prop["highbid"]
		info = {"type": "users",
			"done_count": sum(ws.done for ws in self.clients if ws.username),
			"all_done": self.all_done,
			"users": sorted(users.items()),
		}
		for ws in self.clients:
			ws.funds = info["funds"] = users.get(ws.username, self.funds)
			info["done"] = ws.done
			ws.send_json(info)

	async def ws_login(self, ws, name, **xtra):
		if ws.username: return None
		ws.username = str(name)[:32]
		ws.send_json({"type": "login", "name": ws.username})
		self.send_users()

	async def ws_done(self, ws, **xtra):
		ws.done = True
		for cli in self.clients:
			if not cli.done:
				self.send_users()
				return
		# Everyone's done. Mode switch!
		self.all_done = True
		people = {cli.username:i for i, cli in enumerate(self.clients) if cli.username}
		self.proporder.sort(key=lambda prop: people.get(self.properties[prop].get("bidder"), -1))
		self.send_users()
		return {"type": "properties", "data": self.properties, "order": self.proporder};

	async def ws_bid(self, ws, name, value, **xtra):
		if self.all_done: return None
		prop = self.properties[name]
		value = int(value)
		minbid = prop["facevalue"] if "bidder" not in prop else prop["highbid"] + 10
		if value < minbid: return None
		if value > ws.funds: return None
		prop["highbid"] = value
		prop["bidder"] = ws.username
		for cli in self.clients: cli.done = False
		self.send_users()
		return {"type": "property", "name": name, "data": prop}

	async def keepalive(self):
		"""Keep the websockets alive

		In some environments, we lose any inactive websockets. So keep telling
		them about users - that's safe, at least.
		"""
		while True:
			await asyncio.sleep(30)
			self.send_users()

	async def websocket(self, ws, login_data):
		ws.username = None; ws.done = False
		self.dying = None # Whenever anyone joins, even if they disconnect fast, reset the death timer.
		self.clients.append(ws)
		await self.ws_login(ws, **login_data)
		print("New socket in %s (now %d)" % (self.id, len(self.clients)))

		ws.send_json({"type": "properties", "data": self.properties, "order": self.proporder});
		async for msg in ws:
			# Ignore non-JSON messages
			if msg.type != WSMsgType.TEXT: continue
			try: msg = json.loads(msg.data)
			except ValueError: continue
			print("MESSAGE", msg)
			if "type" not in msg or "data" not in msg: continue
			f = getattr(self, "ws_" + msg["type"], None)
			if not f: continue
			try:
				resp = await f(ws, **msg["data"])
			except Exception as e:
				print("Exception in ws handler:")
				print(e)
				continue
			if resp is None: continue
			for client in self.clients:
				client.send_json(resp)

		self.clients.remove(ws)
		await ws.close()
		print("Socket gone from %s (%d left)" % (self.id, len(self.clients)))
		if not self.clients:
			asyncio.ensure_future(self.die())
		return ws

	async def die(self):
		"""Destroy this room after a revive delay"""
		sentinel = object()
		self.dying = sentinel
		print("Room %s dying" % self.id)
		await asyncio.sleep(60)
		if self.dying is sentinel:
			# If it's not sentinel, we got revived. Maybe the
			# other connection is in dying mode, maybe not;
			# either way, we aren't in charge of death.
			assert not self.clients
			del rooms[self.id]
			print("Room %s dead - %d rooms left" % (self.id, len(rooms)))
		else:
			if self.dying:
				print("Room %s revived-but-still-dying" % self.id)
			else:
				print("Room %s revived" % self.id)

def route(url):
	def deco(f):
		app.router.add_get(url, f)
		return f
	return deco

@route("/")
async def home(req):
	with open("build/index.html") as f:
		return web.Response(text=f.read(), content_type="text/html")

@route("/ws")
async def websocket(req):
	ws = web.WebSocketResponse()
	await ws.prepare(req)
	async for msg in ws:
		if msg.type != WSMsgType.TEXT: continue
		try:
			msg = json.loads(msg.data)
			if msg["type"] != "login": continue
			room = msg["data"]["room"][:32]
			if room: break
		except (ValueError, KeyError, TypeError):
			# Any parsing error, just wait for another message
			continue
	else:
		# Something went wrong with the handshake. Kick
		# the client and let them reconnect.
		await ws.close()
		return ws
	if room not in rooms: Room(room)
	return await rooms[room].websocket(ws, msg["data"])

# After all the custom routes, handle everything else by loading static files.
app.router.add_static("/", path="build", name="static")

# Lifted from appension
async def serve_http(loop, port, sock=None):
	if sock:
		srv = await loop.create_server(app.make_handler(), sock=sock)
	else:
		srv = await loop.create_server(app.make_handler(), "0.0.0.0", port)
		sock = srv.sockets[0]
	print("Listening on %s:%s" % sock.getsockname(), file=sys.stderr)

def run(port=8080, sock=None):
	loop = asyncio.get_event_loop()
	loop.run_until_complete(serve_http(loop, port, sock))
	# TODO: Announce that we're "ready" in whatever way
	try: loop.run_forever()
	except KeyboardInterrupt: pass

if __name__ == '__main__':
	# Look for a socket provided by systemd
	sock = None
	try:
		pid = int(os.environ.get("LISTEN_PID", ""))
		fd_count = int(os.environ.get("LISTEN_FDS", ""))
	except ValueError:
		pid = fd_count = 0
	if pid == os.getpid() and fd_count >= 1:
		# The PID matches - we've been given at least one socket.
		# The sd_listen_fds docs say that they should start at FD 3.
		sock = socket.socket(fileno=3)
		print("Got %d socket(s)" % fd_count, file=sys.stderr)
	run(port=int(os.environ.get("PORT", "8080")), sock=sock)
<|endoftext|>


<commit_before>import pytest
from tensorflow.keras import optimizers
from tensorflow.keras.layers import Dense
from tensorflow.keras.models import Sequential
from tensorflow.keras.activations import relu

from elephas.ml_model import ElephasEstimator, load_ml_estimator, ElephasTransformer, load_ml_transformer
from elephas.ml.adapter import to_data_frame

from pyspark.mllib.evaluation import MulticlassMetrics, RegressionMetrics
from pyspark.ml import Pipeline

from elephas.utils.warnings import ElephasWarning

def test_serialization_transformer(classification_model):
    transformer = ElephasTransformer()
    transformer.set_keras_model_config(classification_model.to_yaml())
    transformer.save("test.h5")
    loaded_model = load_ml_transformer("test.h5")
    assert loaded_model.get_model().to_yaml() == classification_model.to_yaml()

def test_serialization_estimator(classification_model):
    estimator = ElephasEstimator()
    estimator.set_keras_model_config(classification_model.to_yaml())
    estimator.set_loss("categorical_crossentropy")

    estimator.save("test.h5")
    loaded_model = load_ml_estimator("test.h5")
    assert loaded_model.get_model().to_yaml() == classification_model.to_yaml()

def test_spark_ml_model_classification(spark_context, classification_model, mnist_data):
    batch_size = 64
    nb_classes = 10
    epochs = 1

    x_train, y_train, x_test, y_test = mnist_data
    x_train = x_train[:1000]
    y_train = y_train[:1000]
    df = to_data_frame(spark_context, x_train, y_train, categorical=True)
    test_df = to_data_frame(spark_context, x_test, y_test, categorical=True)

    sgd = optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
    sgd_conf = optimizers.serialize(sgd)

    # Initialize Spark ML Estimator
    estimator = ElephasEstimator()
    estimator.set_keras_model_config(classification_model.to_yaml())
    estimator.set_optimizer_config(sgd_conf)
    estimator.set_mode("synchronous")
    estimator.set_loss("categorical_crossentropy")
    estimator.set_metrics(['acc'])
    estimator.set_epochs(epochs)
    estimator.set_batch_size(batch_size)
    estimator.set_validation_split(0.1)
    estimator.set_categorical_labels(True)
    estimator.set_nb_classes(nb_classes)

    # Fitting a model returns a Transformer
    pipeline = Pipeline(stages=[estimator])
    fitted_pipeline = pipeline.fit(df)

    # Evaluate Spark model by evaluating the underlying model
    prediction = fitted_pipeline.transform(test_df)
    pnl = prediction.select("label", "prediction")
    pnl.show(100)

    prediction_and_label = pnl.rdd.map(lambda row: (row.label, row.prediction))
    metrics = MulticlassMetrics(prediction_and_label)
    print(metrics.accuracy)

def test_functional_model(spark_context, classification_model_functional, mnist_data):
    batch_size = 64
    epochs = 1

    x_train, y_train, x_test, y_test = mnist_data
    x_train = x_train[:1000]
    y_train = y_train[:1000]
    df = to_data_frame(spark_context, x_train, y_train, categorical=True)
    test_df = to_data_frame(spark_context, x_test, y_test, categorical=True)

    sgd = optimizers.SGD()
    sgd_conf = optimizers.serialize(sgd)
    estimator = ElephasEstimator()
    estimator.set_keras_model_config(classification_model_functional.to_yaml())
    estimator.set_optimizer_config(sgd_conf)
    estimator.set_mode("synchronous")
    estimator.set_loss("categorical_crossentropy")
    estimator.set_metrics(['acc'])
    estimator.set_epochs(epochs)
    estimator.set_batch_size(batch_size)
    estimator.set_validation_split(0.1)
    estimator.set_categorical_labels(True)
    estimator.set_nb_classes(10)
    pipeline = Pipeline(stages=[estimator])
    fitted_pipeline = pipeline.fit(df)
    prediction = fitted_pipeline.transform(test_df)
    pnl = prediction.select("label", "prediction")
    pnl.show(100)

    prediction_and_label = pnl.rdd.map(lambda row: (row.label, row.prediction))
    metrics = MulticlassMetrics(prediction_and_label)
    print(metrics.accuracy)

def test_regression_model(spark_context, regression_model, boston_housing_dataset):
    batch_size = 64
    epochs = 10

    x_train, y_train, x_test, y_test = boston_housing_dataset
    df = to_data_frame(spark_context, x_train, y_train)
    test_df = to_data_frame(spark_context, x_test, y_test)

    sgd = optimizers.SGD(lr=0.00001)
    sgd_conf = optimizers.serialize(sgd)
    estimator = ElephasEstimator()
    estimator.set_keras_model_config(regression_model.to_yaml())
    estimator.set_optimizer_config(sgd_conf)
    estimator.set_mode("synchronous")
    estimator.set_loss("mae")
    estimator.set_metrics(['mae'])
    estimator.set_epochs(epochs)
    estimator.set_batch_size(batch_size)
    estimator.set_validation_split(0.01)
    estimator.set_categorical_labels(False)

    pipeline = Pipeline(stages=[estimator])
    fitted_pipeline = pipeline.fit(df)
    prediction = fitted_pipeline.transform(test_df)
    pnl = prediction.select("label", "prediction")
    pnl.show(100)

    prediction_and_observations = pnl.rdd.map(lambda row: (row.label, row.prediction))
    metrics = RegressionMetrics(prediction_and_observations)
    print(metrics.r2)

def test_set_cols_deprecated(spark_context, regression_model, boston_housing_dataset):
    with pytest.deprecated_call():
        batch_size = 64
        epochs = 10

        x_train, y_train, x_test, y_test = boston_housing_dataset
        df = to_data_frame(spark_context, x_train, y_train)
        df = df.withColumnRenamed('features', 'scaled_features')
        df = df.withColumnRenamed('label', 'ground_truth')
        test_df = to_data_frame(spark_context, x_test, y_test)
        test_df = test_df.withColumnRenamed('features', 'scaled_features')
        test_df = test_df.withColumnRenamed('label', 'ground_truth')

        sgd = optimizers.SGD(lr=0.00001)
        sgd_conf = optimizers.serialize(sgd)
        estimator = ElephasEstimator()
        estimator.set_keras_model_config(regression_model.to_yaml())
        estimator.set_optimizer_config(sgd_conf)
        estimator.setFeaturesCol('scaled_features')
        estimator.setOutputCol('output')
        estimator.setLabelCol('ground_truth')
        estimator.set_mode("synchronous")
        estimator.set_loss("mae")
        estimator.set_metrics(['mae'])
        estimator.set_epochs(epochs)
        estimator.set_batch_size(batch_size)
        estimator.set_validation_split(0.01)
        estimator.set_categorical_labels(False)

        pipeline = Pipeline(stages=[estimator])
        fitted_pipeline = pipeline.fit(df)
        prediction = fitted_pipeline.transform(test_df)
        pnl = prediction.select("ground_truth", "output")
        pnl.show(100)

        prediction_and_observations = pnl.rdd.map(lambda row: (row['ground_truth'], row['output']))
        metrics = RegressionMetrics(prediction_and_observations)
        print(metrics.r2)

def test_set_cols(spark_context, regression_model, boston_housing_dataset):
    batch_size = 64
    epochs = 10

    x_train, y_train, x_test, y_test = boston_housing_dataset
    df = to_data_frame(spark_context, x_train, y_train)
    df = df.withColumnRenamed('features', 'scaled_features')
    df = df.withColumnRenamed('label', 'ground_truth')
    test_df = to_data_frame(spark_context, x_test, y_test)
    test_df = test_df.withColumnRenamed('features', 'scaled_features')
    test_df = test_df.withColumnRenamed('label', 'ground_truth')

    sgd = optimizers.SGD(lr=0.00001)
    sgd_conf = optimizers.serialize(sgd)
    estimator = ElephasEstimator(labelCol='ground_truth', outputCol='output', featuresCol='scaled_features')
    estimator.set_keras_model_config(regression_model.to_yaml())
    estimator.set_optimizer_config(sgd_conf)
    estimator.set_mode("synchronous")
    estimator.set_loss("mae")
    estimator.set_metrics(['mae'])
    estimator.set_epochs(epochs)
    estimator.set_batch_size(batch_size)
    estimator.set_validation_split(0.01)
    estimator.set_categorical_labels(False)

    pipeline = Pipeline(stages=[estimator])
    fitted_pipeline = pipeline.fit(df)
    prediction = fitted_pipeline.transform(test_df)
    pnl = prediction.select("ground_truth", "output")
    pnl.show(100)

    prediction_and_observations = pnl.rdd.map(lambda row: (row['ground_truth'], row['output']))
    metrics = RegressionMetrics(prediction_and_observations)
    print(metrics.r2)

def test_custom_objects(spark_context, boston_housing_dataset):
    def custom_activation(x):
        return 2 * relu(x)
    model = Sequential()
    model.add(Dense(64, input_shape=(13,)))
    model.add(Dense(64, activation=custom_activation))
    model.add(Dense(1, activation='linear'))
    x_train, y_train, x_test, y_test = boston_housing_dataset
    df = to_data_frame(spark_context, x_train, y_train)
    test_df = to_data_frame(spark_context, x_test, y_test)

    sgd = optimizers.SGD(lr=0.00001)
    sgd_conf = optimizers.serialize(sgd)
    estimator = ElephasEstimator()
    estimator.set_keras_model_config(model.to_yaml())
    estimator.set_optimizer_config(sgd_conf)
    estimator.set_mode("synchronous")
    estimator.set_loss("mae")
    estimator.set_metrics(['mae'])
    estimator.set_epochs(10)
    estimator.set_batch_size(32)
    estimator.set_validation_split(0.01)
    estimator.set_categorical_labels(False)
    estimator.set_custom_objects({'custom_activation': custom_activation})

    pipeline = Pipeline(stages=[estimator])
    fitted_pipeline = pipeline.fit(df)
    prediction = fitted_pipeline.transform(test_df)

def test_predict_classes_probability(spark_context, classification_model, mnist_data):
    batch_size = 64
    nb_classes = 10
    epochs = 1

    x_train, y_train, x_test, y_test = mnist_data
    x_train = x_train[:1000]
    y_train = y_train[:1000]
    df = to_data_frame(spark_context, x_train, y_train, categorical=True)
    test_df = to_data_frame(spark_context, x_test, y_test, categorical=True)

    sgd = optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
    sgd_conf = optimizers.serialize(sgd)

    # Initialize Spark ML Estimator
    estimator = ElephasEstimator()
    estimator.set_keras_model_config(classification_model.to_yaml())
    estimator.set_optimizer_config(sgd_conf)
    estimator.set_mode("synchronous")
    estimator.set_loss("categorical_crossentropy")
    estimator.set_metrics(['acc'])
    estimator.set_predict_classes(False)
    estimator.set_epochs(epochs)
    estimator.set_batch_size(batch_size)
    estimator.set_validation_split(0.1)
    estimator.set_categorical_labels(True)
    estimator.set_nb_classes(nb_classes)

    # Fitting a model returns a Transformer
    pipeline = Pipeline(stages=[estimator])
    fitted_pipeline = pipeline.fit(df)

    # Evaluate Spark model by evaluating the underlying model
    prediction = fitted_pipeline.transform(test_df)
    pnl = prediction.select("label", "prediction")
    pnl.show(100)

def test_set_predict_classes_regression_warning(spark_context, regression_model):
    with pytest.warns(ElephasWarning):
        estimator = ElephasEstimator()
        estimator.set_loss("mae")
        estimator.set_metrics(['mae'])
        estimator.set_categorical_labels(False)
        estimator.set_predict_classes(True)
<commit_msg>update test<commit_after>import pytest
from tensorflow.keras import optimizers
from tensorflow.keras.layers import Dense
from tensorflow.keras.models import Sequential
from tensorflow.keras.activations import relu

from elephas.ml_model import ElephasEstimator, load_ml_estimator, ElephasTransformer, load_ml_transformer
from elephas.ml.adapter import to_data_frame

from pyspark.mllib.evaluation import MulticlassMetrics, RegressionMetrics
from pyspark.ml import Pipeline

from elephas.utils.warnings import ElephasWarning

def test_serialization_transformer(classification_model):
    transformer = ElephasTransformer()
    transformer.set_keras_model_config(classification_model.to_yaml())
    transformer.save("test.h5")
    loaded_model = load_ml_transformer("test.h5")
    assert loaded_model.get_model().to_yaml() == classification_model.to_yaml()

def test_serialization_estimator(classification_model):
    estimator = ElephasEstimator()
    estimator.set_keras_model_config(classification_model.to_yaml())
    estimator.set_loss("categorical_crossentropy")

    estimator.save("test.h5")
    loaded_model = load_ml_estimator("test.h5")
    assert loaded_model.get_model().to_yaml() == classification_model.to_yaml()

def test_spark_ml_model_classification(spark_context, classification_model, mnist_data):
    batch_size = 64
    nb_classes = 10
    epochs = 1

    x_train, y_train, x_test, y_test = mnist_data
    x_train = x_train[:1000]
    y_train = y_train[:1000]
    df = to_data_frame(spark_context, x_train, y_train, categorical=True)
    test_df = to_data_frame(spark_context, x_test, y_test, categorical=True)

    sgd = optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
    sgd_conf = optimizers.serialize(sgd)

    # Initialize Spark ML Estimator
    estimator = ElephasEstimator()
    estimator.set_keras_model_config(classification_model.to_yaml())
    estimator.set_optimizer_config(sgd_conf)
    estimator.set_mode("synchronous")
    estimator.set_loss("categorical_crossentropy")
    estimator.set_metrics(['acc'])
    estimator.set_epochs(epochs)
    estimator.set_batch_size(batch_size)
    estimator.set_validation_split(0.1)
    estimator.set_categorical_labels(True)
    estimator.set_nb_classes(nb_classes)

    # Fitting a model returns a Transformer
    pipeline = Pipeline(stages=[estimator])
    fitted_pipeline = pipeline.fit(df)

    # Evaluate Spark model by evaluating the underlying model
    prediction = fitted_pipeline.transform(test_df)
    pnl = prediction.select("label", "prediction")
    pnl.show(100)

    prediction_and_label = pnl.rdd.map(lambda row: (row.label, row.prediction))
    metrics = MulticlassMetrics(prediction_and_label)
    print(metrics.accuracy)

def test_functional_model(spark_context, classification_model_functional, mnist_data):
    batch_size = 64
    epochs = 1

    x_train, y_train, x_test, y_test = mnist_data
    x_train = x_train[:1000]
    y_train = y_train[:1000]
    df = to_data_frame(spark_context, x_train, y_train, categorical=True)
    test_df = to_data_frame(spark_context, x_test, y_test, categorical=True)

    sgd = optimizers.SGD()
    sgd_conf = optimizers.serialize(sgd)
    estimator = ElephasEstimator()
    estimator.set_keras_model_config(classification_model_functional.to_yaml())
    estimator.set_optimizer_config(sgd_conf)
    estimator.set_mode("synchronous")
    estimator.set_loss("categorical_crossentropy")
    estimator.set_metrics(['acc'])
    estimator.set_epochs(epochs)
    estimator.set_batch_size(batch_size)
    estimator.set_validation_split(0.1)
    estimator.set_categorical_labels(True)
    estimator.set_nb_classes(10)
    pipeline = Pipeline(stages=[estimator])
    fitted_pipeline = pipeline.fit(df)
    prediction = fitted_pipeline.transform(test_df)
    pnl = prediction.select("label", "prediction")
    pnl.show(100)

    prediction_and_label = pnl.rdd.map(lambda row: (row.label, row.prediction))
    metrics = MulticlassMetrics(prediction_and_label)
    print(metrics.accuracy)

def test_regression_model(spark_context, regression_model, boston_housing_dataset):
    batch_size = 64
    epochs = 10

    x_train, y_train, x_test, y_test = boston_housing_dataset
    df = to_data_frame(spark_context, x_train, y_train)
    test_df = to_data_frame(spark_context, x_test, y_test)

    sgd = optimizers.SGD(lr=0.00001)
    sgd_conf = optimizers.serialize(sgd)
    estimator = ElephasEstimator()
    estimator.set_keras_model_config(regression_model.to_yaml())
    estimator.set_optimizer_config(sgd_conf)
    estimator.set_mode("synchronous")
    estimator.set_loss("mae")
    estimator.set_metrics(['mae'])
    estimator.set_epochs(epochs)
    estimator.set_batch_size(batch_size)
    estimator.set_validation_split(0.01)
    estimator.set_categorical_labels(False)

    pipeline = Pipeline(stages=[estimator])
    fitted_pipeline = pipeline.fit(df)
    prediction = fitted_pipeline.transform(test_df)
    pnl = prediction.select("label", "prediction")
    pnl.show(100)

    prediction_and_observations = pnl.rdd.map(lambda row: (row.label, row.prediction))
    metrics = RegressionMetrics(prediction_and_observations)
    print(metrics.r2)

def test_set_cols_deprecated(spark_context, regression_model, boston_housing_dataset):
    with pytest.deprecated_call():
        batch_size = 64
        epochs = 10

        x_train, y_train, x_test, y_test = boston_housing_dataset
        df = to_data_frame(spark_context, x_train, y_train)
        df = df.withColumnRenamed('features', 'scaled_features')
        df = df.withColumnRenamed('label', 'ground_truth')
        test_df = to_data_frame(spark_context, x_test, y_test)
        test_df = test_df.withColumnRenamed('features', 'scaled_features')
        test_df = test_df.withColumnRenamed('label', 'ground_truth')

        sgd = optimizers.SGD(lr=0.00001)
        sgd_conf = optimizers.serialize(sgd)
        estimator = ElephasEstimator()
        estimator.set_keras_model_config(regression_model.to_yaml())
        estimator.set_optimizer_config(sgd_conf)
        estimator.setFeaturesCol('scaled_features')
        estimator.setOutputCol('output')
        estimator.setLabelCol('ground_truth')
        estimator.set_mode("synchronous")
        estimator.set_loss("mae")
        estimator.set_metrics(['mae'])
        estimator.set_epochs(epochs)
        estimator.set_batch_size(batch_size)
        estimator.set_validation_split(0.01)
        estimator.set_categorical_labels(False)

        pipeline = Pipeline(stages=[estimator])
        fitted_pipeline = pipeline.fit(df)
        prediction = fitted_pipeline.transform(test_df)
        pnl = prediction.select("ground_truth", "output")
        pnl.show(100)

        prediction_and_observations = pnl.rdd.map(lambda row: (row['ground_truth'], row['output']))
        metrics = RegressionMetrics(prediction_and_observations)
        print(metrics.r2)

def test_set_cols(spark_context, regression_model, boston_housing_dataset):
    batch_size = 64
    epochs = 10

    x_train, y_train, x_test, y_test = boston_housing_dataset
    df = to_data_frame(spark_context, x_train, y_train)
    df = df.withColumnRenamed('features', 'scaled_features')
    df = df.withColumnRenamed('label', 'ground_truth')
    test_df = to_data_frame(spark_context, x_test, y_test)
    test_df = test_df.withColumnRenamed('features', 'scaled_features')
    test_df = test_df.withColumnRenamed('label', 'ground_truth')

    sgd = optimizers.SGD(lr=0.00001)
    sgd_conf = optimizers.serialize(sgd)
    estimator = ElephasEstimator(labelCol='ground_truth', outputCol='output', featuresCol='scaled_features')
    estimator.set_keras_model_config(regression_model.to_yaml())
    estimator.set_optimizer_config(sgd_conf)
    estimator.set_mode("synchronous")
    estimator.set_loss("mae")
    estimator.set_metrics(['mae'])
    estimator.set_epochs(epochs)
    estimator.set_batch_size(batch_size)
    estimator.set_validation_split(0.01)
    estimator.set_categorical_labels(False)

    pipeline = Pipeline(stages=[estimator])
    fitted_pipeline = pipeline.fit(df)
    prediction = fitted_pipeline.transform(test_df)
    pnl = prediction.select("ground_truth", "output")
    pnl.show(100)

    prediction_and_observations = pnl.rdd.map(lambda row: (row['ground_truth'], row['output']))
    metrics = RegressionMetrics(prediction_and_observations)
    print(metrics.r2)

def test_custom_objects(spark_context, boston_housing_dataset):
    def custom_activation(x):
        return 2 * relu(x)
    model = Sequential()
    model.add(Dense(64, input_shape=(13,)))
    model.add(Dense(64, activation=custom_activation))
    model.add(Dense(1, activation='linear'))
    x_train, y_train, x_test, y_test = boston_housing_dataset
    df = to_data_frame(spark_context, x_train, y_train)
    test_df = to_data_frame(spark_context, x_test, y_test)

    sgd = optimizers.SGD(lr=0.00001)
    sgd_conf = optimizers.serialize(sgd)
    estimator = ElephasEstimator()
    estimator.set_keras_model_config(model.to_yaml())
    estimator.set_optimizer_config(sgd_conf)
    estimator.set_mode("synchronous")
    estimator.set_loss("mae")
    estimator.set_metrics(['mae'])
    estimator.set_epochs(10)
    estimator.set_batch_size(32)
    estimator.set_validation_split(0.01)
    estimator.set_categorical_labels(False)
    estimator.set_custom_objects({'custom_activation': custom_activation})

    pipeline = Pipeline(stages=[estimator])
    fitted_pipeline = pipeline.fit(df)
    prediction = fitted_pipeline.transform(test_df)

def test_predict_classes_probability(spark_context, classification_model, mnist_data):
    batch_size = 64
    nb_classes = 10
    epochs = 1

    x_train, y_train, x_test, y_test = mnist_data
    x_train = x_train[:1000]
    y_train = y_train[:1000]
    df = to_data_frame(spark_context, x_train, y_train, categorical=True)
    test_df = to_data_frame(spark_context, x_test, y_test, categorical=True)

    sgd = optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
    sgd_conf = optimizers.serialize(sgd)

    # Initialize Spark ML Estimator
    estimator = ElephasEstimator()
    estimator.set_keras_model_config(classification_model.to_yaml())
    estimator.set_optimizer_config(sgd_conf)
    estimator.set_mode("synchronous")
    estimator.set_loss("categorical_crossentropy")
    estimator.set_metrics(['acc'])
    estimator.set_predict_classes(False)
    estimator.set_epochs(epochs)
    estimator.set_batch_size(batch_size)
    estimator.set_validation_split(0.1)
    estimator.set_categorical_labels(True)
    estimator.set_nb_classes(nb_classes)

    # Fitting a model returns a Transformer
    pipeline = Pipeline(stages=[estimator])
    fitted_pipeline = pipeline.fit(df)

    results = fitted_pipeline.transform(test_df)
    # we should have an array of 10 elements in the prediction column, since we have 10 classes
    # and therefore 10 probabilities
    assert len(results.take(1)[0].prediction) == 10

def test_set_predict_classes_regression_warning(spark_context, regression_model):
    with pytest.warns(ElephasWarning):
        estimator = ElephasEstimator()
        estimator.set_loss("mae")
        estimator.set_metrics(['mae'])
        estimator.set_categorical_labels(False)
        estimator.set_predict_classes(True)
<|endoftext|>


<commit_before># Licensed under a 3-clause BSD style license - see LICENSE.rst

import abc

import numpy as np

from astropy.utils import OrderedDict
from astropy.extern import six

from matplotlib.lines import Line2D, Path
from matplotlib.patches import PathPatch

__all__ = ['Spine', 'BaseFrame', 'RectangularFrame', 'EllipticalFrame']

class Spine(object):

    def __init__(self, parent_axes, transform):

        self.parent_axes = parent_axes
        self.transform = transform

        self.data = None
        self.pixel = None
        self.world = None

    @property
    def data(self):
        return self._data

    @data.setter
    def data(self, value):
        if value is None:
            self._data = None
            self._pixel = None
            self._world = None
        else:
            self._data = value
            self._pixel = self.parent_axes.transData.transform(self._data)
            self._world = self.transform.transform(self._data)
            self._update_normal()

    @property
    def pixel(self):
        return self._pixel

    @pixel.setter
    def pixel(self, value):
        if value is None:
            self._data = None
            self._pixel = None
            self._world = None
        else:
            self._data = self.parent_axes.transData.inverted().transform(self._data)
            self._pixel = value
            self._world = self.transform.transform(self._data)
            self._update_normal()

    @property
    def world(self):
        return self._world

    @world.setter
    def world(self, value):
        if value is None:
            self._data = None
            self._pixel = None
            self._world = None
        else:
            self._data = self.transform.transform(value)
            self._pixel = self.parent_axes.transData.transform(self._data)
            self._world = value
            self._update_normal()

    def _update_normal(self):
        # Find angle normal to border and inwards, in display coordinate
        dx = self.pixel[1:, 0] - self.pixel[:-1, 0]
        dy = self.pixel[1:, 1] - self.pixel[:-1, 1]
        self.normal_angle = np.degrees(np.arctan2(dx, -dy))

@six.add_metaclass(abc.ABCMeta)
class BaseFrame(OrderedDict):

    def __init__(self, parent_axes, transform, path=None):

        super(BaseFrame, self).__init__()

        self.parent_axes = parent_axes
        self._transform = transform
        self._linewidth = None
        self._color = 'black'
        self._path = path

        for axis in self.spine_names:
            self[axis] = Spine(parent_axes, transform)

    @property
    def origin(self):
        ymin, ymax = self.parent_axes.get_ylim()
        return 'lower' if ymin < ymax else 'upper'

    @property
    def transform(self):
        return self._transform

    @transform.setter
    def transform(self, value):
        self._transform = value
        for axis in self:
            self[axis].transform = value

    def _update_patch_path(self):

        self.update_spines()
        x, y = [], []
        for axis in self:
            x.append(self[axis].data[:, 0])
            y.append(self[axis].data[:, 1])
        vertices = np.vstack([np.hstack(x), np.hstack(y)]).transpose()

        if self._path is None:
            self._path = Path(vertices)
        else:
            self._path.vertices = vertices

    @property
    def patch(self):
        self._update_patch_path()
        return PathPatch(self._path, transform=self.parent_axes.transData,
                         facecolor='white', edgecolor='white')

    def draw(self, renderer):
        for axis in self:
            x, y = self[axis].pixel[:, 0], self[axis].pixel[:, 1]
            line = Line2D(x, y, linewidth=self._linewidth, color=self._color, zorder=1000)
            line.draw(renderer)

    def sample(self, n_samples):

        self.update_spines()

        spines = OrderedDict()

        for axis in self:

            data = self[axis].data
            p = np.linspace(0., 1., data.shape[0])
            p_new = np.linspace(0., 1., n_samples)
            spines[axis] = Spine(self.parent_axes, self.transform)
            spines[axis].data = np.array([np.interp(p_new, p, data[:, 0]),
                                          np.interp(p_new, p, data[:, 1])]).transpose()

        return spines

    def set_color(self, color):
        """
        Sets the color of the frame.

        Parameters
        ----------
        color : string
            The color of the frame.
        """
        self._color = color

    def set_linewidth(self, linewidth):
        """
        Sets the linewidth of the frame.

        Parameters
        ----------
        linewidth : float
            The linewidth of the frame in points.
        """
        self._linewidth = linewidth

    @abc.abstractmethod
    def update_spines(self):
        raise NotImplementedError("")

class RectangularFrame(BaseFrame):

    spine_names = 'brtl'

    def update_spines(self):

        xmin, xmax = self.parent_axes.get_xlim()
        ymin, ymax = self.parent_axes.get_ylim()

        self['b'].data = np.array(([xmin, ymin], [xmax, ymin]))
        self['r'].data = np.array(([xmax, ymin], [xmax, ymax]))
        self['t'].data = np.array(([xmax, ymax], [xmin, ymax]))
        self['l'].data = np.array(([xmin, ymax], [xmin, ymin]))

class EllipticalFrame(BaseFrame):

    spine_names = 'c'

    def update_spines(self):

        xmin, xmax = self.parent_axes.get_xlim()
        ymin, ymax = self.parent_axes.get_ylim()

        xmid = 0.5 * (xmax + xmin)
        ymid = 0.5 * (ymax + ymin)

        dx = xmid - xmin
        dy = ymid - ymin

        theta = np.linspace(0., 2 * np.pi, 1000)
        self['c'].data = np.array([xmid + dx * np.cos(theta),
                                   ymid + dy * np.sin(theta)]).transpose()
<commit_msg>OrderedDict should be importted from collections<commit_after># Licensed under a 3-clause BSD style license - see LICENSE.rst

import abc
from collections import OrderedDict
import numpy as np

from astropy.extern import six

from matplotlib.lines import Line2D, Path
from matplotlib.patches import PathPatch

__all__ = ['Spine', 'BaseFrame', 'RectangularFrame', 'EllipticalFrame']

class Spine(object):

    def __init__(self, parent_axes, transform):

        self.parent_axes = parent_axes
        self.transform = transform

        self.data = None
        self.pixel = None
        self.world = None

    @property
    def data(self):
        return self._data

    @data.setter
    def data(self, value):
        if value is None:
            self._data = None
            self._pixel = None
            self._world = None
        else:
            self._data = value
            self._pixel = self.parent_axes.transData.transform(self._data)
            self._world = self.transform.transform(self._data)
            self._update_normal()

    @property
    def pixel(self):
        return self._pixel

    @pixel.setter
    def pixel(self, value):
        if value is None:
            self._data = None
            self._pixel = None
            self._world = None
        else:
            self._data = self.parent_axes.transData.inverted().transform(self._data)
            self._pixel = value
            self._world = self.transform.transform(self._data)
            self._update_normal()

    @property
    def world(self):
        return self._world

    @world.setter
    def world(self, value):
        if value is None:
            self._data = None
            self._pixel = None
            self._world = None
        else:
            self._data = self.transform.transform(value)
            self._pixel = self.parent_axes.transData.transform(self._data)
            self._world = value
            self._update_normal()

    def _update_normal(self):
        # Find angle normal to border and inwards, in display coordinate
        dx = self.pixel[1:, 0] - self.pixel[:-1, 0]
        dy = self.pixel[1:, 1] - self.pixel[:-1, 1]
        self.normal_angle = np.degrees(np.arctan2(dx, -dy))

@six.add_metaclass(abc.ABCMeta)
class BaseFrame(OrderedDict):

    def __init__(self, parent_axes, transform, path=None):

        super(BaseFrame, self).__init__()

        self.parent_axes = parent_axes
        self._transform = transform
        self._linewidth = None
        self._color = 'black'
        self._path = path

        for axis in self.spine_names:
            self[axis] = Spine(parent_axes, transform)

    @property
    def origin(self):
        ymin, ymax = self.parent_axes.get_ylim()
        return 'lower' if ymin < ymax else 'upper'

    @property
    def transform(self):
        return self._transform

    @transform.setter
    def transform(self, value):
        self._transform = value
        for axis in self:
            self[axis].transform = value

    def _update_patch_path(self):

        self.update_spines()
        x, y = [], []
        for axis in self:
            x.append(self[axis].data[:, 0])
            y.append(self[axis].data[:, 1])
        vertices = np.vstack([np.hstack(x), np.hstack(y)]).transpose()

        if self._path is None:
            self._path = Path(vertices)
        else:
            self._path.vertices = vertices

    @property
    def patch(self):
        self._update_patch_path()
        return PathPatch(self._path, transform=self.parent_axes.transData,
                         facecolor='white', edgecolor='white')

    def draw(self, renderer):
        for axis in self:
            x, y = self[axis].pixel[:, 0], self[axis].pixel[:, 1]
            line = Line2D(x, y, linewidth=self._linewidth, color=self._color, zorder=1000)
            line.draw(renderer)

    def sample(self, n_samples):

        self.update_spines()

        spines = OrderedDict()

        for axis in self:

            data = self[axis].data
            p = np.linspace(0., 1., data.shape[0])
            p_new = np.linspace(0., 1., n_samples)
            spines[axis] = Spine(self.parent_axes, self.transform)
            spines[axis].data = np.array([np.interp(p_new, p, data[:, 0]),
                                          np.interp(p_new, p, data[:, 1])]).transpose()

        return spines

    def set_color(self, color):
        """
        Sets the color of the frame.

        Parameters
        ----------
        color : string
            The color of the frame.
        """
        self._color = color

    def set_linewidth(self, linewidth):
        """
        Sets the linewidth of the frame.

        Parameters
        ----------
        linewidth : float
            The linewidth of the frame in points.
        """
        self._linewidth = linewidth

    @abc.abstractmethod
    def update_spines(self):
        raise NotImplementedError("")

class RectangularFrame(BaseFrame):

    spine_names = 'brtl'

    def update_spines(self):

        xmin, xmax = self.parent_axes.get_xlim()
        ymin, ymax = self.parent_axes.get_ylim()

        self['b'].data = np.array(([xmin, ymin], [xmax, ymin]))
        self['r'].data = np.array(([xmax, ymin], [xmax, ymax]))
        self['t'].data = np.array(([xmax, ymax], [xmin, ymax]))
        self['l'].data = np.array(([xmin, ymax], [xmin, ymin]))

class EllipticalFrame(BaseFrame):

    spine_names = 'c'

    def update_spines(self):

        xmin, xmax = self.parent_axes.get_xlim()
        ymin, ymax = self.parent_axes.get_ylim()

        xmid = 0.5 * (xmax + xmin)
        ymid = 0.5 * (ymax + ymin)

        dx = xmid - xmin
        dy = ymid - ymin

        theta = np.linspace(0., 2 * np.pi, 1000)
        self['c'].data = np.array([xmid + dx * np.cos(theta),
                                   ymid + dy * np.sin(theta)]).transpose()
<|endoftext|>


<commit_before>import json
from django_prbac.exceptions import PermissionDenied
from django_prbac.utils import ensure_request_has_privilege
from corehq import toggles, privileges
from corehq.apps.export.exceptions import BadExportConfiguration
from corehq.apps.reports.standard import export
from corehq.apps.reports.models import FormExportSchema, HQGroupExportConfiguration, CaseExportSchema
from corehq.apps.reports.standard.export import DeidExportReport
from couchexport.models import ExportTable, ExportSchema, ExportColumn
from django.utils.translation import ugettext as _
from dimagi.utils.decorators.memoized import memoized
from corehq.apps.commtrack.models import StockExportColumn
from corehq.apps.domain.models import Domain

USERNAME_TRANSFORM = 'corehq.apps.export.transforms.user_id_to_username'
OWNERNAME_TRANSFORM = 'corehq.apps.export.transforms.owner_id_to_display'
CASENAME_TRANSFORM = 'corehq.apps.export.transforms.case_id_to_case_name'

class AbstractProperty(object):
    def __get__(self, instance, owner):
        raise NotImplementedError()

class CustomExportHelper(object):

    ExportSchemaClass = AbstractProperty()
    ExportReport = AbstractProperty()
    export_title = AbstractProperty()

    allow_deid = False
    allow_repeats = True

    subclasses_map = {}  # filled in below

    export_type = 'form'

    @property
    def default_order(self):
        return {}

    @classmethod
    def make(cls, request, export_type, domain=None, export_id=None):
        export_type = export_type or request.GET.get('request_type', 'form')
        return cls.subclasses_map[export_type](request, domain, export_id=export_id)

    def update_custom_params(self):
        if len(self.custom_export.tables) > 0:
            if self.export_stock:
                self.custom_export.tables[0].columns.append(
                    StockExportColumn(domain=self.domain, index='_id')
                )

    def format_config_for_javascript(self, table_configuration):
        return table_configuration

    def has_stock_column(self):
        return any(
            col.doc_type == 'StockExportColumn'
            for col in self.custom_export.tables[0].columns
        ) if self.custom_export.tables else False

    class DEID(object):
        options = (
            ('', ''),
            (_('Sensitive ID'), 'couchexport.deid.deid_ID'),
            (_('Sensitive Date'), 'couchexport.deid.deid_date'),
        )
        json_options = [{'label': label, 'value': value}
                        for label, value in options]

    def __init__(self, request, domain, export_id=None):
        self.request = request
        self.domain = domain
        self.presave = False
        self.transform_dates = False
        self.creating_new_export = not bool(export_id)

        if export_id:
            self.custom_export = self.ExportSchemaClass.get(export_id)
            # also update the schema to include potential new stuff
            self.custom_export.update_schema()

            # enable configuring saved exports from this page
            saved_group = HQGroupExportConfiguration.get_for_domain(self.domain)
            self.presave = export_id in saved_group.custom_export_ids

            self.export_stock = self.has_stock_column()

            try:
                assert self.custom_export.doc_type == 'SavedExportSchema', 'bad export doc type'
                assert self.custom_export.type == self.export_type, 'wrong export type specified'
                assert self.custom_export.index[0] == domain, 'bad export doc domain'
            except AssertionError, e:
                raise BadExportConfiguration(str(e))
        else:
            self.custom_export = self.ExportSchemaClass(type=self.export_type)
            self.export_stock = False

    @property
    @memoized
    def post_data(self):
        return json.loads(self.request.raw_post_data)

    def update_custom_export(self):
        """
        Updates custom_export object from the request
        and saves to the db
        """

        post_data = self.post_data

        custom_export_json = post_data['custom_export']

        SAFE_KEYS = ('default_format', 'is_safe', 'name', 'schema_id', 'transform_dates')
        for key in SAFE_KEYS:
            self.custom_export[key] = custom_export_json[key]

        # update the custom export index (to stay in sync)
        schema_id = self.custom_export.schema_id
        schema = ExportSchema.get(schema_id)
        self.custom_export.index = schema.index

        self.presave = post_data['presave']
        self.export_stock = post_data['export_stock']

        self.custom_export.tables = [
            ExportTable.wrap(table)
            for table in custom_export_json['tables']
        ]

        table_dict = dict((t.index, t) for t in self.custom_export.tables)
        for table in self.custom_export.tables:
            if table.index in table_dict:
                table_dict[table.index].columns = table.columns
            else:
                self.custom_export.tables.append(
                    ExportTable(
                        index=table.index,
                        display=self.custom_export.name,
                        columns=table.columns
                    )
                )

        self.update_custom_params()
        self.custom_export.custom_validate()
        self.custom_export.save()

        if self.presave:
            HQGroupExportConfiguration.add_custom_export(self.domain, self.custom_export.get_id)
        else:
            HQGroupExportConfiguration.remove_custom_export(self.domain, self.custom_export.get_id)

    def get_context(self):
        table_configuration = self.format_config_for_javascript(self.custom_export.table_configuration)
        return {
            'custom_export': self.custom_export,
            'default_order': self.default_order,
            'deid_options': self.DEID.json_options,
            'presave': self.presave,
            'export_stock': self.export_stock,
            'DeidExportReport_name': DeidExportReport.name,
            'table_configuration': table_configuration,
            'domain': self.domain,
            'commtrack_domain': Domain.get_by_name(self.domain).commtrack_enabled,
            'helper': {
                'back_url': self.ExportReport.get_url(domain=self.domain),
                'export_title': self.export_title,
                'slug': self.ExportReport.slug,
                'allow_deid': self.allow_deid,
                'allow_repeats': self.allow_repeats
            }
        }

class FormCustomExportHelper(CustomExportHelper):

    ExportSchemaClass = FormExportSchema
    ExportReport = export.ExcelExportReport

    allow_repeats = True

    default_questions = ["form.case.@case_id", "form.meta.timeEnd", "_id", "id", "form.meta.username"]
    questions_to_show = default_questions + ["form.meta.timeStart", "received_on"]

    @property
    def export_title(self):
        return _('Export Submissions to Excel')

    def __init__(self, request, domain, export_id=None):
        super(FormCustomExportHelper, self).__init__(request, domain, export_id)
        if not self.custom_export.app_id:
            self.custom_export.app_id = request.GET.get('app_id')

    @property
    def allow_deid(self):
        try:
            ensure_request_has_privilege(self.request, privileges.DEIDENTIFIED_DATA)
            return True
        except PermissionDenied:
            return False

    def update_custom_params(self):
        p = self.post_data['custom_export']
        e = self.custom_export
        e.include_errors = p['include_errors']
        e.app_id = p['app_id']

    @property
    @memoized
    def default_order(self):
        return self.custom_export.get_default_order()

    def update_table_conf_with_questions(self, table_conf):
        column_conf = table_conf[0].get("column_configuration", [])

        current_questions = set(self.custom_export.question_order)
        remaining_questions = current_questions.copy()

        def is_special_type(q):
            return any([q.startswith('form.#'), q.startswith('form.@'), q.startswith('form.case.'),
                        q.startswith('form.meta.'), q.startswith('form.subcase_')])

        def generate_additional_columns(requires_case):
            ret = []
            case_name_col = CustomColumn(slug='case_name', index='form.case.@case_id', display='info.case_name',
                                         transform=CASENAME_TRANSFORM, show=True, selected=True)
            if not requires_case:
                case_name_col.show, case_name_col.selected, case_name_col.tag = False, False, 'deleted'
            matches = filter(case_name_col.match, column_conf)
            if matches:
                # hack/annoying - also might have to re-add the case id column which can get
                # overwritten by case name if only that is set.
                case_id_cols = filter(lambda col: col['index'] == 'form.case.@case_id', column_conf)
                if len(case_id_cols) <= 1:
                    ret.append(ExportColumn(
                        index='form.case.@case_id',
                        display='info.case_id',
                        show=True,
                    ).to_config_format(selected=False))

                for match in matches:
                    case_name_col.format_for_javascript(match)

            elif filter(lambda col: col["index"] == case_name_col.index, column_conf):
                ret.append(case_name_col.default_column())
            return ret

        for col in column_conf:
            question = col["index"]
            if question in remaining_questions:
                remaining_questions.discard(question)
                col["show"] = True
            if question.startswith("form.") and not is_special_type(question) and question not in current_questions:
                col["tag"] = "deleted"
                col["show"] = False
            if question in self.questions_to_show:
                col["show"] = True
            if self.creating_new_export and (question in self.default_questions or question in current_questions):
                col["selected"] = True

        requires_case = self.custom_export.uses_cases()

        case_cols = filter(lambda col: col["index"] == 'form.case.@case_id', column_conf)
        if not requires_case:
            for col in case_cols:
                if col['index'] == 'form.case.@case_id':
                    col['tag'], col['show'], col['selected'] = 'deleted', False, False
        elif not case_cols:
            column_conf.append({
                'index': 'form.case.@case_id',
                'show': True,
                'is_sensitive': False,
                'selected': True,
                'transform': None,
                'tag': None,
                'display': ''
            })

        column_conf.extend(generate_additional_columns(requires_case))
        column_conf.extend([
            ExportColumn(
                index=q,
                display='',
                show=True,
            ).to_config_format(selected=self.creating_new_export)
            for q in remaining_questions
        ])

        # show all questions in repeat groups by default
        for conf in table_conf:
            if conf["index"].startswith('#.form.'):
                for col in conf.get("column_configuration", []):
                    col["show"] = True

        table_conf[0]["column_configuration"] = column_conf
        return table_conf

    def get_context(self):
        ctxt = super(FormCustomExportHelper, self).get_context()
        self.update_table_conf_with_questions(ctxt["table_configuration"])
        return ctxt

class CustomColumn(object):

    def __init__(self, slug, index, display, transform, is_sensitive=False, tag=None, show=False, selected=False):
        self.slug = slug
        self.index = index
        self.display = display
        self.transform = transform
        self.is_sensitive = is_sensitive
        self.tag = tag
        self.show = show
        self.selected = selected

    def match(self, col):
         return col['index'] == self.index and col['transform'] == self.transform

    def format_for_javascript(self, col):
        # this is js --> js conversion so the name is pretty bad
        # couch --> javascript UI code
        col['special'] = self.slug

    def default_column(self):
        # this is kinda hacky - mirrors ExportColumn.to_config_format to add custom columns
        # to the existing export UI
        return {
            'index': self.index,
            'selected': self.selected,
            'display': self.display,
            'transform': self.transform,
            "is_sensitive": self.is_sensitive,
            'tag': self.tag,
            'special': self.slug,
            'show': self.show,
        }

class CaseCustomExportHelper(CustomExportHelper):

    ExportSchemaClass = CaseExportSchema
    ExportReport = export.CaseExportReport

    export_type = 'case'

    default_properties = ["_id", "closed", "closed_on", "modified_on", "opened_on", "info.owner_name", "id"]
    properties_to_show = ["identifier", "referenced_id", "referenced_type", "id", "doc_type"]
    default_transformed_properties = ["info.closed_by_username", "info.last_modified_by_username",
                                      "info.opened_by_username", "info.owner_name"]
    meta_properties = ["_id", "closed", "closed_by", "closed_on", "domain", "computed_modified_on_",
                       "server_modified_on", "modified_on", "opened_by", "opened_on", "owner_id",
                       "user_id", "type", "version", "external_id"]
    server_properties = ["_rev", "doc_type", "-deletion_id", "initial_processing_complete"]
    row_properties = ["id"]

    @property
    def export_title(self):
        return _('Export Cases, Referrals, and Users')

    def format_config_for_javascript(self, table_configuration):
        custom_columns = [
            CustomColumn(slug='last_modified_by_username', index='user_id',
                         display='info.last_modified_by_username', transform=USERNAME_TRANSFORM),
            CustomColumn(slug='opened_by_username', index='opened_by',
                         display='info.opened_by_username', transform=USERNAME_TRANSFORM),
            CustomColumn(slug='closed_by_username', index='closed_by',
                         display='info.closed_by_username', transform=USERNAME_TRANSFORM),
            CustomColumn(slug='owner_name', index='owner_id', display='info.owner_name',
                         transform=OWNERNAME_TRANSFORM),
        ]
        main_table_columns = table_configuration[0]['column_configuration']
        for custom in custom_columns:
            matches = filter(custom.match, main_table_columns)
            if not matches:
                main_table_columns.append(custom.default_column())
            else:
                for match in matches:
                    custom.format_for_javascript(match)

        return table_configuration

    def update_table_conf(self, table_conf):
        column_conf = table_conf[0].get("column_configuration", [])
        current_properties = set(self.custom_export.case_properties)
        remaining_properties = current_properties.copy()

        def is_special_type(p):
            return any([p in self.meta_properties, p in self.server_properties, p in self.row_properties])

        for col in column_conf:
            prop = col["index"]
            display = col.get('display') or prop
            if prop in remaining_properties:
                remaining_properties.discard(prop)
                col["show"] = True
            if not is_special_type(prop) and prop not in current_properties:
                col["tag"] = "deleted"
                col["show"] = False
            if prop in self.default_properties + list(current_properties) or \
                            display in self.default_transformed_properties:
                col["show"] = True
                if self.creating_new_export:
                    col["selected"] = True

        column_conf.extend([
            ExportColumn(
                index=prop,
                display='',
                show=True,
            ).to_config_format(selected=self.creating_new_export)
            for prop in filter(lambda prop: not prop.startswith("parent/"), remaining_properties)
        ])

        table_conf[0]["column_configuration"] = column_conf

        for table in table_conf:
            for col in table.get("column_configuration", []):
                if col["index"] in self.properties_to_show:
                    col["show"] = True

        return table_conf

    def get_context(self):
        ctxt = super(CaseCustomExportHelper, self).get_context()
        self.update_table_conf(ctxt["table_configuration"])
        return ctxt

CustomExportHelper.subclasses_map.update({
    'form': FormCustomExportHelper,
    'case': CaseCustomExportHelper,
})
<commit_msg>import<commit_after>import json
from django_prbac.exceptions import PermissionDenied
from django_prbac.utils import ensure_request_has_privilege
from corehq import privileges
from corehq.apps.export.exceptions import BadExportConfiguration
from corehq.apps.reports.standard import export
from corehq.apps.reports.models import FormExportSchema, HQGroupExportConfiguration, CaseExportSchema
from corehq.apps.reports.standard.export import DeidExportReport
from couchexport.models import ExportTable, ExportSchema, ExportColumn
from django.utils.translation import ugettext as _
from dimagi.utils.decorators.memoized import memoized
from corehq.apps.commtrack.models import StockExportColumn
from corehq.apps.domain.models import Domain

USERNAME_TRANSFORM = 'corehq.apps.export.transforms.user_id_to_username'
OWNERNAME_TRANSFORM = 'corehq.apps.export.transforms.owner_id_to_display'
CASENAME_TRANSFORM = 'corehq.apps.export.transforms.case_id_to_case_name'

class AbstractProperty(object):
    def __get__(self, instance, owner):
        raise NotImplementedError()

class CustomExportHelper(object):

    ExportSchemaClass = AbstractProperty()
    ExportReport = AbstractProperty()
    export_title = AbstractProperty()

    allow_deid = False
    allow_repeats = True

    subclasses_map = {}  # filled in below

    export_type = 'form'

    @property
    def default_order(self):
        return {}

    @classmethod
    def make(cls, request, export_type, domain=None, export_id=None):
        export_type = export_type or request.GET.get('request_type', 'form')
        return cls.subclasses_map[export_type](request, domain, export_id=export_id)

    def update_custom_params(self):
        if len(self.custom_export.tables) > 0:
            if self.export_stock:
                self.custom_export.tables[0].columns.append(
                    StockExportColumn(domain=self.domain, index='_id')
                )

    def format_config_for_javascript(self, table_configuration):
        return table_configuration

    def has_stock_column(self):
        return any(
            col.doc_type == 'StockExportColumn'
            for col in self.custom_export.tables[0].columns
        ) if self.custom_export.tables else False

    class DEID(object):
        options = (
            ('', ''),
            (_('Sensitive ID'), 'couchexport.deid.deid_ID'),
            (_('Sensitive Date'), 'couchexport.deid.deid_date'),
        )
        json_options = [{'label': label, 'value': value}
                        for label, value in options]

    def __init__(self, request, domain, export_id=None):
        self.request = request
        self.domain = domain
        self.presave = False
        self.transform_dates = False
        self.creating_new_export = not bool(export_id)

        if export_id:
            self.custom_export = self.ExportSchemaClass.get(export_id)
            # also update the schema to include potential new stuff
            self.custom_export.update_schema()

            # enable configuring saved exports from this page
            saved_group = HQGroupExportConfiguration.get_for_domain(self.domain)
            self.presave = export_id in saved_group.custom_export_ids

            self.export_stock = self.has_stock_column()

            try:
                assert self.custom_export.doc_type == 'SavedExportSchema', 'bad export doc type'
                assert self.custom_export.type == self.export_type, 'wrong export type specified'
                assert self.custom_export.index[0] == domain, 'bad export doc domain'
            except AssertionError, e:
                raise BadExportConfiguration(str(e))
        else:
            self.custom_export = self.ExportSchemaClass(type=self.export_type)
            self.export_stock = False

    @property
    @memoized
    def post_data(self):
        return json.loads(self.request.raw_post_data)

    def update_custom_export(self):
        """
        Updates custom_export object from the request
        and saves to the db
        """

        post_data = self.post_data

        custom_export_json = post_data['custom_export']

        SAFE_KEYS = ('default_format', 'is_safe', 'name', 'schema_id', 'transform_dates')
        for key in SAFE_KEYS:
            self.custom_export[key] = custom_export_json[key]

        # update the custom export index (to stay in sync)
        schema_id = self.custom_export.schema_id
        schema = ExportSchema.get(schema_id)
        self.custom_export.index = schema.index

        self.presave = post_data['presave']
        self.export_stock = post_data['export_stock']

        self.custom_export.tables = [
            ExportTable.wrap(table)
            for table in custom_export_json['tables']
        ]

        table_dict = dict((t.index, t) for t in self.custom_export.tables)
        for table in self.custom_export.tables:
            if table.index in table_dict:
                table_dict[table.index].columns = table.columns
            else:
                self.custom_export.tables.append(
                    ExportTable(
                        index=table.index,
                        display=self.custom_export.name,
                        columns=table.columns
                    )
                )

        self.update_custom_params()
        self.custom_export.custom_validate()
        self.custom_export.save()

        if self.presave:
            HQGroupExportConfiguration.add_custom_export(self.domain, self.custom_export.get_id)
        else:
            HQGroupExportConfiguration.remove_custom_export(self.domain, self.custom_export.get_id)

    def get_context(self):
        table_configuration = self.format_config_for_javascript(self.custom_export.table_configuration)
        return {
            'custom_export': self.custom_export,
            'default_order': self.default_order,
            'deid_options': self.DEID.json_options,
            'presave': self.presave,
            'export_stock': self.export_stock,
            'DeidExportReport_name': DeidExportReport.name,
            'table_configuration': table_configuration,
            'domain': self.domain,
            'commtrack_domain': Domain.get_by_name(self.domain).commtrack_enabled,
            'helper': {
                'back_url': self.ExportReport.get_url(domain=self.domain),
                'export_title': self.export_title,
                'slug': self.ExportReport.slug,
                'allow_deid': self.allow_deid,
                'allow_repeats': self.allow_repeats
            }
        }

class FormCustomExportHelper(CustomExportHelper):

    ExportSchemaClass = FormExportSchema
    ExportReport = export.ExcelExportReport

    allow_repeats = True

    default_questions = ["form.case.@case_id", "form.meta.timeEnd", "_id", "id", "form.meta.username"]
    questions_to_show = default_questions + ["form.meta.timeStart", "received_on"]

    @property
    def export_title(self):
        return _('Export Submissions to Excel')

    def __init__(self, request, domain, export_id=None):
        super(FormCustomExportHelper, self).__init__(request, domain, export_id)
        if not self.custom_export.app_id:
            self.custom_export.app_id = request.GET.get('app_id')

    @property
    def allow_deid(self):
        try:
            ensure_request_has_privilege(self.request, privileges.DEIDENTIFIED_DATA)
            return True
        except PermissionDenied:
            return False

    def update_custom_params(self):
        p = self.post_data['custom_export']
        e = self.custom_export
        e.include_errors = p['include_errors']
        e.app_id = p['app_id']

    @property
    @memoized
    def default_order(self):
        return self.custom_export.get_default_order()

    def update_table_conf_with_questions(self, table_conf):
        column_conf = table_conf[0].get("column_configuration", [])

        current_questions = set(self.custom_export.question_order)
        remaining_questions = current_questions.copy()

        def is_special_type(q):
            return any([q.startswith('form.#'), q.startswith('form.@'), q.startswith('form.case.'),
                        q.startswith('form.meta.'), q.startswith('form.subcase_')])

        def generate_additional_columns(requires_case):
            ret = []
            case_name_col = CustomColumn(slug='case_name', index='form.case.@case_id', display='info.case_name',
                                         transform=CASENAME_TRANSFORM, show=True, selected=True)
            if not requires_case:
                case_name_col.show, case_name_col.selected, case_name_col.tag = False, False, 'deleted'
            matches = filter(case_name_col.match, column_conf)
            if matches:
                # hack/annoying - also might have to re-add the case id column which can get
                # overwritten by case name if only that is set.
                case_id_cols = filter(lambda col: col['index'] == 'form.case.@case_id', column_conf)
                if len(case_id_cols) <= 1:
                    ret.append(ExportColumn(
                        index='form.case.@case_id',
                        display='info.case_id',
                        show=True,
                    ).to_config_format(selected=False))

                for match in matches:
                    case_name_col.format_for_javascript(match)

            elif filter(lambda col: col["index"] == case_name_col.index, column_conf):
                ret.append(case_name_col.default_column())
            return ret

        for col in column_conf:
            question = col["index"]
            if question in remaining_questions:
                remaining_questions.discard(question)
                col["show"] = True
            if question.startswith("form.") and not is_special_type(question) and question not in current_questions:
                col["tag"] = "deleted"
                col["show"] = False
            if question in self.questions_to_show:
                col["show"] = True
            if self.creating_new_export and (question in self.default_questions or question in current_questions):
                col["selected"] = True

        requires_case = self.custom_export.uses_cases()

        case_cols = filter(lambda col: col["index"] == 'form.case.@case_id', column_conf)
        if not requires_case:
            for col in case_cols:
                if col['index'] == 'form.case.@case_id':
                    col['tag'], col['show'], col['selected'] = 'deleted', False, False
        elif not case_cols:
            column_conf.append({
                'index': 'form.case.@case_id',
                'show': True,
                'is_sensitive': False,
                'selected': True,
                'transform': None,
                'tag': None,
                'display': ''
            })

        column_conf.extend(generate_additional_columns(requires_case))
        column_conf.extend([
            ExportColumn(
                index=q,
                display='',
                show=True,
            ).to_config_format(selected=self.creating_new_export)
            for q in remaining_questions
        ])

        # show all questions in repeat groups by default
        for conf in table_conf:
            if conf["index"].startswith('#.form.'):
                for col in conf.get("column_configuration", []):
                    col["show"] = True

        table_conf[0]["column_configuration"] = column_conf
        return table_conf

    def get_context(self):
        ctxt = super(FormCustomExportHelper, self).get_context()
        self.update_table_conf_with_questions(ctxt["table_configuration"])
        return ctxt

class CustomColumn(object):

    def __init__(self, slug, index, display, transform, is_sensitive=False, tag=None, show=False, selected=False):
        self.slug = slug
        self.index = index
        self.display = display
        self.transform = transform
        self.is_sensitive = is_sensitive
        self.tag = tag
        self.show = show
        self.selected = selected

    def match(self, col):
         return col['index'] == self.index and col['transform'] == self.transform

    def format_for_javascript(self, col):
        # this is js --> js conversion so the name is pretty bad
        # couch --> javascript UI code
        col['special'] = self.slug

    def default_column(self):
        # this is kinda hacky - mirrors ExportColumn.to_config_format to add custom columns
        # to the existing export UI
        return {
            'index': self.index,
            'selected': self.selected,
            'display': self.display,
            'transform': self.transform,
            "is_sensitive": self.is_sensitive,
            'tag': self.tag,
            'special': self.slug,
            'show': self.show,
        }

class CaseCustomExportHelper(CustomExportHelper):

    ExportSchemaClass = CaseExportSchema
    ExportReport = export.CaseExportReport

    export_type = 'case'

    default_properties = ["_id", "closed", "closed_on", "modified_on", "opened_on", "info.owner_name", "id"]
    properties_to_show = ["identifier", "referenced_id", "referenced_type", "id", "doc_type"]
    default_transformed_properties = ["info.closed_by_username", "info.last_modified_by_username",
                                      "info.opened_by_username", "info.owner_name"]
    meta_properties = ["_id", "closed", "closed_by", "closed_on", "domain", "computed_modified_on_",
                       "server_modified_on", "modified_on", "opened_by", "opened_on", "owner_id",
                       "user_id", "type", "version", "external_id"]
    server_properties = ["_rev", "doc_type", "-deletion_id", "initial_processing_complete"]
    row_properties = ["id"]

    @property
    def export_title(self):
        return _('Export Cases, Referrals, and Users')

    def format_config_for_javascript(self, table_configuration):
        custom_columns = [
            CustomColumn(slug='last_modified_by_username', index='user_id',
                         display='info.last_modified_by_username', transform=USERNAME_TRANSFORM),
            CustomColumn(slug='opened_by_username', index='opened_by',
                         display='info.opened_by_username', transform=USERNAME_TRANSFORM),
            CustomColumn(slug='closed_by_username', index='closed_by',
                         display='info.closed_by_username', transform=USERNAME_TRANSFORM),
            CustomColumn(slug='owner_name', index='owner_id', display='info.owner_name',
                         transform=OWNERNAME_TRANSFORM),
        ]
        main_table_columns = table_configuration[0]['column_configuration']
        for custom in custom_columns:
            matches = filter(custom.match, main_table_columns)
            if not matches:
                main_table_columns.append(custom.default_column())
            else:
                for match in matches:
                    custom.format_for_javascript(match)

        return table_configuration

    def update_table_conf(self, table_conf):
        column_conf = table_conf[0].get("column_configuration", [])
        current_properties = set(self.custom_export.case_properties)
        remaining_properties = current_properties.copy()

        def is_special_type(p):
            return any([p in self.meta_properties, p in self.server_properties, p in self.row_properties])

        for col in column_conf:
            prop = col["index"]
            display = col.get('display') or prop
            if prop in remaining_properties:
                remaining_properties.discard(prop)
                col["show"] = True
            if not is_special_type(prop) and prop not in current_properties:
                col["tag"] = "deleted"
                col["show"] = False
            if prop in self.default_properties + list(current_properties) or \
                            display in self.default_transformed_properties:
                col["show"] = True
                if self.creating_new_export:
                    col["selected"] = True

        column_conf.extend([
            ExportColumn(
                index=prop,
                display='',
                show=True,
            ).to_config_format(selected=self.creating_new_export)
            for prop in filter(lambda prop: not prop.startswith("parent/"), remaining_properties)
        ])

        table_conf[0]["column_configuration"] = column_conf

        for table in table_conf:
            for col in table.get("column_configuration", []):
                if col["index"] in self.properties_to_show:
                    col["show"] = True

        return table_conf

    def get_context(self):
        ctxt = super(CaseCustomExportHelper, self).get_context()
        self.update_table_conf(ctxt["table_configuration"])
        return ctxt

CustomExportHelper.subclasses_map.update({
    'form': FormCustomExportHelper,
    'case': CaseCustomExportHelper,
})
<|endoftext|>


<commit_before>from dronekit_sitl import SITL
import dronekit
import time
from dronekit_sitl import version_list
from dronekit_sitl import download
import sys

copter_args = ['-S', '--model', 'quad', '--home=-35.363261,149.165230,584,353']

sitl = SITL()
sitl.stop()
sitl.download('copter', '3.3')
sitl.launch(copter_args, verbose=True, await_ready=True, use_saved_data=False)
sitl.stop()

sys.exit(0)

versions = version_list()
models = list(versions.keys())
print models.sort()

download('copter','3.3', None)

connection_string = sitl.connection_string()
vehicle = dronekit.connect(connection_string, wait_ready=True)

new_compass_use = 10

sitl.block_until_ready()

sitl.poll()
sitl.using_sim

sitl.stop()
sitl.poll()

sitl.launch(copter_args)
sitl.stop()

print("Changing COMPASS_USE to {0}".format(new_compass_use))
while vehicle.parameters["COMPASS_USE"] != new_compass_use:
    vehicle.parameters["COMPASS_USE"] = new_compass_use
    time.sleep(0.1)

print("Changed COMPASS_USE to {0}".format(new_compass_use))
time.sleep(5) # give parameters time to write
sitl.stop()
vehicle.close()

# Now see if it persisted
sitl.launch(copter_args, await_ready=True, use_saved_data=True)
vehicle = dronekit.connect(connection_string, wait_ready=True)
print vehicle.parameters["COMPASS_USE"]
vehicle.close()
sitl.stop()

<commit_msg>La Aurora<commit_after>from dronekit_sitl import SITL
import dronekit
import time
from dronekit_sitl import version_list
from dronekit_sitl import download
import sys

copter_args = ['-S', '--model', 'quad', '--home=-35.363261,149.165230,584,353']

sitl = SITL()
sitl.launch(copter_args, await_ready=True, use_saved_data=True)
vehicle = dronekit.connect(connection_string, wait_ready=True)
print vehicle.parameters["COMPASS_USE"]
vehicle.close()
sitl.stop()

sys.exit(0)

sitl = SITL()
sitl.stop()
sitl.download('copter', '3.3')
sitl.launch(copter_args, verbose=True, await_ready=True, use_saved_data=False)
sitl.stop()

sys.exit(0)

versions = version_list()
models = list(versions.keys())
print models.sort()

download('copter','3.3', None)

connection_string = sitl.connection_string()
vehicle = dronekit.connect(connection_string, wait_ready=True)

new_compass_use = 10

sitl.block_until_ready()

sitl.poll()
sitl.using_sim

sitl.stop()
sitl.poll()

sitl.launch(copter_args)
sitl.stop()

print("Changing COMPASS_USE to {0}".format(new_compass_use))
while vehicle.parameters["COMPASS_USE"] != new_compass_use:
    vehicle.parameters["COMPASS_USE"] = new_compass_use
    time.sleep(0.1)

print("Changed COMPASS_USE to {0}".format(new_compass_use))
time.sleep(5) # give parameters time to write
sitl.stop()
vehicle.close()

# Now see if it persisted

<|endoftext|>