<commit_before># coding: utf-8
from __future__ import unicode_literals

import pytest

@pytest.mark.xfail
@pytest.mark.parametrize('text', ["This is a string ", "This is a string\u0020"])
def test_issue792(en_tokenizer, text):
    """Test for Issue #792: Trailing whitespace is removed after parsing."""
    doc = en_tokenizer(text)
    assert(doc.text_with_ws == text)
<commit_msg>Fix formatting<commit_after># coding: utf-8
from __future__ import unicode_literals

import pytest

@pytest.mark.xfail
@pytest.mark.parametrize('text', ["This is a string ", "This is a string\u0020"])
def test_issue792(en_tokenizer, text):
    """Test for Issue #792: Trailing whitespace is removed after parsing."""
    doc = en_tokenizer(text)
    assert doc.text_with_ws == text
<|endoftext|>


<commit_before>#!/usr/bin/env python3

from autobahn.asyncio.websocket \
	import WebSocketServerProtocol, WebSocketServerFactory

import logging as lg
import asyncio
import json
import random
import time
import os
import sys

lg.basicConfig(stream = sys.stdout , level = lg.DEBUG)

PROCESSORS_SERVER_IP = '0.0.0.0'
PROCESSORS_SERVER_PORT = 12121

CLIENTS_SERVER_IP = '0.0.0.0'
CLIENTS_SERVER_PORT = 21212

MAX_PONG_WAIT_TIME = 5000 # Milliseconds
PING_INTERVAL = 5 # Seconds

MAX_JOB_PER_PROCESSOR = 8
MAX_JOB_REACHED_REST_TIME = 0.1 # Seconds

IDLE_ID = None
IDLE_PROCESS_ID = None
IDLE_SCRIPT = "function(){return 123;}"
IDLE_ARGS = []
IDLE_RESULT = 123
IDLE_INTERVAL = 30 # Seconds

MAXIMUM_JOB_CODE_REPEAT_COUNT = 32

MAX_FAILURES = 8

IP_JOB_COUNT_LIMIT = 1000
IP_DURATION_LIMIT = 30000 # Milliseconds

SSL_CERT_FILE = '/etc/letsencrypt/live/pooljs.ir/cert.pem'
SSL_KEY_FILE = '/etc/letsencrypt/live/pooljs.ir/privkey.pem'

class Job:

	def __init__(self,code,websocket,args,identity,process_id):
		self.process_id = process_id
		self.code = code
		self.websocket = websocket # Job owner websocket
		self.args = args
		self.identity = identity # Every Job has an identity in the client side
		self.fails = 0 # Jobs will be deleted from the list when having multiple failures (MAX_FAILURES)

class IpLimit:

	def __init__(self,duration_limit,count_limit):
		self.duration_limit = duration_limit
		self.count_limit = count_limit
		self.expiry_time = None
		self.count = 0 # Count of Jobs ran by this IP since the last reset

ip_limit = {} # Mapping IP as strings to IpLimits
job_id_queue = asyncio.Queue() # Queue of requested Job ids
job_id_counter = 0 # A counter for generating Job ids
process_id_counter = 0 # A counter for generating Process ids
jobs = {} # Mapping Job ids as integers to Jobs
processor_websockets = set()
client_websockets = set()
processor_exists = asyncio.Condition()

# Get current time in seconds
def now():
	return int(time.time() * 1000.0)

class ProcessorProtocol(WebSocketServerProtocol):

	def __init__(self, *args, **kwargs):
		super().__init__(*args, **kwargs)
		self.job_ids = []
		self.last_ping_time = None
		self.last_pong_time = None
		self.last_process_id = None
		self.last_process_id_repeat_count = 0

	def send_job(self,job_id,code,args,process_id):
		if process_id == self.last_process_id:
			self.last_process_id_repeat_count += 1
		else:
			self.last_process_id = process_id
			self.last_process_id_repeat_count = 1
		message = { "id": job_id,
					"process_id": process_id,
					"args": args }
		if self.last_process_id_repeat_count <= MAXIMUM_JOB_CODE_REPEAT_COUNT:
			message["code"] = code
		if not self.last_ping_time or (self.last_pong_time and self.last_ping_time < self.last_pong_time):
			self.last_ping_time = now()
		self.sendMessage(json.dumps(message).encode('utf-8'),False)
		if job_id is not None: # Do not add Idle Jobs to the list
			self.job_ids.append(job_id)

	async def onOpen(self):
		# Notify the balancer a new Processor has been added
		await processor_exists.acquire()
		processor_websockets.add(self)
		processor_exists.notify_all()
		processor_exists.release()

	async def onMessage(self, payload, isBinary):
		msg = json.loads(payload.decode('utf8'))
		job_id = msg["id"]
		if job_id is not None:
			self.last_pong_time = now()
			try:
				if not msg["error"]:
					jobs[job_id].websocket.result_available(job_id,msg["result"],False)
					jobs[job_id].websocket.job_ids.remove(job_id)
					del jobs[job_id]
				else:
					jobs[job_id].fails += 1
					# Delete the Job from the list when it has multiple failures
					if jobs[job_id].fails > MAX_FAILURES:
						# Send and error to the Client
						jobs[job_id].websocket.result_available(job_id,None,True)
						del jobs[job_id]
					else:
						await job_id_queue.put(job_id)
			except KeyError:
				pass
			if job_id in self.job_ids:
				self.job_ids.remove(job_id)
		else: # This is an Idle Job!
			if msg["result"] == IDLE_RESULT:
				self.last_pong_time = now()

	# Returns Job ids to revive
	async def cleanup(self):
		if self in processor_websockets:
			processor_websockets.remove(self)
		for job_id in self.job_ids:
			try:
				jobs[job_id].fails += 1
				# Delete the Job from the list when it has multiple failures
				if jobs[job_id].fails > MAX_FAILURES:
					# Send and error to the Client
					jobs[job_id].websocket.result_available(job_id,None,True)
					del jobs[job_id]
				else:
					await job_id_queue.put(job_id)
			except KeyError:
				pass
		del self.job_ids[:]

	async def onClose(self, wasClean, code, reason):
		await self.cleanup()
		lg.debug("Processor closed. Cleanly?: {}. Code: {}, Reason: {}".format(wasClean, code, reason))

class ClientProtocol(WebSocketServerProtocol):

	def __init__(self, *args, **kwargs):
		super().__init__(*args, **kwargs)
		self.job_ids = [] # For saving the Job ids requested by this Client
		self.buff = [] # For buffering the Job results
		self.buff_size = 1
		self.ip = None
		self.ip_limit = None

	def onConnect(self, request):
		self.ip = request.peer.split(':')[1]
		if self.ip not in ip_limit:
			ip_limit[self.ip] = IpLimit(IP_DURATION_LIMIT,IP_JOB_COUNT_LIMIT)
		# Each IP has a unique instance of IpLimit
		self.ip_limit = ip_limit[self.ip]

	def onOpen(self):
		client_websockets.add(self)

	def result_available(self,job_id,result,error):
		if job_id in jobs:
			if error:
				try:
					message = { "type": "result",
								"results": [[None,jobs[job_id].identity]],
								"error": True }
					self.sendMessage(json.dumps(message).encode('utf-8'),False)
				except:
					pass # None of our business!
			else:
				self.buff.append([result,jobs[job_id].identity])
				if len(self.buff) >= self.buff_size:
					self.flush()

	def flush(self):
		try:
			message = { "type": "result",
						"results": self.buff,
						"error": False }
			self.sendMessage(json.dumps(message).encode('utf-8'),False)
			del self.buff[:]
		except:
			pass # None of our business!

	def info(self):
		try:
			message = { "type": "info",
						"processorsCount": len(processor_websockets),
						"jobsCount": len(jobs) }
			self.sendMessage(json.dumps(message).encode('utf-8'),False)
		except:
			pass # None of our business!

	def limit(self):
		try:
			remaining = self.ip_limit.duration_limit - now() + self.ip_limit.expiry_time
			message = { "type": "limit",
						"remaining": remaining }
			self.sendMessage(json.dumps(message).encode('utf-8'),False)
		except:
			pass # None of our business!

	async def new_job(self,code,args,identity,process_id):
		global job_id_counter

		if self.ip_limit.count < self.ip_limit.count_limit:
			jobs[job_id_counter] = Job(code,self,args,identity,process_id)
			await job_id_queue.put(job_id_counter)
			self.job_ids.append(job_id_counter)
			job_id_counter += 1
			self.ip_limit.count += 1
			return False
		else:
			self.ip_limit.expiry_time = now()
			self.limit()
			return True

	async def onMessage(self, payload, isBinary):
		global process_id_counter
		msg = json.loads(payload.decode('utf8'))

		if msg["type"] == "flush":
			self.flush()

		elif msg["type"] == "info":
			self.info()

		elif msg["type"] == "set":
			if msg["property"] == "bufferSize":
				self.buff_size = msg["value"]
				self.flush()

		else:
			if self.ip_limit.expiry_time:
				if now() - self.ip_limit.expiry_time > self.ip_limit.duration_limit:
					self.ip_limit.expiry_time = None
					self.ip_limit.count = 0
				else:
					self.limit()
					return

			if msg["type"] == "run":
				await self.new_job(msg["code"],msg["args"],msg["id"],process_id_counter)
				process_id_counter += 1

			elif msg["type"] == "for":
				for i in range(msg["start"],msg["end"]):
					if await self.new_job(msg["code"],[i] + msg["extraArgs"],msg["id"],process_id_counter):
						break
				process_id_counter += 1

			elif msg["type"] == "forEach":
				for arg in msg["argsList"]:
					if await self.new_job(msg["code"],[arg] + msg["extraArgs"],msg["id"],process_id_counter):
						break
				process_id_counter += 1

	async def onClose(self, wasClean, code, reason):
		if self in client_websockets:
			client_websockets.remove(self)

		for j in self.job_ids:
			try:
				del jobs[j]
			except KeyError:
				pass
		del self.job_ids[:]

# Balance the Jobs between Processors
async def balancer():
	while True:
		job_id = await job_id_queue.get()

		# Wait for at least one Processor
		await processor_exists.acquire()
		await processor_exists.wait_for(lambda:len(processor_websockets) > 0)
		processor_exists.release()

		websocket = random.sample(processor_websockets, 1)[0]
		if len(websocket.job_ids) < MAX_JOB_PER_PROCESSOR:
			if job_id in jobs:
				try:
					websocket.send_job(job_id, jobs[job_id].code, jobs[job_id].args, jobs[job_id].process_id)
				except:
					lg.debug("An exception occurred while sending a Job.")
					await job_id_queue.put(job_id) # Revive the job
		else:
			await job_id_queue.put(job_id) # Revive the job
			await asyncio.sleep(MAX_JOB_REACHED_REST_TIME) # It seems server is busy, sleep for a second!

# Close not-responding sockets and revive Jobs
async def watcher():
	while True:
		must_close = []
		for ws in processor_websockets:
			if ws.last_ping_time:
				if not ws.last_pong_time or ws.last_pong_time < ws.last_ping_time:
					elapsed = now() - ws.last_ping_time
					if elapsed > MAX_PONG_WAIT_TIME:
						must_close.append(ws)
		for ws in must_close:
			lg.debug("Processor took too long to respond! Closing...")
			await ws.cleanup()
			ws.sendClose()
		await asyncio.sleep(PING_INTERVAL)

# Generating idle processes
async def idle():
	while True:
		for ws in processor_websockets:
			try:
				ws.send_job(IDLE_ID, IDLE_SCRIPT, IDLE_ARGS, IDLE_PROCESS_ID)
			except:
				pass # No need to revive Idle Jobs!
		await asyncio.sleep(IDLE_INTERVAL)

if __name__ == '__main__':
	ssl_ctx = None
	if os.path.isfile(SSL_CERT_FILE) and os.path.isfile(SSL_KEY_FILE):
		import ssl
		ssl_ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
		ssl_ctx.load_cert_chain(certfile = SSL_CERT_FILE, keyfile = SSL_KEY_FILE)

	loop = asyncio.get_event_loop()

	clientFactory = WebSocketServerFactory()
	clientFactory.protocol = ClientProtocol
	clientCoro = loop.create_server(clientFactory, CLIENTS_SERVER_IP, CLIENTS_SERVER_PORT, ssl=ssl_ctx)

	processorFactory = WebSocketServerFactory()
	processorFactory.protocol = ProcessorProtocol
	processorCoro = loop.create_server(processorFactory, PROCESSORS_SERVER_IP, PROCESSORS_SERVER_PORT, ssl=ssl_ctx)

	all_tasks = asyncio.gather(clientCoro,processorCoro,balancer(),watcher(),idle())

	try:
		server = loop.run_until_complete(all_tasks)
	except KeyboardInterrupt:
		# In order to close properly when CTRL-C has been pressed
		all_tasks.cancel()
		loop.run_forever()
		all_tasks.exception()
	finally:
		loop.close()
<commit_msg>managing group of jobs as processes and adding process failure strategies<commit_after>#!/usr/bin/env python3

from autobahn.asyncio.websocket \
	import WebSocketServerProtocol, WebSocketServerFactory

import logging as lg
import asyncio
import json
import random
import time
import os
import sys

lg.basicConfig(stream = sys.stdout , level = lg.DEBUG)

PROCESSORS_SERVER_IP = '0.0.0.0'
PROCESSORS_SERVER_PORT = 12121

CLIENTS_SERVER_IP = '0.0.0.0'
CLIENTS_SERVER_PORT = 21212

MAX_PONG_WAIT_TIME = 5000 # Milliseconds
PING_INTERVAL = 5 # Seconds

MAX_JOB_PER_PROCESSOR = 8
MAX_JOB_REACHED_REST_TIME = 0.1 # Seconds

IDLE_ID = None
IDLE_PROCESS_ID = None
IDLE_SCRIPT = "function(){return 123;}"
IDLE_ARGS = []
IDLE_RESULT = 123
IDLE_INTERVAL = 30 # Seconds

MAXIMUM_JOB_CODE_REPEAT_COUNT = 32

MAX_FAILURES = 4
MAX_PROCESS_FAILURES = MAX_FAILURES * 5

IP_JOB_COUNT_LIMIT = 1000
IP_DURATION_LIMIT = 30000 # Milliseconds

SSL_CERT_FILE = '/etc/letsencrypt/live/pooljs.ir/cert.pem'
SSL_KEY_FILE = '/etc/letsencrypt/live/pooljs.ir/privkey.pem'

class Process:

	def __init__(self,identity,code,websocket):
		self.identity = identity
		self.code = code
		self.websocket = websocket # Job owner websocket
		self.job_ids = []
		self.fails = 0

class Job:

	def __init__(self,identity,process,args):
		self.identity = identity # Every Job has an identity in the client side
		self.process = process
		self.args = args
		self.fails = 0 # Jobs will be deleted from the list when having multiple failures (MAX_FAILURES)

class IpLimit:

	def __init__(self,duration_limit,count_limit):
		self.duration_limit = duration_limit
		self.count_limit = count_limit
		self.expiry_time = None
		self.count = 0 # Count of Jobs ran by this IP since the last reset

ip_limit = {} # Mapping IP as strings to IpLimits
job_id_queue = asyncio.Queue() # Queue of requested Job ids
job_id_counter = 0 # A counter for generating Job ids
process_id_counter = 0 # A counter for generating Process ids
jobs = {} # Mapping Job ids as integers to Jobs
processor_websockets = set()
client_websockets = set()
processor_exists = asyncio.Condition()

# Get current time in seconds
def now():
	return int(time.time() * 1000.0)

class ProcessorProtocol(WebSocketServerProtocol):

	def __init__(self, *args, **kwargs):
		super().__init__(*args, **kwargs)
		self.job_ids = []
		self.last_ping_time = None
		self.last_pong_time = None
		self.last_process_id = None
		self.last_process_id_repeat_count = 0

	def send_job(self,job_id,code,args,process_id):
		if process_id == self.last_process_id:
			self.last_process_id_repeat_count += 1
		else:
			self.last_process_id = process_id
			self.last_process_id_repeat_count = 1
		message = { "id": job_id,
					"process_id": process_id,
					"args": args }
		if self.last_process_id_repeat_count <= MAXIMUM_JOB_CODE_REPEAT_COUNT:
			message["code"] = code
		if not self.last_ping_time or (self.last_pong_time and self.last_ping_time < self.last_pong_time):
			self.last_ping_time = now()
		self.sendMessage(json.dumps(message).encode('utf-8'),False)
		if job_id is not None: # Do not add Idle Jobs to the list
			self.job_ids.append(job_id)

	async def onOpen(self):
		# Notify the balancer a new Processor has been added
		await processor_exists.acquire()
		processor_websockets.add(self)
		processor_exists.notify_all()
		processor_exists.release()

	async def job_fail(job_id):
		jobs[job_id].fails += 1
		jobs[job_id].process.fails += 1
		# Delete the Job from the list when it has multiple failures
		if jobs[job_id].fails > MAX_FAILURES:
			# Send and error to the Client
			jobs[job_id].process.websocket.result_available(job_id,None,True)
			jobs[job_id].process.job_ids.remove(job_id)
			del jobs[job_id]
		else:
			await job_id_queue.put(job_id)
		if jobs[job_id].process.fails > MAX_PROCESS_FAILURES:
			for jid in jobs[job_id].process.job_ids:
				del jobs[jid]

	async def onMessage(self, payload, isBinary):
		msg = json.loads(payload.decode('utf8'))
		job_id = msg["id"]
		if job_id is not None:
			self.last_pong_time = now()
			try:
				if not msg["error"]:
					jobs[job_id].process.websocket.result_available(job_id,msg["result"],False)
					jobs[job_id].process.websocket.job_ids.remove(job_id)
					jobs[job_id].process.job_ids.remove(job_id)
					del jobs[job_id]
				else:
					await job_fail(job_id)
			except KeyError:
				pass
			if job_id in self.job_ids:
				self.job_ids.remove(job_id)
		else: # This is an Idle Job!
			if msg["result"] == IDLE_RESULT:
				self.last_pong_time = now()

	# Returns Job ids to revive
	async def cleanup(self):
		if self in processor_websockets:
			processor_websockets.remove(self)
		for job_id in self.job_ids:
			try:
				await job_fail(job_id)
			except KeyError:
				pass
		del self.job_ids[:]

	async def onClose(self, wasClean, code, reason):
		await self.cleanup()
		lg.debug("Processor closed. Cleanly?: {}. Code: {}, Reason: {}".format(wasClean, code, reason))

class ClientProtocol(WebSocketServerProtocol):

	def __init__(self, *args, **kwargs):
		super().__init__(*args, **kwargs)
		self.job_ids = [] # For saving the Job ids requested by this Client
		self.buff = [] # For buffering the Job results
		self.buff_size = 1
		self.ip = None
		self.ip_limit = None

	def onConnect(self, request):
		self.ip = request.peer.split(':')[1]
		if self.ip not in ip_limit:
			ip_limit[self.ip] = IpLimit(IP_DURATION_LIMIT,IP_JOB_COUNT_LIMIT)
		# Each IP has a unique instance of IpLimit
		self.ip_limit = ip_limit[self.ip]

	def onOpen(self):
		client_websockets.add(self)

	def result_available(self,job_id,result,error):
		if job_id in jobs:
			if error:
				try:
					message = { "type": "result",
								"results": [[None,jobs[job_id].identity]],
								"error": True }
					self.sendMessage(json.dumps(message).encode('utf-8'),False)
				except:
					pass # None of our business!
			else:
				self.buff.append([result,jobs[job_id].identity])
				if len(self.buff) >= self.buff_size:
					self.flush()

	def flush(self):
		try:
			message = { "type": "result",
						"results": self.buff,
						"error": False }
			self.sendMessage(json.dumps(message).encode('utf-8'),False)
			del self.buff[:]
		except:
			pass # None of our business!

	def info(self):
		try:
			message = { "type": "info",
						"processorsCount": len(processor_websockets),
						"jobsCount": len(jobs) }
			self.sendMessage(json.dumps(message).encode('utf-8'),False)
		except:
			pass # None of our business!

	def limit(self):
		try:
			remaining = self.ip_limit.duration_limit - now() + self.ip_limit.expiry_time
			message = { "type": "limit",
						"remaining": remaining }
			self.sendMessage(json.dumps(message).encode('utf-8'),False)
		except:
			pass # None of our business!

	async def new_job(self,identity,process,args):
		global job_id_counter

		if self.ip_limit.count < self.ip_limit.count_limit:
			jobs[job_id_counter] = Job(identity,process,args)
			process.job_ids.append(job_id_counter)
			await job_id_queue.put(job_id_counter)
			self.job_ids.append(job_id_counter)
			job_id_counter += 1
			self.ip_limit.count += 1
			return False
		else:
			self.ip_limit.expiry_time = now()
			self.limit()
			return True

	async def onMessage(self, payload, isBinary):
		global process_id_counter
		msg = json.loads(payload.decode('utf8'))

		if msg["type"] == "flush":
			self.flush()

		elif msg["type"] == "info":
			self.info()

		elif msg["type"] == "set":
			if msg["property"] == "bufferSize":
				self.buff_size = msg["value"]
				self.flush()

		else:
			if self.ip_limit.expiry_time:
				if now() - self.ip_limit.expiry_time > self.ip_limit.duration_limit:
					self.ip_limit.expiry_time = None
					self.ip_limit.count = 0
				else:
					self.limit()
					return

			if msg["type"] == "run":
				proc = Process(process_id_counter,msg["code"],self)
				await self.new_job(msg["id"],proc,msg["args"])
				process_id_counter += 1

			elif msg["type"] == "for":
				proc = Process(process_id_counter,msg["code"],self)
				for i in range(msg["start"],msg["end"]):
					if await self.new_job(msg["id"],proc,[i] + msg["extraArgs"]):
						break
				process_id_counter += 1

			elif msg["type"] == "forEach":
				proc = Process(process_id_counter,msg["code"],self)
				for arg in msg["argsList"]:
					if await self.new_job(msg["id"],proc,[arg] + msg["extraArgs"]):
						break
				process_id_counter += 1

	async def onClose(self, wasClean, code, reason):
		if self in client_websockets:
			client_websockets.remove(self)

		for j in self.job_ids:
			try:
				del jobs[j]
			except KeyError:
				pass
		del self.job_ids[:]

# Balance the Jobs between Processors
async def balancer():
	while True:
		job_id = await job_id_queue.get()

		# Wait for at least one Processor
		await processor_exists.acquire()
		await processor_exists.wait_for(lambda:len(processor_websockets) > 0)
		processor_exists.release()

		websocket = random.sample(processor_websockets, 1)[0]
		if len(websocket.job_ids) < MAX_JOB_PER_PROCESSOR:
			if job_id in jobs:
				try:
					websocket.send_job(job_id, jobs[job_id].process.code, jobs[job_id].args, jobs[job_id].process.identity)
				except:
					lg.debug("An exception occurred while sending a Job.")
					await job_id_queue.put(job_id) # Revive the job
		else:
			await job_id_queue.put(job_id) # Revive the job
			await asyncio.sleep(MAX_JOB_REACHED_REST_TIME) # It seems server is busy, sleep for a second!

# Close not-responding sockets and revive Jobs
async def watcher():
	while True:
		must_close = []
		for ws in processor_websockets:
			if ws.last_ping_time:
				if not ws.last_pong_time or ws.last_pong_time < ws.last_ping_time:
					elapsed = now() - ws.last_ping_time
					if elapsed > MAX_PONG_WAIT_TIME:
						must_close.append(ws)
		for ws in must_close:
			lg.debug("Processor took too long to respond! Closing...")
			await ws.cleanup()
			ws.sendClose()
		await asyncio.sleep(PING_INTERVAL)

# Generating idle processes
async def idle():
	while True:
		for ws in processor_websockets:
			try:
				ws.send_job(IDLE_ID, IDLE_SCRIPT, IDLE_ARGS, IDLE_PROCESS_ID)
			except:
				pass # No need to revive Idle Jobs!
		await asyncio.sleep(IDLE_INTERVAL)

if __name__ == '__main__':
	ssl_ctx = None
	if os.path.isfile(SSL_CERT_FILE) and os.path.isfile(SSL_KEY_FILE):
		import ssl
		ssl_ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
		ssl_ctx.load_cert_chain(certfile = SSL_CERT_FILE, keyfile = SSL_KEY_FILE)

	loop = asyncio.get_event_loop()

	clientFactory = WebSocketServerFactory()
	clientFactory.protocol = ClientProtocol
	clientCoro = loop.create_server(clientFactory, CLIENTS_SERVER_IP, CLIENTS_SERVER_PORT, ssl=ssl_ctx)

	processorFactory = WebSocketServerFactory()
	processorFactory.protocol = ProcessorProtocol
	processorCoro = loop.create_server(processorFactory, PROCESSORS_SERVER_IP, PROCESSORS_SERVER_PORT, ssl=ssl_ctx)

	all_tasks = asyncio.gather(clientCoro,processorCoro,balancer(),watcher(),idle())

	try:
		server = loop.run_until_complete(all_tasks)
	except KeyboardInterrupt:
		# In order to close properly when CTRL-C has been pressed
		all_tasks.cancel()
		loop.run_forever()
		all_tasks.exception()
	finally:
		loop.close()
<|endoftext|>


<commit_before>from __future__ import unicode_literals
import json
import os
import random
import contextlib
import shutil
import pytest
import tempfile
from pathlib import Path

import pathlib
from ...gold import GoldParse
from ...pipeline import EntityRecognizer
from ...lang.en import English

try:
    unicode
except NameError:
    unicode = str

@pytest.fixture
def train_data():
    return [
            ["hey",[]],
            ["howdy",[]],
            ["hey there",[]],
            ["hello",[]],
            ["hi",[]],
            ["i'm looking for a place to eat",[]],
            ["i'm looking for a place in the north of town",[[31,36,"location"]]],
            ["show me chinese restaurants",[[8,15,"cuisine"]]],
            ["show me chines restaurants",[[8,14,"cuisine"]]],
            ["yes",[]],
            ["yep",[]],
            ["yeah",[]],
            ["show me a mexican place in the centre",[[31,37,"location"], [10,17,"cuisine"]]],
            ["bye",[]],["goodbye",[]],
            ["good bye",[]],
            ["stop",[]],
            ["end",[]],
            ["i am looking for an indian spot",[[20,26,"cuisine"]]],
            ["search for restaurants",[]],
            ["anywhere in the west",[[16,20,"location"]]],
            ["central indian restaurant",[[0,7,"location"],[8,14,"cuisine"]]],
            ["indeed",[]],
            ["that's right",[]],
            ["ok",[]],
            ["great",[]]
    ]

@pytest.fixture
def additional_entity_types():
    return ['cuisine', 'location']

@contextlib.contextmanager
def temp_save_model(model):
    model_dir = Path(tempfile.mkdtemp())
    # store the fine tuned model
    with (model_dir / "config.json").open('w') as file_:
        data = json.dumps(model.cfg)
        if not isinstance(data, unicode):
            data = data.decode('utf8')
        file_.write(data)
    model.model.dump((model_dir / 'model').as_posix())
    yield model_dir
    shutil.rmtree(model_dir.as_posix())

@pytest.mark.models('en')
def test_issue910(EN, train_data, additional_entity_types):
    '''Test that adding entities and resuming training works passably OK.
    There are two issues here:

    1) We have to readd labels. This isn't very nice.
    2) There's no way to set the learning rate for the weight update, so we
        end up out-of-scale, causing it to learn too fast.
    '''
    doc = EN(u"I am looking for a restaurant in Berlin")
    ents_before_train = [(ent.label_, ent.text) for ent in doc.ents]
    # Fine tune the ner model
    for entity_type in additional_entity_types:
        nlp.entity.add_label(entity_type)

    nlp.entity.model.learn_rate = 0.001
    for itn in range(10):
        random.shuffle(train_data)
        for raw_text, entity_offsets in train_data:
            doc = nlp.make_doc(raw_text)
            nlp.tagger(doc)
            gold = GoldParse(doc, entities=entity_offsets)
            loss = nlp.entity.update(doc, gold)

    with temp_save_model(nlp.entity) as model_dir:
        # Load the fine tuned model
        loaded_ner = EntityRecognizer.load(model_dir, nlp.vocab)

    for raw_text, entity_offsets in train_data:
        doc = nlp.make_doc(raw_text)
        nlp.tagger(doc)
        loaded_ner(doc)
        ents = {(ent.start_char, ent.end_char): ent.label_ for ent in doc.ents}
        for start, end, label in entity_offsets:
            if (start, end) not in ents:
                print(ents)
            assert ents[(start, end)] == label
<commit_msg>Fix typo<commit_after>from __future__ import unicode_literals
import json
import os
import random
import contextlib
import shutil
import pytest
import tempfile
from pathlib import Path

import pathlib
from ...gold import GoldParse
from ...pipeline import EntityRecognizer
from ...lang.en import English

try:
    unicode
except NameError:
    unicode = str

@pytest.fixture
def train_data():
    return [
            ["hey",[]],
            ["howdy",[]],
            ["hey there",[]],
            ["hello",[]],
            ["hi",[]],
            ["i'm looking for a place to eat",[]],
            ["i'm looking for a place in the north of town",[[31,36,"location"]]],
            ["show me chinese restaurants",[[8,15,"cuisine"]]],
            ["show me chines restaurants",[[8,14,"cuisine"]]],
            ["yes",[]],
            ["yep",[]],
            ["yeah",[]],
            ["show me a mexican place in the centre",[[31,37,"location"], [10,17,"cuisine"]]],
            ["bye",[]],["goodbye",[]],
            ["good bye",[]],
            ["stop",[]],
            ["end",[]],
            ["i am looking for an indian spot",[[20,26,"cuisine"]]],
            ["search for restaurants",[]],
            ["anywhere in the west",[[16,20,"location"]]],
            ["central indian restaurant",[[0,7,"location"],[8,14,"cuisine"]]],
            ["indeed",[]],
            ["that's right",[]],
            ["ok",[]],
            ["great",[]]
    ]

@pytest.fixture
def additional_entity_types():
    return ['cuisine', 'location']

@contextlib.contextmanager
def temp_save_model(model):
    model_dir = Path(tempfile.mkdtemp())
    # store the fine tuned model
    with (model_dir / "config.json").open('w') as file_:
        data = json.dumps(model.cfg)
        if not isinstance(data, unicode):
            data = data.decode('utf8')
        file_.write(data)
    model.model.dump((model_dir / 'model').as_posix())
    yield model_dir
    shutil.rmtree(model_dir.as_posix())

@pytest.mark.models('en')
def test_issue910(EN, train_data, additional_entity_types):
    '''Test that adding entities and resuming training works passably OK.
    There are two issues here:

    1) We have to readd labels. This isn't very nice.
    2) There's no way to set the learning rate for the weight update, so we
        end up out-of-scale, causing it to learn too fast.
    '''
    nlp = EN
    doc = nlp(u"I am looking for a restaurant in Berlin")
    ents_before_train = [(ent.label_, ent.text) for ent in doc.ents]
    # Fine tune the ner model
    for entity_type in additional_entity_types:
        nlp.entity.add_label(entity_type)

    nlp.entity.model.learn_rate = 0.001
    for itn in range(10):
        random.shuffle(train_data)
        for raw_text, entity_offsets in train_data:
            doc = nlp.make_doc(raw_text)
            nlp.tagger(doc)
            gold = GoldParse(doc, entities=entity_offsets)
            loss = nlp.entity.update(doc, gold)

    with temp_save_model(nlp.entity) as model_dir:
        # Load the fine tuned model
        loaded_ner = EntityRecognizer.load(model_dir, nlp.vocab)

    for raw_text, entity_offsets in train_data:
        doc = nlp.make_doc(raw_text)
        nlp.tagger(doc)
        loaded_ner(doc)
        ents = {(ent.start_char, ent.end_char): ent.label_ for ent in doc.ents}
        for start, end, label in entity_offsets:
            if (start, end) not in ents:
                print(ents)
            assert ents[(start, end)] == label
<|endoftext|>


<commit_before>NAME = "com_googlesource_gerrit_bazlets"

def load_bazlets(
        commit,
        local_path = None):
    if not local_path:
        native.git_repository(
            name = NAME,
            remote = "https://gerrit.googlesource.com/bazlets",
            commit = commit,
        )
    else:
        native.local_repository(
            name = NAME,
            path = local_path,
        )
<commit_msg>bazlets: Replace native.git_repository with skylark rule<commit_after>load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")

NAME = "com_googlesource_gerrit_bazlets"

def load_bazlets(
        commit,
        local_path = None):
    if not local_path:
        git_repository(
            name = NAME,
            remote = "https://gerrit.googlesource.com/bazlets",
            commit = commit,
        )
    else:
        native.local_repository(
            name = NAME,
            path = local_path,
        )
<|endoftext|>


<commit_before>from __future__ import print_function
from copy import copy

from django.conf import settings
from django.core.management import call_command
from django.core.management.base import BaseCommand

class Command(BaseCommand):
    help = "Call 'migrate' for each configured database"

    def add_arguments(self, parser):
        parser.add_argument('app_label', nargs='?',
            help='App label of an application to synchronize the state.')
        parser.add_argument('migration_name', nargs='?',
            help=(
                'Database state will be brought to the state after that '
                'migration. Use the name "zero" to unapply all migrations.'
            ),
        )
        parser.add_argument('--noinput', action='store_false', dest='interactive', default=True,
            help='Tells Django to NOT prompt the user for input of any kind.')
        parser.add_argument('--fake', action='store_true', dest='fake', default=False,
            help='Mark migrations as run without actually running them.')
        parser.add_argument('--list', '-l', action='store_true', dest='list', default=False,
            help='Show a list of all known migrations and which are applied.')

    def handle(self, app_label, migration_name, **options):
        args = []
        if app_label is not None:
            args.append(app_label)
        if migration_name is not None:
            args.append(migration_name)
        for db_alias in settings.DATABASES.keys():
            print('\n======================= Migrating DB: {} ======================='.format(db_alias))
            if not settings.DATABASES[db_alias].get('MIGRATE', True):
                print("Skipping.")
            call_options = copy(options)
            call_options['database'] = db_alias
            call_command(
                'migrate',
                *args,
                **call_options
            )
<commit_msg>Actually skip migration<commit_after>from __future__ import print_function
from copy import copy

from django.conf import settings
from django.core.management import call_command
from django.core.management.base import BaseCommand

class Command(BaseCommand):
    help = "Call 'migrate' for each configured database"

    def add_arguments(self, parser):
        parser.add_argument('app_label', nargs='?',
            help='App label of an application to synchronize the state.')
        parser.add_argument('migration_name', nargs='?',
            help=(
                'Database state will be brought to the state after that '
                'migration. Use the name "zero" to unapply all migrations.'
            ),
        )
        parser.add_argument('--noinput', action='store_false', dest='interactive', default=True,
            help='Tells Django to NOT prompt the user for input of any kind.')
        parser.add_argument('--fake', action='store_true', dest='fake', default=False,
            help='Mark migrations as run without actually running them.')
        parser.add_argument('--list', '-l', action='store_true', dest='list', default=False,
            help='Show a list of all known migrations and which are applied.')

    def handle(self, app_label, migration_name, **options):
        args = []
        if app_label is not None:
            args.append(app_label)
        if migration_name is not None:
            args.append(migration_name)
        for db_alias in settings.DATABASES.keys():
            print('\n======================= Migrating DB: {} ======================='.format(db_alias))
            if not settings.DATABASES[db_alias].get('MIGRATE', True):
                print("Skipping.")
                continue
            call_options = copy(options)
            call_options['database'] = db_alias
            call_command(
                'migrate',
                *args,
                **call_options
            )
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import logging

from dateutil import parser as dateutil
from lxml import etree
from lxml import objectify

from speeches.importers.import_base import ImporterBase
from speeches.models import Section, Speech, Speaker

logger = logging.getLogger(__name__)

class ImportAkomaNtoso (ImporterBase):
    start_date = None

    def import_document(self, document_path):
        tree = objectify.parse(document_path)
        self.xml = tree.getroot()
        self.ns = self.xml.nsmap.get(None, None)
        return self.parse_document()

    def parse_document(self):
        debate = self.xml.debate

        if self.ns:
            people = debate.findall('an:meta/an:references/an:TLCPerson', namespaces={'an': self.ns})
        else:
            people = debate.findall('meta/references/TLCPerson')
        if people is None: people = []
        for person in people:
            id = person.get('id')
            href = person.get('href')
            try:
                speaker = Speaker.objects.get(instance=self.instance, identifiers__identifier=href)
            except Speaker.DoesNotExist:
                speaker = Speaker(instance=self.instance, name=person.get('showAs'))
                if self.commit:
                    speaker.save()
                    speaker.identifiers.create(identifier=href, scheme='Akoma Ntoso import')

            self.speakers[id] = speaker

        if self.ns:
            docDate = debate.find('an:coverPage//an:docDate|an:preface//an:docDate', namespaces={'an': self.ns})
        else:
            docDate = debate.find('coverPage//docDate|preface//docDate')
        if docDate is not None:
            self.start_date = dateutil.parse(docDate.get('date'))

        if self.ns:
            docTitle = debate.find('an:coverPage//an:docTitle|an:preface//an:docTitle', namespaces={'an': self.ns})
        else:
            docTitle = debate.find('coverPage//docTitle|preface//docTitle')
        if docTitle is None:
            section = None
        else:
            section = self.make(Section, parent=None, title=docTitle.text)

        self.visit(debate.debateBody, section)

    def get_tag(self, node):
        return etree.QName(node.tag).localname

    def get_text(self, node):
        paras = [ node.text ]
        paras += [ etree.tostring(child, encoding='utf-8')
                    for child in node.iterchildren()
                    if self.get_tag(child) not in ('num', 'heading', 'subheading', 'from') ]
        return ''.join(filter(None, paras))

    def construct_title(self, node):
        title = []
        if hasattr(node, 'num'):
            title.append(node.num.text)
        if hasattr(node, 'heading'):
            title.append(node.heading.text)
        if hasattr(node, 'subheading'):
            title.append(node.subheading.text)
        title = ' '.join(title)
        return title

    def construct_datetime(self, time):
        if not time:
            return (None, None)
        dt = dateutil.parse(time)
        return dt.date(), dt.time()

    def get_speaker(self, child):
        display_name = child['from'].text
        by_ref = child.get('by')
        speaker = self.speakers[by_ref[1:]]
        return speaker, display_name

    def handle_tag(self, node, section):
        """If we need to do something out of the ordinary handling elements,
        subclass it here"""
        return False

    def visit(self, node, section):
        for child in node.iterchildren():
            tagname = self.get_tag(child)
            if tagname in ('num', 'heading', 'subheading'):
                # this will already have been extracted
                continue
            if tagname in ('debateSection', 'administrationOfOath', 'rollCall',
                    'prayers', 'oralStatements', 'writtenStatements',
                    'personalStatements', 'ministerialStatements',
                    'resolutions', 'nationalInterest', 'declarationOfVote',
                    'communication', 'petitions', 'papers', 'noticesOfMotion',
                    'questions', 'address', 'proceduralMotions',
                    'pointOfOrder', 'adjournment'):
                title = self.construct_title(child)
                childSection = self.make(Section, parent=section, title=title)
                self.visit(child, childSection)
            elif tagname in ('speech', 'question', 'answer'):
                title = self.construct_title(child)
                text = self.get_text(child)
                start_date, start_time = self.construct_datetime(child.get('startTime'))
                end_date, end_time = self.construct_datetime(child.get('endTime'))
                speaker, display_name = self.get_speaker(child)
                speech = self.make(Speech,
                        section = section,
                        title = title,
                        start_date = start_date or self.start_date,
                        start_time = start_time,
                        end_date = end_date,
                        end_time = end_time,
                        text = text,
                        speaker = speaker,
                        speaker_display = display_name,
                )
            elif tagname in ('scene', 'narrative', 'summary', 'other'):
                text = self.get_text(child)

                speech = self.make(Speech,
                        section = section,
                        start_date = self.start_date,
                        text = text,
                )
            else:
                success = self.handle_tag(child, section)
                if not success:
                    logger.error('%s unrecognised, "%s" - %s' % (child.tag, child, self.get_text(child)))
<commit_msg>lxml find() doesn't seem to respect XPath's pipe character<commit_after># -*- coding: utf-8 -*-

import logging

from dateutil import parser as dateutil
from lxml import etree
from lxml import objectify

from speeches.importers.import_base import ImporterBase
from speeches.models import Section, Speech, Speaker

logger = logging.getLogger(__name__)

class ImportAkomaNtoso (ImporterBase):
    start_date = None

    def import_document(self, document_path):
        tree = objectify.parse(document_path)
        self.xml = tree.getroot()
        self.ns = self.xml.nsmap.get(None, None)
        return self.parse_document()

    def parse_document(self):
        debate = self.xml.debate

        if self.ns:
            people = debate.findall('an:meta/an:references/an:TLCPerson', namespaces={'an': self.ns})
        else:
            people = debate.findall('meta/references/TLCPerson')
        if people is None: people = []
        for person in people:
            id = person.get('id')
            href = person.get('href')
            try:
                speaker = Speaker.objects.get(instance=self.instance, identifiers__identifier=href)
            except Speaker.DoesNotExist:
                speaker = Speaker(instance=self.instance, name=person.get('showAs'))
                if self.commit:
                    speaker.save()
                    speaker.identifiers.create(identifier=href, scheme='Akoma Ntoso import')

            self.speakers[id] = speaker

        if self.ns:
            docDate = debate.xpath('an:coverPage//an:docDate|an:preface//an:docDate', namespaces={'an': self.ns})
        else:
            docDate = debate.xpath('coverPage//docDate|preface//docDate')
        if docDate:
            self.start_date = dateutil.parse(docDate[0].get('date'))

        if self.ns:
            docTitle = debate.xpath('an:coverPage//an:docTitle|an:preface//an:docTitle', namespaces={'an': self.ns})
        else:
            docTitle = debate.xpath('coverPage//docTitle|preface//docTitle')
        if docTitle:
            section = self.make(Section, parent=None, title=docTitle[0].text)
        else:
            section = None

        self.visit(debate.debateBody, section)

    def get_tag(self, node):
        return etree.QName(node.tag).localname

    def get_text(self, node):
        paras = [ node.text ]
        paras += [ etree.tostring(child, encoding='utf-8')
                    for child in node.iterchildren()
                    if self.get_tag(child) not in ('num', 'heading', 'subheading', 'from') ]
        return ''.join(filter(None, paras))

    def construct_title(self, node):
        title = []
        if hasattr(node, 'num'):
            title.append(node.num.text)
        if hasattr(node, 'heading'):
            title.append(node.heading.text)
        if hasattr(node, 'subheading'):
            title.append(node.subheading.text)
        title = ' '.join(title)
        return title

    def construct_datetime(self, time):
        if not time:
            return (None, None)
        dt = dateutil.parse(time)
        return dt.date(), dt.time()

    def get_speaker(self, child):
        display_name = child['from'].text
        by_ref = child.get('by')
        speaker = self.speakers[by_ref[1:]]
        return speaker, display_name

    def handle_tag(self, node, section):
        """If we need to do something out of the ordinary handling elements,
        subclass it here"""
        return False

    def visit(self, node, section):
        for child in node.iterchildren():
            tagname = self.get_tag(child)
            if tagname in ('num', 'heading', 'subheading'):
                # this will already have been extracted
                continue
            if tagname in ('debateSection', 'administrationOfOath', 'rollCall',
                    'prayers', 'oralStatements', 'writtenStatements',
                    'personalStatements', 'ministerialStatements',
                    'resolutions', 'nationalInterest', 'declarationOfVote',
                    'communication', 'petitions', 'papers', 'noticesOfMotion',
                    'questions', 'address', 'proceduralMotions',
                    'pointOfOrder', 'adjournment'):
                title = self.construct_title(child)
                childSection = self.make(Section, parent=section, title=title)
                self.visit(child, childSection)
            elif tagname in ('speech', 'question', 'answer'):
                title = self.construct_title(child)
                text = self.get_text(child)
                start_date, start_time = self.construct_datetime(child.get('startTime'))
                end_date, end_time = self.construct_datetime(child.get('endTime'))
                speaker, display_name = self.get_speaker(child)
                speech = self.make(Speech,
                        section = section,
                        title = title,
                        start_date = start_date or self.start_date,
                        start_time = start_time,
                        end_date = end_date,
                        end_time = end_time,
                        text = text,
                        speaker = speaker,
                        speaker_display = display_name,
                )
            elif tagname in ('scene', 'narrative', 'summary', 'other'):
                text = self.get_text(child)

                speech = self.make(Speech,
                        section = section,
                        start_date = self.start_date,
                        text = text,
                )
            else:
                success = self.handle_tag(child, section)
                if not success:
                    logger.error('%s unrecognised, "%s" - %s' % (child.tag, child, self.get_text(child)))
<|endoftext|>


<commit_before>'''
Copyright 2015 by Tobias Houska
This file is part of Statistical Parameter Estimation Tool (SPOTPY).

:author: Tobias Houska

This example implements the external hydrological model HYMOD into SPOTPY.
'''
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
import numpy as np
import spotpy
import os
import multiprocessing as mp
from distutils.dir_util import copy_tree, remove_tree
#from shutil import rmtree
import sys

class spot_setup(object):
    def __init__(self,parallel='seq'):

        self.params = [spotpy.parameter.Uniform('cmax',low=1.0 , high=500,  optguess=412.33),
                       spotpy.parameter.Uniform('bexp',low=0.1 , high=2.0,  optguess=0.1725),
                       spotpy.parameter.Uniform('alpha',low=0.1 , high=0.99, optguess=0.8127),
                       spotpy.parameter.Uniform('Ks',low=0.0 , high=0.10, optguess=0.0404),
                       spotpy.parameter.Uniform('Kq',low=0.1 , high=0.99, optguess=0.5592)]

        self.curdir = os.getcwd()
        self.owd = os.path.realpath(__file__)+os.sep+'..'
        self.hymod_path = self.owd+os.sep+'hymod_exe'
        self.evals = list(np.genfromtxt(self.hymod_path+os.sep+'bound.txt',skip_header=65)[:,3])[:730]
        self.Factor = 1944 * (1000 * 1000 ) / (1000 * 60 * 60 * 24)
        self.parallel = parallel

    def parameters(self):
        return spotpy.parameter.generate(self.params)

    def simulation(self,x):

        if self.parallel == 'seq':
            call = ''
        elif self.parallel == 'mpi':
            call = str(int(os.environ['OMPI_COMM_WORLD_RANK'])+2)
            copy_tree(self.hymod_path, self.hymod_path+call)

        elif self.parallel == 'mpc':
            call =str(os.getpid())
            copy_tree(self.hymod_path, self.hymod_path+call)
        else:
            raise 'No call variable was assigned'

        os.chdir(self.hymod_path+call)

        try:
            if sys.version_info.major == 2:
                params = file('Param.in', 'w')
            elif sys.version_info.major == 3:
                params = open('Param.in','w')

            for i in range(len(x)):
                if i == len(x):
                    params.write(str(round(x[i],5)))
                else:
                    params.write(str(round(x[i],5))+' ')
            params.close()
            os.system('HYMODsilent.exe')

            #try:
            if sys.version_info.major == 2:
                SimRR = file('Q.out', 'r')
            elif sys.version_info.major == 3:
                SimRR = open('Q.out', 'r')
            else:
                raise Exception("Your python is too old for this example")
            simulations=[]
            for i in range(64):
                SimRR.readline()
            for i in range(730):
                val= SimRR.readline()
                #print(i,val)
                simulations.append(float(val)*self.Factor)
            SimRR.close()
            #except:
            #    SimRR = 795 * [0]
        except:
            'Model has failed'
            simulations=[np.nan]*795 #Assign bad values - model might have crashed
        os.chdir(self.curdir)
        if self.parallel == 'mpi' or self.parallel == 'mpc':
            remove_tree(self.hymod_path+call)
        return simulations

    def evaluation(self):
        return self.evals

    def objectivefunction(self,simulation,evaluation, params=None):
        #like = spotpy.likelihoods.gaussianLikelihoodMeasErrorOut(evaluation,simulation)     # Works good
        like = spotpy.objectivefunctions.nashsutcliffe(evaluation,simulation)     # Works good
        return like<commit_msg>Adds short parallelization explanation<commit_after>'''
Copyright 2015 by Tobias Houska
This file is part of Statistical Parameter Estimation Tool (SPOTPY).

:author: Tobias Houska

This example implements the external hydrological model HYMOD into SPOTPY.
'''
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
import numpy as np
import spotpy
import os
import multiprocessing as mp
from distutils.dir_util import copy_tree, remove_tree
#from shutil import rmtree
import sys

class spot_setup(object):
    def __init__(self,parallel='seq'):

        self.params = [spotpy.parameter.Uniform('cmax',low=1.0 , high=500,  optguess=412.33),
                       spotpy.parameter.Uniform('bexp',low=0.1 , high=2.0,  optguess=0.1725),
                       spotpy.parameter.Uniform('alpha',low=0.1 , high=0.99, optguess=0.8127),
                       spotpy.parameter.Uniform('Ks',low=0.0 , high=0.10, optguess=0.0404),
                       spotpy.parameter.Uniform('Kq',low=0.1 , high=0.99, optguess=0.5592)]

        self.curdir = os.getcwd()
        self.owd = os.path.realpath(__file__)+os.sep+'..'
        self.hymod_path = self.owd+os.sep+'hymod_exe'
        self.evals = list(np.genfromtxt(self.hymod_path+os.sep+'bound.txt',skip_header=65)[:,3])[:730]
        self.Factor = 1944 * (1000 * 1000 ) / (1000 * 60 * 60 * 24)
        self.parallel = parallel

    def parameters(self):
        return spotpy.parameter.generate(self.params)

    def simulation(self,x):

        if self.parallel == 'seq':
            call = ''
        elif self.parallel == 'mpi':
            #Running n parallel, care has to be taken when files are read or written
            #Therefor we check the ID of the current computer core
            call = str(int(os.environ['OMPI_COMM_WORLD_RANK'])+2)
            #And generate a new folder with all underlying files
            copy_tree(self.hymod_path, self.hymod_path+call)

        elif self.parallel == 'mpc':
            #Running n parallel, care has to be taken when files are read or written
            #Therefor we check the ID of the current computer core
            call =str(os.getpid())
            #And generate a new folder with all underlying files
            copy_tree(self.hymod_path, self.hymod_path+call)
        else:
            raise 'No call variable was assigned'

        os.chdir(self.hymod_path+call)
        try:
            if sys.version_info.major == 2:
                params = file('Param.in', 'w')
            elif sys.version_info.major == 3:
                params = open('Param.in','w')

            for i in range(len(x)):
                if i == len(x):
                    params.write(str(round(x[i],5)))
                else:
                    params.write(str(round(x[i],5))+' ')
            params.close()
            os.system('HYMODsilent.exe')

            #try:
            if sys.version_info.major == 2:
                SimRR = file('Q.out', 'r')
            elif sys.version_info.major == 3:
                SimRR = open('Q.out', 'r')
            else:
                raise Exception("Your python is too old for this example")
            simulations=[]
            for i in range(64):
                SimRR.readline()
            for i in range(730):
                val= SimRR.readline()
                simulations.append(float(val)*self.Factor)
            SimRR.close()
        except:
            'Model has failed'
            simulations=[np.nan]*795 #Assign bad values - model might have crashed
        os.chdir(self.curdir)
        if self.parallel == 'mpi' or self.parallel == 'mpc':
            remove_tree(self.hymod_path+call)
        return simulations

    def evaluation(self):
        return self.evals

    def objectivefunction(self,simulation,evaluation, params=None):
        like = spotpy.objectivefunctions.nashsutcliffe(evaluation,simulation)     # Works good
        return like<|endoftext|>


<commit_before># coding: utf-8

from __future__ import absolute_import
import ui
import console
import dialogs
from . import ElementParameterDictionaryInputView

class ElementRuntimeView (object):
	def __init__(self, thememanager):
		self.element = None
		self.params = []
		self.dictionaryParam = None
		self.tv = None
		self.dictView = None
		self.thememanager = thememanager

	@ui.in_background
	def tableview_did_select(self, tableview, section, row):
		param = self.params[row]
		name = param.displayName
		value = param.value
		noVariable = True
		if param.isVariableAllowed:
			choice = console.alert(title='Variable Available', message = 'Would you like to choose a variable?', button1='Choose Variable', button2 = 'Ask when run', button3='Don\'t use a variable')
			if choice == 1:
				noVariable = False
				param.useVariable = True
				param.variableName = console.input_alert(title='Option',message='Please enter the variable name')
				param.value = None
			elif choice == 2:
				noVariable = False
				param.useVariable = True
				param.value = None
		if name == None or name == '':
			name = param.name
		if value == None:
			value = ''
		if noVariable and param.type == 'string':
			param.value = console.input_alert(name, '', value)
		elif noVariable and param.type == 'int':
			param.value = int(console.input_alert(name,'',str(value)))
		elif noVariable and param.type == 'variable':
			pass
		elif noVariable and param.type == 'list':
			ret = dialogs.list_dialog(title=name,items=param.allowedValues, multiple=param.multipleAllowed)
			yo = ''
			if not ret == None:
				if isinstance(ret,list):
					for item in ret:
						yo += item+','
				else:
					yo = ret
				yo = yo.rstrip(',')
			param.value = yo
		elif noVariable and param.type == 'dictionary':
			self.dictionaryParam = param
			self.dictView = ElementParameterDictionaryInputView.get_view(dictionary=param.value, title=name, cb=self.dictionaryReturn, thememanager = self.thememanager)
			self.tv = tableview
			self.dictView.title_bar_color = self.thememanager.main_bar_colour
			self.dictView.tint_color = self.thememanager.main_tint_colour
			self.dictView.present(orientations=['portrait'])
		elif noVariable and param.type == 'Boolean':
			pass
		tableview.reload()

	def dictionaryReturn(self, sender):
		self.dictionaryParam.value = self.dictView.data_source.dictionary
		self.dictView.close()
		self.tv.reload()

	def tableview_title_for_header(self, tableview, section):
		return 'Parameters'

	def tableview_number_of_sections(self, tableview):
		return 1

	def tableview_number_of_rows(self, tableview, section):
		return len(self.params)

	def tableview_cell_for_row(self, tableview, section, row):
		param = self.params[row]
		name = param.displayName
		cell = None
		if name == None or name == '':
			name = param.name
		if param.type == 'bool':
			cell = ui.TableViewCell()
			cell.selectable = False
			switch = ui.Switch()
			switch.name = param.name
			switch.value = param.value
			switch.y = cell.center.y - switch.height/2
			switch.x = cell.width + switch.width/2
			switch.action = self.switch_change
			cell.add_subview(switch)
		elif param.type == 'slider':
			cell = ui.TableViewCell()
			cell.selectable = False
			slider = ui.Slider()
			slider.name = param.name
			slider.value = param.value
			slider.y = cell.center.y - slider.height/2
			slider.x = cell.width-15
			slider.action = self.switch_change
			cell.add_subview(slider)
		else:
			cell = ui.TableViewCell('value1')
			if not param.value == None:
				cell.detail_text_label.text = str(param.value)
			cell.detail_text_label.text_color = self.thememanager.main_text_colour

		cell.text_label.text = name
		cell.background_color = self.thememanager.main_background_colour
		cell.text_label.text_color = self.thememanager.main_text_colour

		return cell

	def tableview_can_delete(self, tableview, section, row):
		return False

	def tableview_can_move(self, tableview, section, row):
		return False

	def tableview_delete(self, tableview, section, row):
		pass

	def tableview_move_row(self, tableview, from_section, from_row, to_section, to_row):
		pass

	def load_element(self, element):
		self.element = element
		self.params = []
		for param in element.get_params():
			if param.display:
				self.params.append(param)

	def switch_change(self, sender):
		for p in self.params:
			if p.name == sender.name:
				p.value = sender.value

table_view = ui.TableView()
def get_view(thememanager):
	dbo = ElementRuntimeView(thememanager=thememanager)
	table_view.name = 'Element'
	table_view.data_source = dbo
	table_view.delegate = dbo
	table_view.background_color = thememanager.main_background_colour
	return table_view

<commit_msg>showed when parameters were using variables<commit_after># coding: utf-8

from __future__ import absolute_import
import ui
import console
import dialogs
from . import ElementParameterDictionaryInputView

class ElementRuntimeView (object):
	def __init__(self, thememanager):
		self.element = None
		self.params = []
		self.dictionaryParam = None
		self.tv = None
		self.dictView = None
		self.thememanager = thememanager

	@ui.in_background
	def tableview_did_select(self, tableview, section, row):
		param = self.params[row]
		name = param.displayName
		value = param.value
		noVariable = True
		if param.isVariableAllowed:
			choice = console.alert(title='Variable Available', message = 'Would you like to choose a variable?', button1='Choose Variable', button2 = 'Ask when run', button3='Don\'t use a variable')
			if choice == 1:
				noVariable = False
				param.useVariable = True
				param.variableName = console.input_alert(title='Option',message='Please enter the variable name')
				param.value = None
			elif choice == 2:
				noVariable = False
				param.useVariable = True
				param.value = None
		if name == None or name == '':
			name = param.name
		if value == None:
			value = ''
		if noVariable and param.type == 'string':
			param.value = console.input_alert(name, '', value)
		elif noVariable and param.type == 'int':
			param.value = int(console.input_alert(name,'',str(value)))
		elif noVariable and param.type == 'variable':
			pass
		elif noVariable and param.type == 'list':
			ret = dialogs.list_dialog(title=name,items=param.allowedValues, multiple=param.multipleAllowed)
			yo = ''
			if not ret == None:
				if isinstance(ret,list):
					for item in ret:
						yo += item+','
				else:
					yo = ret
				yo = yo.rstrip(',')
			param.value = yo
		elif noVariable and param.type == 'dictionary':
			self.dictionaryParam = param
			self.dictView = ElementParameterDictionaryInputView.get_view(dictionary=param.value, title=name, cb=self.dictionaryReturn, thememanager = self.thememanager)
			self.tv = tableview
			self.dictView.title_bar_color = self.thememanager.main_bar_colour
			self.dictView.tint_color = self.thememanager.main_tint_colour
			self.dictView.present(orientations=['portrait'])
		elif noVariable and param.type == 'Boolean':
			pass
		tableview.reload()

	def dictionaryReturn(self, sender):
		self.dictionaryParam.value = self.dictView.data_source.dictionary
		self.dictView.close()
		self.tv.reload()

	def tableview_title_for_header(self, tableview, section):
		return 'Parameters'

	def tableview_number_of_sections(self, tableview):
		return 1

	def tableview_number_of_rows(self, tableview, section):
		return len(self.params)

	def tableview_cell_for_row(self, tableview, section, row):
		param = self.params[row]
		name = param.displayName
		cell = None
		if name == None or name == '':
			name = param.name
		if param.type == 'bool':
			cell = ui.TableViewCell()
			cell.selectable = False
			switch = ui.Switch()
			switch.name = param.name
			switch.value = param.value
			switch.y = cell.center.y - switch.height/2
			switch.x = cell.width + switch.width/2
			switch.action = self.switch_change
			cell.add_subview(switch)
		elif param.type == 'slider':
			cell = ui.TableViewCell()
			cell.selectable = False
			slider = ui.Slider()
			slider.name = param.name
			slider.value = param.value
			slider.y = cell.center.y - slider.height/2
			slider.x = cell.width-15
			slider.action = self.switch_change
			cell.add_subview(slider)
		else:
			cell = ui.TableViewCell('value1')
			if param.useVariable:
				if not param.variableName == None and not param.variableName == '':
					cell.detail_text_label.text = 'Variable: ' + str(param.variableName)
				else:
					cell.detail_text_label.text = 'Variable: Choose at runtime'
			elif not param.value == None:
				cell.detail_text_label.text = str(param.value)
			cell.detail_text_label.text_color = self.thememanager.main_text_colour

		cell.text_label.text = name
		cell.background_color = self.thememanager.main_background_colour
		cell.text_label.text_color = self.thememanager.main_text_colour

		return cell

	def tableview_can_delete(self, tableview, section, row):
		return False

	def tableview_can_move(self, tableview, section, row):
		return False

	def tableview_delete(self, tableview, section, row):
		pass

	def tableview_move_row(self, tableview, from_section, from_row, to_section, to_row):
		pass

	def load_element(self, element):
		self.element = element
		self.params = []
		for param in element.get_params():
			if param.display:
				self.params.append(param)

	def switch_change(self, sender):
		for p in self.params:
			if p.name == sender.name:
				p.value = sender.value

table_view = ui.TableView()
def get_view(thememanager):
	dbo = ElementRuntimeView(thememanager=thememanager)
	table_view.name = 'Element'
	table_view.data_source = dbo
	table_view.delegate = dbo
	table_view.background_color = thememanager.main_background_colour
	return table_view

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"Main module defining filesystem and file classes"
from __future__ import absolute_import

import os
import ctypes
import sys
import subprocess
import warnings
import fnmatch
PY3 = sys.version_info.major > 2
from hdfs3.lib import _lib

from .compatibility import FileNotFoundError, PermissionError

def get_default_host():
    """ Guess namenode by looking in this machine's hadoop conf. """
    confd = os.environ.get('HADOOP_CONF_DIR', os.environ.get('HADOOP_INSTALL',
                           '') + '/hadoop/conf')
    try:
        host = open(os.sep.join([confd, 'masters'])).readlines()[1][:-1]
    except IOError:
        host = 'default'
    return host

def ensure_byte(s):
    """ Give strings that ctypes is guaranteed to handle """
    if PY3:
        if isinstance(s, str):
            return s.encode('ascii')
        elif isinstance(s, bytes):
            return s
        else:
            raise ValueError('Could not convert %s to bytes' % s)
    else:  # in PY2, strings are fine for ctypes
        return s

def ensure_string(s):
    """ Ensure that the result is a string

    >>> ensure_string(b'123')
    u'123'
    """
    if hasattr(s, 'decode'):
        return s.decode()
    return s

def ensure_trailing_slash(s):
    """ Ensure that string ends with a slash

    >>> ensure_trailing_slash('/user/directory')
    '/user/directory/'
    >>> ensure_trailing_slash('/user/directory/')
    '/user/directory/'
    """
    if not s.endswith('/'):
        s += '/'
    return s

def init_kerb():
    """ Find Kerberos credentials.

    Use system kinit to find credentials.
    Set up by editing krb5.conf
    """
    raise NotImplementedError("Please set your credentials manually")
    out1 = subprocess.check_call(['kinit'])
    out2 = subprocess.check_call(['klist'])
    HDFileSystem.ticket_cache = None
    HDFileSystem.token = None

class HDFileSystem():
    """ Connection to an HDFS namenode

    >>> hdfs = HDFileSystem(localhost='127.0.0.1', port=50070)  # doctest: +SKIP
    """
    def __init__(self, host=None, port=50070, user=None, ticket_cache=None,
            token=None, pars=None, connect=True):
        """
        Parameters
        ----------

        host : str (default from config files)
            namenode (name or IP)

        port : int (9000)
            connection port

        user, ticket_cache, token : str
            kerberos things

        pars : {str: str}
            other parameters for hadoop
        """
        self.host = host or get_default_host()
        self.port = port
        self.user = user
        self.ticket_cache = ticket_cache
        self.pars = pars
        self.token = None  # Delegation token (generated)
        self._handle = None
        if connect:
            self.connect()

    def __getstate__(self):
        d = self.__dict__.copy()
        del d['_handle']
        return d

    def __setstate__(self, state):
        self.__dict__.update(state)
        self._handle = None
        self.connect()

    def connect(self):
        """ Connect to the name node

        This happens automatically at startup
        """
        if self._handle:
            raise ValueError("Already connected")

        o = _lib.hdfsNewBuilder()
        _lib.hdfsBuilderSetNameNodePort(o, self.port)
        _lib.hdfsBuilderSetNameNode(o, ensure_byte(self.host))
        if self.user:
            _lib.hdfsBuilderSetUserName(o, ensure_byte(self.user))
        if self.ticket_cache:
            _lib.hdfsBuilderSetKerbTicketCachePath(o, ensure_byte(self.ticket_cache))
        if self.token:
            _lib.hdfsBuilderSetToken(o, ensure_byte(self.token))
        if self.pars:
            for par in self.pars:
                if not  _lib.hdfsBuilderConfSetStr(o, ensure_byte(par),
                                          ensure_byte(self.pars(par))) == 0:
                    warnings.warn('Setting conf parameter %s failed' % par)
        fs = _lib.hdfsBuilderConnect(o)
        if fs:
            self._handle = fs
            #if self.token:   # TODO: find out what a delegation token is
            #    self._token = _lib.hdfsGetDelegationToken(self._handle,
            #                                             ensure_byte(self.user))
        else:
            raise RuntimeError('Connection Failed')

    def disconnect(self):
        """ Disconnect from name node """
        if self._handle:
            _lib.hdfsDisconnect(self._handle)
        self._handle = None

    def open(self, path, mode='r', repl=1, buff=0, block_size=0):
        """ Open a file for reading or writing

        Parameters
        ----------
        path: string
            Path of file on HDFS
        mode: string
            One of 'r', 'w', or 'a'
        repl: int
            Replication factor
        block_size: int
            Size of data-node blocks if writing
        """
        if not self._handle:
            raise IOError("Filesystem not connected")
        if block_size and mode != 'w':
            raise ValueError('Block size only valid when writing new file')
        assert self._handle, "Filesystem not connected"
        return HDFile(self, path, mode, repl=repl, buff=buff,
                block_size=block_size)

    def du(self, path, total=False, deep=False):
        if isinstance(total, str):
            total = total=='True'
        if isinstance(deep, str):
            deep = deep=='True'
        fi = self.ls(path)
        if len(fi) == 0:
            raise IOError('Not Found')
        if deep:
            for apath in fi:
                if apath['kind'] == 68:  # directory
                    fi.extend(self.ls(apath['name']))
        if total:
            return {path: sum(f['size'] for f in fi)}
        return {p['name']: p['size'] for p in fi}

    def df(self):
        cap = _lib.hdfsGetCapacity(self._handle)
        used = _lib.hdfsGetUsed(self._handle)
        return {'capacity': cap, 'used': used, 'free': 100*(cap-used)/cap}

    def get_block_locations(self, path, start=0, length=0):
        """ Fetch physical locations of blocks """
        if not self._handle:
            raise IOError("Filesystem not connected")
        start = int(start) or 0
        length = int(length) or self.info(path)['size']
        nblocks = ctypes.c_int(0)
        out = _lib.hdfsGetFileBlockLocations(self._handle, ensure_byte(path),
                                ctypes.c_int64(start), ctypes.c_int64(length),
                                ctypes.byref(nblocks))
        locs = []
        for i in range(nblocks.value):
            block = out[i]
            hosts = [block.hosts[i] for i in
                     range(block.numOfNodes)]
            locs.append({'hosts': hosts, 'length': block.length,
                         'offset': block.offset})
        _lib.hdfsFreeFileBlockLocations(out, nblocks)
        return locs

    def info(self, path):
        """ File information """
        if not self.exists(path):
            raise FileNotFoundError(path)
        fi = _lib.hdfsGetPathInfo(self._handle, ensure_byte(path)).contents
        out = struct_to_dict(fi)
        _lib.hdfsFreeFileInfo(ctypes.byref(fi), 1)
        return out

    def glob(self, path):
        if "*" not in path:
            path = path + "*"
        if '/' in path[:path.index('*')]:
            ind = path[:path.index('*')].rindex('/')
            root = path[:ind+1]
        else:
            root = '/'
        allfiles = self.du(root, False, True).keys()
        out = [f for f in allfiles if fnmatch.fnmatch(ensure_string(f), path)]
        return out

    def ls(self, path):
        if not self.exists(path):
            raise FileNotFoundError(path)
        num = ctypes.c_int(0)
        fi = _lib.hdfsListDirectory(self._handle, ensure_byte(path), ctypes.byref(num))
        out = [struct_to_dict(fi[i]) for i in range(num.value)]
        _lib.hdfsFreeFileInfo(fi, num.value)
        return out

    def __repr__(self):
        state = ['Disconnected', 'Connected'][self._handle is not None]
        return 'hdfs://%s:%s, %s' % (self.host, self.port, state)

    def __del__(self):
        if self._handle:
            self.disconnect()

    def mkdir(self, path):
        out = _lib.hdfsCreateDirectory(self._handle, ensure_byte(path))
        return out == 0

    def set_replication(self, path, repl):
        out = _lib.hdfsSetReplication(self._handle, ensure_byte(path),
                                     ctypes.c_int16(int(repl)))
        return out == 0

    def mv(self, path1, path2):
        if not self.exists(path1):
            raise FileNotFoundError(path1)
        out = _lib.hdfsRename(self._handle, ensure_byte(path1), ensure_byte(path2))
        return out == 0

    def rm(self, path, recursive=True):
        "Use recursive for `rm -r`, i.e., delete directory and contents"
        if not self.exists(path):
            raise FileNotFoundError(path)
        out = _lib.hdfsDelete(self._handle, ensure_byte(path), bool(recursive))
        return out == 0

    def exists(self, path):
        out = _lib.hdfsExists(self._handle, ensure_byte(path) )
        return out == 0

    def truncate(self, path, pos):
        # Does not appear to ever succeed
        out = _lib.hdfsTruncate(self._handle, ensure_byte(path),
                               ctypes.c_int64(int(pos)), 0)
        return out == 0

    def chmod(self, path, mode):
        "Mode in numerical format (give as octal, if convenient)"
        if not self.exists(path):
            raise FileNotFoundError(path)
        out = _lib.hdfsChmod(self._handle, ensure_byte(path), ctypes.c_short(int(mode)))
        return out == 0

    def chown(self, path, owner, group):
        "Change owner/group"
        if not self.exists(path):
            raise FileNotFoundError(path)
        out = _lib.hdfsChown(self._handle, ensure_byte(path), ensure_byte(owner),
                            ensure_byte(group))
        return out == 0

    def cat(self, path):
        """ Return contents of file """
        if not self.exists(path):
            raise FileNotFoundError(path)
        buffers = []
        with self.open(path, 'r') as f:
            result = f.read()
        return result

    def get(self, path, filename):
        """ Copy HDFS file to local """
        #TODO: _lib.hdfsCopy() may do this more efficiently
        if not self.exists(path):
            raise FileNotFoundError(path)
        with self.open(path, 'r') as f:
            with open(filename, 'wb') as f2:
                out = 1
                while out:
                    out = f.read()
                    f2.write(out)

    def getmerge(self, path, filename):
        """ Concat all files in path (a directory) to output file """
        files = self.ls(path)
        with open(filename, 'wb') as f2:
            for apath in files:
                with self.open(apath['name'], 'r') as f:
                    out = 1
                    while out:
                        out = f.read()
                        f2.write(out)

    def put(self, filename, path, chunk=2**16):
        """ Copy local file to path in HDFS """
        #TODO: _lib.hdfsCopy() may do this more efficiently
        with self.open(path, 'w') as f:
            with open(filename, 'rb') as f2:
                while True:
                    out = f2.read(chunk)
                    if len(out) == 0:
                        break
                    f.write(out)

    def tail(self, path, size=None):
        """ Return last bytes of file """
        size = int(size) or 1024
        length = self.du(path)[ensure_trailing_slash(path)]
        if size > length:
            return self.cat(path)
        with self.open(path, 'r') as f:
            f.seek(length - size)
            return f.read(size)

    def touch(self, path):
        """ Create zero-length file """
        self.open(path, 'w').close()

def struct_to_dict(s):
    return dict((name, getattr(s, name)) for (name, p) in s._fields_)

class HDFile(object):
    """ File on HDFS """
    def __init__(self, fs, path, mode, repl=1, buff=0, block_size=0):
        """ Called by open on a HDFileSystem """
        self.fs = fs
        self.path = path
        self.repl = repl
        self._fs = fs._handle
        self.buffer = b''
        self._handle = None
        self.encoding = 'ascii'
        m = {'w': 1, 'r': 0, 'a': 1025}[mode]
        self.mode = mode
        self.block_size = block_size
        out = _lib.hdfsOpenFile(self._fs, ensure_byte(path), m, buff,
                            ctypes.c_short(repl), ctypes.c_int64(block_size))
        if not out:
            raise IOError("File open failed")
        self._handle = out
        assert self._handle > 0

    def read(self, length=None):
        """ Read bytes from open file """
        if not _lib.hdfsFileIsOpenForRead(self._handle):
            raise IOError('File not read mode')
        buffers = []

        if length is None:
            out = 1
            while out:
                out = self.read(2**16)
                buffers.append(out)
        else:
            while length:
                bufsize = min(2**16, length)
                p = ctypes.create_string_buffer(bufsize)
                ret = _lib.hdfsRead(self._fs, self._handle, p, ctypes.c_int32(bufsize))
                if ret == 0:
                    break
                if ret > 0:
                    if ret <= bufsize:
                        buffers.append(p.raw[:ret])
                    length -= ret
                else:
                    raise IOError('Read Failed:', -ret)

        return b''.join(buffers)

    def readline(self):
        """ Read a buffered line, text mode. """
        lines = getattr(self, 'lines', [])
        if len(lines) < 1:
            buff = self.read()
            if len(buff) == 0:   #EOF
                remains = self.buffer.decode(self.encoding)
                if remains:
                    self.buffer = b''
                    return remains
                raise EOFError
            buff = (self.buffer + buff).decode(self.encoding)
            self.lines = buff.split('\n')
        return self.lines.pop(0)

    def _genline(self):
        while True:
            try:
                yield self.readline()
            except EOFError:
                raise StopIteration

    def __iter__(self):
        return self._genline()

    def readlines(self):
        return list(self)

    def tell(self):
        out = _lib.hdfsTell(self._fs, self._handle)
        if out == -1:
            raise IOError('Tell Failed')
        return out

    def seek(self, loc):
        out = _lib.hdfsSeek(self._fs, self._handle, ctypes.c_int64(loc))
        if out == -1:
            raise IOError('Seek Failed')

    def info(self):
        """ Filesystem metadata about this file """
        return self.fs.info(self.path)

    def write(self, data):
        data = ensure_byte(data)
        if not _lib.hdfsFileIsOpenForWrite(self._handle):
            raise IOError('File not write mode')
        if not _lib.hdfsWrite(self._fs, self._handle, data, len(data)) == len(data):
            raise IOError('Write failed')

    def flush(self):
        _lib.hdfsFlush(self._fs, self._handle)

    def close(self):
        self.flush()
        _lib.hdfsCloseFile(self._fs, self._handle)
        self._handle = None  # _libhdfs releases memory
        self.mode = 'closed'

    def get_block_locs(self):
        """
        Get host locations and offsets of all blocks that compose this file
        """
        return self.fs.get_block_locations(self.path)

    def __del__(self):
        self.close()

    def __repr__(self):
        return 'hdfs://%s:%s%s, %s' % (self.fs.host, self.fs.port,
                                            self.path, self.mode)

    def __enter__(self):
        return self

    def __exit__(self, *args):
        self.close()
<commit_msg>Asserts should have been removed<commit_after># -*- coding: utf-8 -*-
"Main module defining filesystem and file classes"
from __future__ import absolute_import

import os
import ctypes
import sys
import subprocess
import warnings
import fnmatch
PY3 = sys.version_info.major > 2
from hdfs3.lib import _lib

from .compatibility import FileNotFoundError, PermissionError

def get_default_host():
    """ Guess namenode by looking in this machine's hadoop conf. """
    confd = os.environ.get('HADOOP_CONF_DIR', os.environ.get('HADOOP_INSTALL',
                           '') + '/hadoop/conf')
    try:
        host = open(os.sep.join([confd, 'masters'])).readlines()[1][:-1]
    except IOError:
        host = 'default'
    return host

def ensure_byte(s):
    """ Give strings that ctypes is guaranteed to handle """
    if PY3:
        if isinstance(s, str):
            return s.encode('ascii')
        elif isinstance(s, bytes):
            return s
        else:
            raise ValueError('Could not convert %s to bytes' % s)
    else:  # in PY2, strings are fine for ctypes
        return s

def ensure_string(s):
    """ Ensure that the result is a string

    >>> ensure_string(b'123')
    u'123'
    """
    if hasattr(s, 'decode'):
        return s.decode()
    return s

def ensure_trailing_slash(s):
    """ Ensure that string ends with a slash

    >>> ensure_trailing_slash('/user/directory')
    '/user/directory/'
    >>> ensure_trailing_slash('/user/directory/')
    '/user/directory/'
    """
    if not s.endswith('/'):
        s += '/'
    return s

def init_kerb():
    """ Find Kerberos credentials.

    Use system kinit to find credentials.
    Set up by editing krb5.conf
    """
    raise NotImplementedError("Please set your credentials manually")
    out1 = subprocess.check_call(['kinit'])
    out2 = subprocess.check_call(['klist'])
    HDFileSystem.ticket_cache = None
    HDFileSystem.token = None

class HDFileSystem():
    """ Connection to an HDFS namenode

    >>> hdfs = HDFileSystem(localhost='127.0.0.1', port=50070)  # doctest: +SKIP
    """
    def __init__(self, host=None, port=50070, user=None, ticket_cache=None,
            token=None, pars=None, connect=True):
        """
        Parameters
        ----------

        host : str (default from config files)
            namenode (name or IP)

        port : int (9000)
            connection port

        user, ticket_cache, token : str
            kerberos things

        pars : {str: str}
            other parameters for hadoop
        """
        self.host = host or get_default_host()
        self.port = port
        self.user = user
        self.ticket_cache = ticket_cache
        self.pars = pars
        self.token = None  # Delegation token (generated)
        self._handle = None
        if connect:
            self.connect()

    def __getstate__(self):
        d = self.__dict__.copy()
        del d['_handle']
        return d

    def __setstate__(self, state):
        self.__dict__.update(state)
        self._handle = None
        self.connect()

    def connect(self):
        """ Connect to the name node

        This happens automatically at startup
        """
        if self._handle:
            raise ValueError("Already connected")

        o = _lib.hdfsNewBuilder()
        _lib.hdfsBuilderSetNameNodePort(o, self.port)
        _lib.hdfsBuilderSetNameNode(o, ensure_byte(self.host))
        if self.user:
            _lib.hdfsBuilderSetUserName(o, ensure_byte(self.user))
        if self.ticket_cache:
            _lib.hdfsBuilderSetKerbTicketCachePath(o, ensure_byte(self.ticket_cache))
        if self.token:
            _lib.hdfsBuilderSetToken(o, ensure_byte(self.token))
        if self.pars:
            for par in self.pars:
                if not  _lib.hdfsBuilderConfSetStr(o, ensure_byte(par),
                                          ensure_byte(self.pars(par))) == 0:
                    warnings.warn('Setting conf parameter %s failed' % par)
        fs = _lib.hdfsBuilderConnect(o)
        if fs:
            self._handle = fs
            #if self.token:   # TODO: find out what a delegation token is
            #    self._token = _lib.hdfsGetDelegationToken(self._handle,
            #                                             ensure_byte(self.user))
        else:
            raise RuntimeError('Connection Failed')

    def disconnect(self):
        """ Disconnect from name node """
        if self._handle:
            _lib.hdfsDisconnect(self._handle)
        self._handle = None

    def open(self, path, mode='r', repl=1, buff=0, block_size=0):
        """ Open a file for reading or writing

        Parameters
        ----------
        path: string
            Path of file on HDFS
        mode: string
            One of 'r', 'w', or 'a'
        repl: int
            Replication factor
        block_size: int
            Size of data-node blocks if writing
        """
        if not self._handle:
            raise IOError("Filesystem not connected")
        if block_size and mode != 'w':
            raise ValueError('Block size only valid when writing new file')
        return HDFile(self, path, mode, repl=repl, buff=buff,
                block_size=block_size)

    def du(self, path, total=False, deep=False):
        if isinstance(total, str):
            total = total=='True'
        if isinstance(deep, str):
            deep = deep=='True'
        fi = self.ls(path)
        if len(fi) == 0:
            raise IOError('Not Found')
        if deep:
            for apath in fi:
                if apath['kind'] == 68:  # directory
                    fi.extend(self.ls(apath['name']))
        if total:
            return {path: sum(f['size'] for f in fi)}
        return {p['name']: p['size'] for p in fi}

    def df(self):
        cap = _lib.hdfsGetCapacity(self._handle)
        used = _lib.hdfsGetUsed(self._handle)
        return {'capacity': cap, 'used': used, 'free': 100*(cap-used)/cap}

    def get_block_locations(self, path, start=0, length=0):
        """ Fetch physical locations of blocks """
        if not self._handle:
            raise IOError("Filesystem not connected")
        start = int(start) or 0
        length = int(length) or self.info(path)['size']
        nblocks = ctypes.c_int(0)
        out = _lib.hdfsGetFileBlockLocations(self._handle, ensure_byte(path),
                                ctypes.c_int64(start), ctypes.c_int64(length),
                                ctypes.byref(nblocks))
        locs = []
        for i in range(nblocks.value):
            block = out[i]
            hosts = [block.hosts[i] for i in
                     range(block.numOfNodes)]
            locs.append({'hosts': hosts, 'length': block.length,
                         'offset': block.offset})
        _lib.hdfsFreeFileBlockLocations(out, nblocks)
        return locs

    def info(self, path):
        """ File information """
        if not self.exists(path):
            raise FileNotFoundError(path)
        fi = _lib.hdfsGetPathInfo(self._handle, ensure_byte(path)).contents
        out = struct_to_dict(fi)
        _lib.hdfsFreeFileInfo(ctypes.byref(fi), 1)
        return out

    def glob(self, path):
        if "*" not in path:
            path = path + "*"
        if '/' in path[:path.index('*')]:
            ind = path[:path.index('*')].rindex('/')
            root = path[:ind+1]
        else:
            root = '/'
        allfiles = self.du(root, False, True).keys()
        out = [f for f in allfiles if fnmatch.fnmatch(ensure_string(f), path)]
        return out

    def ls(self, path):
        if not self.exists(path):
            raise FileNotFoundError(path)
        num = ctypes.c_int(0)
        fi = _lib.hdfsListDirectory(self._handle, ensure_byte(path), ctypes.byref(num))
        out = [struct_to_dict(fi[i]) for i in range(num.value)]
        _lib.hdfsFreeFileInfo(fi, num.value)
        return out

    def __repr__(self):
        state = ['Disconnected', 'Connected'][self._handle is not None]
        return 'hdfs://%s:%s, %s' % (self.host, self.port, state)

    def __del__(self):
        if self._handle:
            self.disconnect()

    def mkdir(self, path):
        out = _lib.hdfsCreateDirectory(self._handle, ensure_byte(path))
        return out == 0

    def set_replication(self, path, repl):
        out = _lib.hdfsSetReplication(self._handle, ensure_byte(path),
                                     ctypes.c_int16(int(repl)))
        return out == 0

    def mv(self, path1, path2):
        if not self.exists(path1):
            raise FileNotFoundError(path1)
        out = _lib.hdfsRename(self._handle, ensure_byte(path1), ensure_byte(path2))
        return out == 0

    def rm(self, path, recursive=True):
        "Use recursive for `rm -r`, i.e., delete directory and contents"
        if not self.exists(path):
            raise FileNotFoundError(path)
        out = _lib.hdfsDelete(self._handle, ensure_byte(path), bool(recursive))
        return out == 0

    def exists(self, path):
        out = _lib.hdfsExists(self._handle, ensure_byte(path) )
        return out == 0

    def truncate(self, path, pos):
        # Does not appear to ever succeed
        out = _lib.hdfsTruncate(self._handle, ensure_byte(path),
                               ctypes.c_int64(int(pos)), 0)
        return out == 0

    def chmod(self, path, mode):
        "Mode in numerical format (give as octal, if convenient)"
        if not self.exists(path):
            raise FileNotFoundError(path)
        out = _lib.hdfsChmod(self._handle, ensure_byte(path), ctypes.c_short(int(mode)))
        return out == 0

    def chown(self, path, owner, group):
        "Change owner/group"
        if not self.exists(path):
            raise FileNotFoundError(path)
        out = _lib.hdfsChown(self._handle, ensure_byte(path), ensure_byte(owner),
                            ensure_byte(group))
        return out == 0

    def cat(self, path):
        """ Return contents of file """
        if not self.exists(path):
            raise FileNotFoundError(path)
        buffers = []
        with self.open(path, 'r') as f:
            result = f.read()
        return result

    def get(self, path, filename):
        """ Copy HDFS file to local """
        #TODO: _lib.hdfsCopy() may do this more efficiently
        if not self.exists(path):
            raise FileNotFoundError(path)
        with self.open(path, 'r') as f:
            with open(filename, 'wb') as f2:
                out = 1
                while out:
                    out = f.read()
                    f2.write(out)

    def getmerge(self, path, filename):
        """ Concat all files in path (a directory) to output file """
        files = self.ls(path)
        with open(filename, 'wb') as f2:
            for apath in files:
                with self.open(apath['name'], 'r') as f:
                    out = 1
                    while out:
                        out = f.read()
                        f2.write(out)

    def put(self, filename, path, chunk=2**16):
        """ Copy local file to path in HDFS """
        #TODO: _lib.hdfsCopy() may do this more efficiently
        with self.open(path, 'w') as f:
            with open(filename, 'rb') as f2:
                while True:
                    out = f2.read(chunk)
                    if len(out) == 0:
                        break
                    f.write(out)

    def tail(self, path, size=None):
        """ Return last bytes of file """
        size = int(size) or 1024
        length = self.du(path)[ensure_trailing_slash(path)]
        if size > length:
            return self.cat(path)
        with self.open(path, 'r') as f:
            f.seek(length - size)
            return f.read(size)

    def touch(self, path):
        """ Create zero-length file """
        self.open(path, 'w').close()

def struct_to_dict(s):
    return dict((name, getattr(s, name)) for (name, p) in s._fields_)

class HDFile(object):
    """ File on HDFS """
    def __init__(self, fs, path, mode, repl=1, buff=0, block_size=0):
        """ Called by open on a HDFileSystem """
        self.fs = fs
        self.path = path
        self.repl = repl
        self._fs = fs._handle
        self.buffer = b''
        self._handle = None
        self.encoding = 'ascii'
        m = {'w': 1, 'r': 0, 'a': 1025}[mode]
        self.mode = mode
        self.block_size = block_size
        out = _lib.hdfsOpenFile(self._fs, ensure_byte(path), m, buff,
                            ctypes.c_short(repl), ctypes.c_int64(block_size))
        if not out:
            raise IOError("File open failed")
        self._handle = out

    def read(self, length=None):
        """ Read bytes from open file """
        if not _lib.hdfsFileIsOpenForRead(self._handle):
            raise IOError('File not read mode')
        buffers = []

        if length is None:
            out = 1
            while out:
                out = self.read(2**16)
                buffers.append(out)
        else:
            while length:
                bufsize = min(2**16, length)
                p = ctypes.create_string_buffer(bufsize)
                ret = _lib.hdfsRead(self._fs, self._handle, p, ctypes.c_int32(bufsize))
                if ret == 0:
                    break
                if ret > 0:
                    if ret <= bufsize:
                        buffers.append(p.raw[:ret])
                    length -= ret
                else:
                    raise IOError('Read Failed:', -ret)

        return b''.join(buffers)

    def readline(self):
        """ Read a buffered line, text mode. """
        lines = getattr(self, 'lines', [])
        if len(lines) < 1:
            buff = self.read()
            if len(buff) == 0:   #EOF
                remains = self.buffer.decode(self.encoding)
                if remains:
                    self.buffer = b''
                    return remains
                raise EOFError
            buff = (self.buffer + buff).decode(self.encoding)
            self.lines = buff.split('\n')
        return self.lines.pop(0)

    def _genline(self):
        while True:
            try:
                yield self.readline()
            except EOFError:
                raise StopIteration

    def __iter__(self):
        return self._genline()

    def readlines(self):
        return list(self)

    def tell(self):
        out = _lib.hdfsTell(self._fs, self._handle)
        if out == -1:
            raise IOError('Tell Failed')
        return out

    def seek(self, loc):
        out = _lib.hdfsSeek(self._fs, self._handle, ctypes.c_int64(loc))
        if out == -1:
            raise IOError('Seek Failed')

    def info(self):
        """ Filesystem metadata about this file """
        return self.fs.info(self.path)

    def write(self, data):
        data = ensure_byte(data)
        if not _lib.hdfsFileIsOpenForWrite(self._handle):
            raise IOError('File not write mode')
        if not _lib.hdfsWrite(self._fs, self._handle, data, len(data)) == len(data):
            raise IOError('Write failed')

    def flush(self):
        _lib.hdfsFlush(self._fs, self._handle)

    def close(self):
        self.flush()
        _lib.hdfsCloseFile(self._fs, self._handle)
        self._handle = None  # _libhdfs releases memory
        self.mode = 'closed'

    def get_block_locs(self):
        """
        Get host locations and offsets of all blocks that compose this file
        """
        return self.fs.get_block_locations(self.path)

    def __del__(self):
        self.close()

    def __repr__(self):
        return 'hdfs://%s:%s%s, %s' % (self.fs.host, self.fs.port,
                                            self.path, self.mode)

    def __enter__(self):
        return self

    def __exit__(self, *args):
        self.close()
<|endoftext|>