<commit_before>import json

from django.conf import settings
from django.http import HttpResponseBadRequest, QueryDict

class RequestDecoder:

    content_type = 'application/json'

    allowed_methods = {
        'POST', 'PUT', 'PATCH',
    }

    def process_request(self, request):
        if request.method not in self.allowed_methods:
            return
        content_type = request.META.get('CONTENT_TYPE')
        if not content_type.startswith(self.content_type):
            return
        encoding = request.encoding or settings.DEFAULT_CHARSET
        try:
            body = request.body.decode(encoding=encoding)
        except UnicodeDecodeError:
            return HttpResponseBadRequest('bad unicode')
        try:
            request.POST = self.json_decode(body, encoding=encoding)
        except ValueError:
            return HttpResponseBadRequest('malformed data')

    @staticmethod
    def json_decode(body, encoding=None):
        data = json.loads(body, encoding=encoding)
        query_dict = QueryDict(mutable=True)
        try:
            query_dict.update(data)
        except AttributeError:
            # data is not a mapping (hasn't items() method)
            raise ValueError
        query_dict._mutable = False
        return query_dict
<commit_msg>use '' as default content type<commit_after>import json

from django.conf import settings
from django.http import HttpResponseBadRequest, QueryDict

class RequestDecoder:

    content_type = 'application/json'

    allowed_methods = {
        'POST', 'PUT', 'PATCH',
    }

    def process_request(self, request):
        if request.method not in self.allowed_methods:
            return
        content_type = request.META.get('CONTENT_TYPE', '')
        if not content_type.startswith(self.content_type):
            return
        encoding = request.encoding or settings.DEFAULT_CHARSET
        try:
            body = request.body.decode(encoding=encoding)
        except UnicodeDecodeError:
            return HttpResponseBadRequest('bad unicode')
        try:
            request.POST = self.json_decode(body, encoding=encoding)
        except ValueError:
            return HttpResponseBadRequest('malformed data')

    @staticmethod
    def json_decode(body, encoding=None):
        data = json.loads(body, encoding=encoding)
        query_dict = QueryDict(mutable=True)
        try:
            query_dict.update(data)
        except AttributeError:
            # data is not a mapping (hasn't items() method)
            raise ValueError
        query_dict._mutable = False
        return query_dict
<|endoftext|>


<commit_before>"""
Rotor class for Enigma Machine

"""

class Rotor
    'Basic Rotor class for Enigma Machine'
    rotorNum = 0

    def __init__(self, config):
        self.config = config
        self.POSITION = 0
        Rotor.rotorNum += 1

    def rotate(self):
        'Increments the current Rotor position'
        self.POSITION += 1

    def setStart(self, position):
        'Takes starting position from 0 to n on the Rotor'
        self.POSITION = position

    def displayPosition(self):
        'returns the value at current Rotor position'
        return self.config[self.POSITION]

<commit_msg>fixed syntax error<commit_after>"""
Rotor class for Enigma Machine

"""

class Rotor:
    'Basic Rotor class for Enigma Machine'
    rotorNum = 0

    def __init__(self, config):
        self.config = config
        self.POSITION = 0
        Rotor.rotorNum += 1

    def rotate(self):
        'Increments the current Rotor position'
        self.POSITION += 1

    def setStart(self, position):
        'Takes starting position from 0 to n on the Rotor'
        self.POSITION = position

    def displayPosition(self):
        'returns the value at current Rotor position'
        return self.config[self.POSITION]

<|endoftext|>


<commit_before>#! /usr/bin/env python

import pycurl					# We need to talk to curl
import pyev						# Our event model
import signal					# For sigint, etc.
import socket					# We'll create a socket for curl so we can watch it
import time						# For some sleeping
import urlparse					# To efficiently parse URLs
import logging

from cStringIO import StringIO	# To fake file descriptors into strings

logger = logging.getLogger('downpour')
logger.addHandler(logging.NullHandler())

class Request(object):
	MAX_RETRIES = 5

	def __init__(self, url):
		logger.debug('Request object for %s' % url)
		self.url     = url
		self.fetcher = None
		self.loop    = None
		self.sock    = None
		self.watcher = None
		# Backoff params
		self.retries = 0
		self.scale   = 2

	def __del__(self):
		logger.debug('Request for %s destroyed.' % self.url)
		try:
			self.watcher.stop()
			self.watcher = None
		except AttributeError:
			pass

	################
	# Override these
	################
	def onSuccess(self, c, contents):
		logger.debug('Successfully fetched %s' % self.url)

	def onError(self, c, contents):
		logger.debug('Error fetching %s' % self.url)

	################
	# The actual callbacks
	################
	def success(self, c, contents):
		self.onSuccess(c, contents)

	def error(self, c, errmsg):
		self.onError(c, errmsg)

	#################
	# Curl callbacks
	#################
	def socket(self, family, socktype, protocol):
		'''Undocumented in stupid pycurl documentation. Found in test scripts'''
		logger.debug('Watching socket for %s' % self.url)
		self.sock = socket.socket(family, socktype, protocol)
		self.watcher = pyev.Io(self.sock.fileno(), pyev.EV_READ | pyev.EV_WRITE, self.loop, self.io)
		self.watcher.start()
		return self.sock

	#################
	# libev callbacks
	#################
	def io(self, watcher, revents):
		#logger.debug('IO Event for %s' % self.url)
		#self.watcher.stop()
		self.fetcher.check()
		#self.watcher.set(self.sock, pyev.EV_READ | pyev.EV_WRITE)
		#self.watcher.start()

class Fetcher(object):
	def __init__(self, poolSize=10):
		self.multi = pycurl.CurlMulti()
		self.multi.handles = []
		self.multi.setopt(pycurl.M_SOCKETFUNCTION, self.socket)
		self.multi.setopt(pycurl.M_TIMERFUNCTION, self.rescheduleTimer)
		self.share = pycurl.CurlShare()
		self.share.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_DNS)
		self.poolSize = poolSize
		self.loop     = pyev.default_loop()
		self.queue    = []
		self.numInFlight = 0
		for i in range(self.poolSize):
			c = pycurl.Curl()
			c.timeout = None
			c.setopt(pycurl.FOLLOWLOCATION, 1)
			c.setopt(pycurl.MAXREDIRS, 5)
			c.setopt(pycurl.CONNECTTIMEOUT, 15)
			c.setopt(pycurl.TIMEOUT, 300)
			c.setopt(pycurl.NOSIGNAL, 1)
			c.setopt(pycurl.FRESH_CONNECT, 1)
			c.setopt(pycurl.FORBID_REUSE, 1)
			c.setopt(pycurl.SHARE, self.share)
			self.multi.handles.append(c)
		self.pool = self.multi.handles[:]
		SIGSTOP = (signal.SIGPIPE, signal.SIGINT, signal.SIGTERM)
		self.watchers = [pyev.Signal(sig, self.loop, self.signal) for sig in SIGSTOP]
		self.evTimer = pyev.Timer(0, 1.0, self.loop, self.timer)
		#self.watchers.append(pyev.Idle(self.loop, self.idle))

	#################
	# Inheritance Interface
	#################
	def __len__(self):
		return self.numInFlight + len(self.queue)

	def extend(self, requests):
		self.queue.extend(requests)
		self.checkQueue()

	def pop(self):
		'''Get the next request'''
		return self.queue.pop(0)

	def push(self, r):
		'''Queue a request'''
		self.queue.append(r)
		self.checkQueue()

	#################
	# libev callbacks
	#################
	def idle(self, watcher, revents):
		self.multi.perform()
		time.sleep(0.1)

	def signal(self, watcher, revents):
		self.stop()

	def timer(self, watcher, revents):
		self.evTimer.stop()
		logger.debug('Timer fired!')
		self.multi.perform()
		self.checkTimeouts()
		self.evTimer.start()

	#################
	# Service
	#################
	def start(self):
		logger.info('Starting...')
		for watcher in self.watchers:
			watcher.start()
		self.evTimer.start()
		self.checkQueue()
		self.loop.start()

	def stop(self):
		logger.info('Stopping')
		self.loop.stop(pyev.EVBREAK_ALL)
		while self.watchers:
			self.watchers.pop().stop()
		self.evTimer.stop()
		self.loop.stop()

	#################
	# Iheritable
	#################
	def onSuccess(self, c):
		pass

	def onError(self, c):
		pass

	def onDone(self, c):
		pass

	#################
	# Internal callbacks
	#################
	def socket(self, a, b, c, d):
		logger.debug('Socket action')

	def success(self, c):
		c.request.success(c, c.fp.getvalue())
		self.onSuccess(c)
		self.done(c)

	#################
	# libcurl callbacks
	#################
	def rescheduleTimer(self, timeout):
		if timeout > 0:
			self.evTimer.stop()
			self.evTimer.set(timeout / 1000.0, 5.0)
			self.evTimer.start()
		else:
			self.evTimer.reset()

	def error(self, c, errno, errmsg):
		logger.debug('%s : (%i) %s' % (c.request.url, errno, errmsg))
		if c.request.retries <= c.request.MAX_RETRIES:
			t = c.request.scale ** c.request.retries
			logger.debug('Request for %s sleeping %i' % (c.url, t))
			time.sleep(t)
			c.fp.close()
			c.fp = StringIO()
			c.setopt(pycurl.WRITEFUNCTION, c.fp.write)
			self.multi.remove_handle(c)
			self.multi.add_handle(c)
		else:
			c.request.error(c, c.errstr())
			self.onError(c)
			self.done(c)

	def done(self, c):
		logger.debug('Done with %s' % c.request.url)
		self.onDone(c)
		c.fp.close()
		c.fp = None
		c.timeout = 0
		self.pool.append(c)
		self.numInFlight -= 1
		self.multi.remove_handle(c)
		self.checkQueue()

	def check(self):
		if self.multi.perform():
			numQ, okList, errList = self.multi.info_read()
			for c in okList:
				# Handle successulf
				self.success(c)
			for c, errno, errmsg in errList:
				# Handle failed
				self.error(c, errno, errmsg)

	def checkQueue(self):
		while len(self.queue) and len(self.pool):
			logger.info('Remaining in queue: %i' % len(self.queue))
			logger.debug('Pool: %i' % len(self.pool))
			# If we have requests and handles to spare,
			req = self.pop()
			if req == None:
				break
			req.fetcher = self
			req.loop    = self.loop
			c = self.pool.pop()
			c.request = req
			c.fp = StringIO()
			c.setopt(pycurl.URL, c.request.url)
			c.setopt(pycurl.HTTPHEADER, ['Host: %s' % urlparse.urlparse(req.url).hostname])
			# When you create a socket, we must register it!
			c.setopt(pycurl.OPENSOCKETFUNCTION, c.request.socket)
			c.setopt(pycurl.WRITEFUNCTION, c.fp.write)
			self.numInFlight += 1
			self.multi.add_handle(c)
			if c.timeout == None:
				logger.debug('Kickstarting...')
				self.multi.socket_action(pycurl.SOCKET_TIMEOUT, 0)
			c.timeout = time.time() + 15.0
			self.multi.perform()

	def checkTimeouts(self):
		now = time.time()
		for c in self.multi.handles:
			if c.timeout and c.timeout < now:
				self.multi.socket_action(pycurl.SOCKET_TIMEOUT, 0)
				break

if __name__ == '__main__':
	formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
	handler   = logging.StreamHandler()
	handler.setLevel(logging.DEBUG)
	handler.setFormatter(formatter)
	logger.addHandler(handler)
	logger.setLevel(logging.DEBUG)
	f = file('urls.txt')
	urls = f.read().strip().split()
	f.close()
	f = Fetcher(20)
	f.extend(Request(url) for url in urls)
	f.start()
	<commit_msg>Trying to get the timer to fire the right number of times :-/<commit_after>#! /usr/bin/env python

import pycurl					# We need to talk to curl
import pyev						# Our event model
import signal					# For sigint, etc.
import socket					# We'll create a socket for curl so we can watch it
import time						# For some sleeping
import urlparse					# To efficiently parse URLs
import logging

from cStringIO import StringIO	# To fake file descriptors into strings

logger = logging.getLogger('downpour')
logger.addHandler(logging.NullHandler())

class Request(object):
	MAX_RETRIES = 5

	def __init__(self, url):
		logger.debug('Request object for %s' % url)
		self.url     = url
		self.fetcher = None
		self.loop    = None
		self.sock    = None
		self.watcher = None
		# Backoff params
		self.retries = 0
		self.scale   = 2

	def __del__(self):
		logger.debug('Request for %s destroyed.' % self.url)
		try:
			self.watcher.stop()
			self.watcher = None
		except AttributeError:
			pass

	################
	# Override these
	################
	def onSuccess(self, c, contents):
		logger.debug('Successfully fetched %s' % self.url)

	def onError(self, c, contents):
		logger.debug('Error fetching %s' % self.url)

	################
	# The actual callbacks
	################
	def success(self, c, contents):
		self.onSuccess(c, contents)

	def error(self, c, errmsg):
		self.onError(c, errmsg)

	#################
	# Curl callbacks
	#################
	def socket(self, family, socktype, protocol):
		'''Undocumented in stupid pycurl documentation. Found in test scripts'''
		logger.debug('Watching socket for %s' % self.url)
		self.sock = socket.socket(family, socktype, protocol)
		self.watcher = pyev.Io(self.sock.fileno(), pyev.EV_READ | pyev.EV_WRITE, self.loop, self.io)
		self.watcher.start()
		return self.sock

	#################
	# libev callbacks
	#################
	def io(self, watcher, revents):
		#logger.debug('IO Event for %s' % self.url)
		#self.watcher.stop()
		self.fetcher.check()
		#self.watcher.set(self.sock, pyev.EV_READ | pyev.EV_WRITE)
		#self.watcher.start()

class Fetcher(object):
	def __init__(self, poolSize=10):
		self.multi = pycurl.CurlMulti()
		self.multi.handles = []
		self.multi.setopt(pycurl.M_SOCKETFUNCTION, self.socket)
		self.multi.setopt(pycurl.M_TIMERFUNCTION, self.rescheduleTimer)
		self.share = pycurl.CurlShare()
		self.share.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_DNS)
		self.poolSize = poolSize
		self.loop     = pyev.default_loop()
		self.queue    = []
		self.numInFlight = 0
		for i in range(self.poolSize):
			c = pycurl.Curl()
			c.timeout = None
			c.setopt(pycurl.FOLLOWLOCATION, 1)
			c.setopt(pycurl.MAXREDIRS, 5)
			c.setopt(pycurl.CONNECTTIMEOUT, 15)
			c.setopt(pycurl.TIMEOUT, 300)
			c.setopt(pycurl.NOSIGNAL, 1)
			c.setopt(pycurl.FRESH_CONNECT, 1)
			c.setopt(pycurl.FORBID_REUSE, 1)
			c.setopt(pycurl.SHARE, self.share)
			self.multi.handles.append(c)
		self.pool = self.multi.handles[:]
		SIGSTOP = (signal.SIGPIPE, signal.SIGINT, signal.SIGTERM)
		self.watchers = [pyev.Signal(sig, self.loop, self.signal) for sig in SIGSTOP]
		self.evTimer = pyev.Timer(0, 1.0, self.loop, self.timer)
		#self.watchers.append(pyev.Idle(self.loop, self.idle))

	#################
	# Inheritance Interface
	#################
	def __len__(self):
		return self.numInFlight + len(self.queue)

	def extend(self, requests):
		self.queue.extend(requests)
		self.checkQueue()

	def pop(self):
		'''Get the next request'''
		return self.queue.pop(0)

	def push(self, r):
		'''Queue a request'''
		self.queue.append(r)
		self.checkQueue()

	#################
	# libev callbacks
	#################
	def idle(self, watcher, revents):
		self.multi.perform()
		time.sleep(0.1)

	def signal(self, watcher, revents):
		self.stop()

	def timer(self, watcher, revents):
		self.evTimer.stop()
		logger.debug('Timer fired!')
		self.multi.perform()
		self.checkTimeouts()
		self.evTimer.start()

	#################
	# Service
	#################
	def start(self):
		logger.info('Starting...')
		for watcher in self.watchers:
			watcher.start()
		self.evTimer.start()
		self.checkQueue()
		self.loop.start()

	def stop(self):
		logger.info('Stopping')
		self.loop.stop(pyev.EVBREAK_ALL)
		while self.watchers:
			self.watchers.pop().stop()
		self.evTimer.stop()
		self.loop.stop()

	#################
	# Iheritable
	#################
	def onSuccess(self, c):
		pass

	def onError(self, c):
		pass

	def onDone(self, c):
		pass

	#################
	# Internal callbacks
	#################
	def socket(self, a, b, c, d):
		logger.debug('Socket action')

	def success(self, c):
		c.request.success(c, c.fp.getvalue())
		self.onSuccess(c)
		self.done(c)

	#################
	# libcurl callbacks
	#################
	def rescheduleTimer(self, timeout):
		if timeout > 0:
			self.evTimer.stop()
			self.evTimer.set(timeout / 1000.0, 0)
			self.evTimer.start()
		else:
			self.evTimer.reset()

	def error(self, c, errno, errmsg):
		logger.debug('%s : (%i) %s' % (c.request.url, errno, errmsg))
		if c.request.retries <= c.request.MAX_RETRIES:
			t = c.request.scale ** c.request.retries
			logger.debug('Request for %s sleeping %i' % (c.url, t))
			time.sleep(t)
			c.fp.close()
			c.fp = StringIO()
			c.setopt(pycurl.WRITEFUNCTION, c.fp.write)
			self.multi.remove_handle(c)
			self.multi.add_handle(c)
		else:
			c.request.error(c, c.errstr())
			self.onError(c)
			self.done(c)

	def done(self, c):
		logger.debug('Done with %s' % c.request.url)
		self.onDone(c)
		c.fp.close()
		c.fp = None
		c.timeout = 0
		self.pool.append(c)
		self.numInFlight -= 1
		self.multi.remove_handle(c)
		self.checkQueue()

	def check(self):
		if self.multi.perform():
			numQ, okList, errList = self.multi.info_read()
			for c in okList:
				# Handle successulf
				self.success(c)
			for c, errno, errmsg in errList:
				# Handle failed
				self.error(c, errno, errmsg)
		self.evTimer.reset()

	def checkQueue(self):
		while len(self.queue) and len(self.pool):
			logger.info('Remaining in queue: %i' % len(self.queue))
			logger.debug('Pool: %i' % len(self.pool))
			# If we have requests and handles to spare,
			req = self.pop()
			if req == None:
				break
			req.fetcher = self
			req.loop    = self.loop
			c = self.pool.pop()
			c.request = req
			c.fp = StringIO()
			c.setopt(pycurl.URL, c.request.url)
			c.setopt(pycurl.HTTPHEADER, ['Host: %s' % urlparse.urlparse(req.url).hostname])
			# When you create a socket, we must register it!
			c.setopt(pycurl.OPENSOCKETFUNCTION, c.request.socket)
			c.setopt(pycurl.WRITEFUNCTION, c.fp.write)
			self.numInFlight += 1
			self.multi.add_handle(c)
			if c.timeout == None:
				logger.debug('Kickstarting...')
				self.multi.socket_action(pycurl.SOCKET_TIMEOUT, 0)
			c.timeout = time.time() + 15.0
			self.multi.perform()

	def checkTimeouts(self):
		now = time.time()
		for c in self.multi.handles:
			if c.timeout and c.timeout < now:
				logger.debug('Application timeout')
				self.multi.socket_action(pycurl.SOCKET_TIMEOUT, 0)
				break

if __name__ == '__main__':
	formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
	handler   = logging.StreamHandler()
	handler.setLevel(logging.DEBUG)
	handler.setFormatter(formatter)
	logger.addHandler(handler)
	logger.setLevel(logging.DEBUG)
	f = file('urls.txt')
	urls = f.read().strip().split()
	f.close()
	f = Fetcher(20)
	f.extend(Request(url) for url in urls)
	f.start()
	<|endoftext|>


<commit_before>"""
Useful callbacks for the Run Engine
"""
import sys
from itertools import count
from prettytable import PrettyTable
import matplotlib.pyplot as plt
from collections import OrderedDict
from .utils import doc_type
from datetime import datetime
import matplotlib.pyplot as plt
import filestore.api as fsapi
from xray_vision.backend.mpl.cross_section_2d import CrossSection
import numpy as np
import filestore

import logging
logger = logging.getLogger(__name__)

class CallbackBase(object):
    def __init__(self):
        super().__init__()

    def __call__(self, doc):
        """Inspect the document, infer its type, and dispatch it."""
        doc_name = doc_type(doc)
        doc_func = getattr(self, doc_name)
        doc_func(doc)

    def event(self, doc):
        logger.debug("CallbackBase: I'm an event with doc = {}".format(doc))

    def descriptor(self, doc):
        logger.debug("CallbackBase: I'm a descriptor with doc = {}".format(doc))

    def start(self, doc):
        logger.debug("CallbackBase: I'm a start with doc = {}".format(doc))

    def stop(self, doc):
        logger.debug("CallbackBase: I'm a stop with doc = {}".format(doc))

class LiveImage(CallbackBase):

    def __init__(self, field):
        """
        Stream 2D images in a cross-section viewer.

        Parameters
        ----------
        field : string
            name of data field in an Event

        Note
        ----
        Required a matplotlib fix not released as of this writing. The
        relevant commit is a951b7.
        """
        super().__init__()
        self.field = field
        fig = plt.figure()
        self.cs = CrossSection(fig)
        self.cs._fig.show()

    def event(self, doc):
        uid = doc['data'][self.field]
        data = fsapi.retrieve(uid)
        self.cs.update_image(data)
        self.cs._fig.canvas.draw()
        self.cs._fig.canvas.flush_events()

class CallbackCounter:
    "As simple as it sounds: count how many times a callback is called."
    # Wrap itertools.count in something we can use as a callback.
    def __init__(self):
        self.counter = count()
        self(None)  # Start counting at 1.

    def __call__(self, doc):
        self.value = next(self.counter)

def print_metadata(doc):
    "Print all fields except uid and time."
    for field, value in sorted(doc.items()):
        # uid is returned by the RunEngine, and time is self-evident
        if field not in ['time', 'uid']:
            print('{0}: {1}'.format(field, value))

def collector(field, output):
    """
    Build a function that appends data to a list.

    This is useful for testing but not advised for general use. (There is
    probably a better way to do whatever you want to do!)

    Parameters
    ----------
    field : str
        the name of a data field in an Event
    output : mutable iterable
        such as a list

    Returns
    -------
    func : function
        expects one argument, an Event dictionary
    """
    def f(event):
        output.append(event['data'][field])

    return f

class LivePlot(CallbackBase):
    def __init__(self, y, x=None, **kwargs):
        """
        Build a function that updates a plot from a stream of Events.

        Parameters
        ----------
        y : str
            the name of a data field in an Event
        x : str or None
            the name of a data field in an Event
            If None, use the Event's sequence number.

        All additional keyword arguments are passed through to ``Axes.plot``.

        Returns
        -------
        func : function
            expects one argument, an Event dictionary

        Examples
        --------
        >>> import matplotlib as pyplot
        >>> fig, ax = plt.subplots()
        >>> my_plotter = live_scalar_plotter(ax, 'det1', 'motor1')
        >>> RE(my_scan, subs={'event': my_plotter})
        """
        super().__init__()
        fig, ax = plt.subplots()
        plt.show()
        self.ax = ax
        self.ax.set_ylabel(y)
        self.ax.set_xlabel(x or 'sequence #')
        self.y = y
        self.x = x
        self.kwargs = kwargs
        self.lines = []

    def start(self, doc):
        # The doc is not used; we just use the singal that a new run began.
        self.x_data, self.y_data = [], []
        self.current_line, = self.ax.plot([], [], **self.kwargs)
        self.lines.append(self.current_line)

    def event(self, doc):
        "Update line with data from this Event."
        # Do repeated 'self' lookups once, for perf.
        x_data = self.x_data
        y_data = self.y_data
        ax = self.ax
        if self.x is not None:
            x_data.append(doc['data'][self.x])
        else:
            x_data.append(doc['seq_num'])
        y_data.append(doc['data'][self.y])
        self.current_line.set_data(x_data, y_data)
        # Rescale and redraw.
        ax.relim(visible_only=True)
        ax.autoscale_view(tight=True)
        ax.figure.canvas.draw()
        try:
            ax.figure.canvas.flush_events()
        except NotImplementedError:
            pass  # Ignore on non-interactive backends.

def format_num(x, max_len=11, pre=5, post=5):
    if (abs(x) > 10**pre or abs(x) < 10**-post) and x != 0:
        x = '%.{}e'.format(post) % x
    else:
        x = '%{}.{}f'.format(pre, post) % x

    return x

class LiveTable(CallbackBase):
    """
    Build a function that prints data from each Event as a row in a table.

    Parameters
    ----------
    fields : list, optional
        names of data fields to include in addition to 'seq_num'
    rowwise : bool
        If True, append each row to stdout. If False, reprint the full updated
        table each time. This is useful if other messsages are interspersed.
    print_header_interval : int
        The number of events to process and print their rows before printing
        the header again

    Examples
    --------
    Show a table with motor and detector readings..

    >>> RE(stepscan(motor, det), subs={'all': LiveTable(['motor', 'det'])})
    +------------+-------------------+----------------+----------------+
    |   seq_num  |             time  |         motor  |   sum(det_2d)  |
    +------------+-------------------+----------------+----------------+
    |         1  |  12:46:47.503068  |         -5.00  |        449.77  |
    |         3  |  12:46:47.682788  |         -3.00  |        460.60  |
    |         4  |  12:46:47.792307  |         -2.00  |        584.77  |
    |         5  |  12:46:47.915401  |         -1.00  |       1056.37  |
    |         7  |  12:46:48.120626  |          1.00  |       1056.50  |
    |         8  |  12:46:48.193028  |          2.00  |        583.33  |
    |         9  |  12:46:48.318454  |          3.00  |        460.99  |
    |        10  |  12:46:48.419579  |          4.00  |        451.53  |
    +------------+-------------------+----------------+----------------+

    """
    base_fields = ['seq_num', 'time']
    base_field_widths = [8, 15]
    data_field_width = 12
    max_pre_decimal = 5
    max_post_decimal = 2

    def __init__(self, fields=None, rowwise=True, print_header_interval=50,
                 max_post_decimal=2, max_pre_decimal=5, data_field_width=12):
        self.data_field_width = data_field_width
        self.max_pre_decimal = max_pre_decimal
        self.max_post_decimal = max_post_decimal
        super(LiveTable, self).__init__()
        self.rowwise = rowwise
        if fields is None:
            fields = []
        self.fields = fields
        self.field_column_names = [field for field in self.fields]
        self.num_events_since_last_header = 1
        self.print_header_interval = print_header_interval
        self._filestore_keys = set()
        # self.create_table()

    def create_table(self):
        self.table = PrettyTable(field_names=(self.base_fields +
                                              self.field_column_names))
        self.table.padding_width = 2
        self.table.align = 'r'
        # format the placeholder fields for the base fields so that the
        # heading prints at the correct width
        base_fields = [' '*width for width in self.base_field_widths]
        # format placeholder fields for the data fields so that the heading
        # prints at the correct width
        data_fields = [' '*self.data_field_width for _ in self.fields]
        self.table.add_row(base_fields + data_fields)
        if self.rowwise:
            self._print_table_header()
        sys.stdout.flush()

    def _print_table_header(self):
        print('\n'.join(str(self.table).split('\n')[:3]))

    ### RunEngine document callbacks

    def start(self, start_document):
        self.create_table()

    def descriptor(self, descriptor):
        # find all keys that are filestore keys
        for key, datakeydict in descriptor['data_keys'].items():
            data_loc = datakeydict.get('external', '')
            if data_loc == 'FILESTORE:':
                self._filestore_keys.add(key)

        # see if any are being shown in the table
        reprint_header = False
        new_names = []
        for key in self.field_column_names:
            if key in self._filestore_keys:
                reprint_header = True
                print('\n')
                print('%s is a non-scalar field. Computing the sum instead' %
                      key)
                key = 'sum(%s)' % key
                key = key[:self.data_field_width]
            new_names.append(key)
        self.field_column_names = new_names
        if reprint_header:
            print('\n\n')
            self.create_table()
            # self._print_table_header()

    def event(self, event_document):
        event_time = str(datetime.fromtimestamp(event_document['time']).time())
        row = [event_document['seq_num'], event_time]
        for field in self.fields:
            val = event_document['data'].get(field, '')
            if field in self._filestore_keys:
                val = fsapi.retrieve(val)
            if isinstance(val, np.ndarray) or isinstance(val, list):
                val = np.sum(np.asarray(val))
            try:
                val = format_num(val,
                                 max_len=self.data_field_width,
                                 pre=self.max_pre_decimal,
                                 post=self.max_post_decimal)
            except Exception:
                val = str(val)[:self.data_field_width]
            row.append(val)
        self.table.add_row(row)

        if self.rowwise:
            # Print the last row of data only.
            print(str(self.table).split('\n')[-2])  # [-1] is the bottom border
            # only print header intermittently for rowwise table printing
            if self.num_events_since_last_header >= self.print_header_interval:
                self._print_table_header()
                self.num_events_since_last_header = 0
            self.num_events_since_last_header += 1
        else:
            # print the whole table
            print(self.table)

        sys.stdout.flush()

    def stop(self, stop_document):
        """Print the last row of the table

        Parameters
        ----------
        stop_document : dict
            Not explicitly used in this function, other than to signal that
            the run has been completed
        """
        print(str(self.table).split('\n')[-1])
        # remove all data from the table
        self.table.clear_rows()
        # reset the filestore keys
        self._filestore_keys = set()
<commit_msg>MNT: minor plotting improvements<commit_after>"""
Useful callbacks for the Run Engine
"""
import sys
from itertools import count
from prettytable import PrettyTable
import matplotlib.pyplot as plt
from collections import OrderedDict
from .utils import doc_type
from datetime import datetime
import matplotlib.pyplot as plt
import filestore.api as fsapi
from xray_vision.backend.mpl.cross_section_2d import CrossSection
import numpy as np
import filestore

import logging
logger = logging.getLogger(__name__)

class CallbackBase(object):
    def __init__(self):
        super().__init__()

    def __call__(self, doc):
        """Inspect the document, infer its type, and dispatch it."""
        doc_name = doc_type(doc)
        doc_func = getattr(self, doc_name)
        doc_func(doc)

    def event(self, doc):
        logger.debug("CallbackBase: I'm an event with doc = {}".format(doc))

    def descriptor(self, doc):
        logger.debug("CallbackBase: I'm a descriptor with doc = {}".format(doc))

    def start(self, doc):
        logger.debug("CallbackBase: I'm a start with doc = {}".format(doc))

    def stop(self, doc):
        logger.debug("CallbackBase: I'm a stop with doc = {}".format(doc))

class LiveImage(CallbackBase):

    def __init__(self, field):
        """
        Stream 2D images in a cross-section viewer.

        Parameters
        ----------
        field : string
            name of data field in an Event

        Note
        ----
        Required a matplotlib fix not released as of this writing. The
        relevant commit is a951b7.
        """
        super().__init__()
        self.field = field
        fig = plt.figure()
        self.cs = CrossSection(fig)
        self.cs._fig.show()

    def event(self, doc):
        uid = doc['data'][self.field]
        data = fsapi.retrieve(uid)
        self.cs.update_image(data)
        self.cs._fig.canvas.draw()
        self.cs._fig.canvas.flush_events()

class CallbackCounter:
    "As simple as it sounds: count how many times a callback is called."
    # Wrap itertools.count in something we can use as a callback.
    def __init__(self):
        self.counter = count()
        self(None)  # Start counting at 1.

    def __call__(self, doc):
        self.value = next(self.counter)

def print_metadata(doc):
    "Print all fields except uid and time."
    for field, value in sorted(doc.items()):
        # uid is returned by the RunEngine, and time is self-evident
        if field not in ['time', 'uid']:
            print('{0}: {1}'.format(field, value))

def collector(field, output):
    """
    Build a function that appends data to a list.

    This is useful for testing but not advised for general use. (There is
    probably a better way to do whatever you want to do!)

    Parameters
    ----------
    field : str
        the name of a data field in an Event
    output : mutable iterable
        such as a list

    Returns
    -------
    func : function
        expects one argument, an Event dictionary
    """
    def f(event):
        output.append(event['data'][field])

    return f

class LivePlot(CallbackBase):
    def __init__(self, y, x=None, **kwargs):
        """
        Build a function that updates a plot from a stream of Events.

        Parameters
        ----------
        y : str
            the name of a data field in an Event
        x : str or None
            the name of a data field in an Event
            If None, use the Event's sequence number.

        All additional keyword arguments are passed through to ``Axes.plot``.

        Returns
        -------
        func : function
            expects one argument, an Event dictionary

        Examples
        --------
        >>> import matplotlib as pyplot
        >>> fig, ax = plt.subplots()
        >>> my_plotter = live_scalar_plotter(ax, 'det1', 'motor1')
        >>> RE(my_scan, subs={'event': my_plotter})
        """
        super().__init__()
        fig, ax = plt.subplots()
        plt.show()
        self.ax = ax
        self.ax.set_ylabel(y)
        self.ax.set_xlabel(x or 'sequence #')
        self.ax.margins(.1)
        self.y = y
        self.x = x
        self.kwargs = kwargs
        self.lines = []

    def start(self, doc):
        # The doc is not used; we just use the singal that a new run began.
        self.x_data, self.y_data = [], []
        self.current_line, = self.ax.plot([], [], **self.kwargs)
        self.lines.append(self.current_line)

    def event(self, doc):
        "Update line with data from this Event."
        # Do repeated 'self' lookups once, for perf.
        x_data = self.x_data
        y_data = self.y_data
        ax = self.ax
        if self.x is not None:
            x_data.append(doc['data'][self.x])
        else:
            x_data.append(doc['seq_num'])
        y_data.append(doc['data'][self.y])
        self.current_line.set_data(x_data, y_data)
        # Rescale and redraw.
        ax.relim(visible_only=True)
        ax.autoscale_view(tight=True)
        ax.figure.canvas.draw()
        try:
            ax.figure.canvas.flush_events()
        except NotImplementedError:
            pass  # Ignore on non-interactive backends.

def format_num(x, max_len=11, pre=5, post=5):
    if (abs(x) > 10**pre or abs(x) < 10**-post) and x != 0:
        x = '%.{}e'.format(post) % x
    else:
        x = '%{}.{}f'.format(pre, post) % x

    return x

class LiveTable(CallbackBase):
    """
    Build a function that prints data from each Event as a row in a table.

    Parameters
    ----------
    fields : list, optional
        names of data fields to include in addition to 'seq_num'
    rowwise : bool
        If True, append each row to stdout. If False, reprint the full updated
        table each time. This is useful if other messsages are interspersed.
    print_header_interval : int
        The number of events to process and print their rows before printing
        the header again

    Examples
    --------
    Show a table with motor and detector readings..

    >>> RE(stepscan(motor, det), subs={'all': LiveTable(['motor', 'det'])})
    +------------+-------------------+----------------+----------------+
    |   seq_num  |             time  |         motor  |   sum(det_2d)  |
    +------------+-------------------+----------------+----------------+
    |         1  |  12:46:47.503068  |         -5.00  |        449.77  |
    |         3  |  12:46:47.682788  |         -3.00  |        460.60  |
    |         4  |  12:46:47.792307  |         -2.00  |        584.77  |
    |         5  |  12:46:47.915401  |         -1.00  |       1056.37  |
    |         7  |  12:46:48.120626  |          1.00  |       1056.50  |
    |         8  |  12:46:48.193028  |          2.00  |        583.33  |
    |         9  |  12:46:48.318454  |          3.00  |        460.99  |
    |        10  |  12:46:48.419579  |          4.00  |        451.53  |
    +------------+-------------------+----------------+----------------+

    """
    base_fields = ['seq_num', 'time']
    base_field_widths = [8, 15]
    data_field_width = 12
    max_pre_decimal = 5
    max_post_decimal = 2

    def __init__(self, fields=None, rowwise=True, print_header_interval=50,
                 max_post_decimal=2, max_pre_decimal=5, data_field_width=12):
        self.data_field_width = data_field_width
        self.max_pre_decimal = max_pre_decimal
        self.max_post_decimal = max_post_decimal
        super(LiveTable, self).__init__()
        self.rowwise = rowwise
        if fields is None:
            fields = []
        self.fields = fields
        self.field_column_names = [field for field in self.fields]
        self.num_events_since_last_header = 1
        self.print_header_interval = print_header_interval
        self._filestore_keys = set()
        # self.create_table()

    def create_table(self):
        self.table = PrettyTable(field_names=(self.base_fields +
                                              self.field_column_names))
        self.table.padding_width = 2
        self.table.align = 'r'
        # format the placeholder fields for the base fields so that the
        # heading prints at the correct width
        base_fields = [' '*width for width in self.base_field_widths]
        # format placeholder fields for the data fields so that the heading
        # prints at the correct width
        data_fields = [' '*self.data_field_width for _ in self.fields]
        self.table.add_row(base_fields + data_fields)
        if self.rowwise:
            self._print_table_header()
        sys.stdout.flush()

    def _print_table_header(self):
        print('\n'.join(str(self.table).split('\n')[:3]))

    ### RunEngine document callbacks

    def start(self, start_document):
        self.create_table()

    def descriptor(self, descriptor):
        # find all keys that are filestore keys
        for key, datakeydict in descriptor['data_keys'].items():
            data_loc = datakeydict.get('external', '')
            if data_loc == 'FILESTORE:':
                self._filestore_keys.add(key)

        # see if any are being shown in the table
        reprint_header = False
        new_names = []
        for key in self.field_column_names:
            if key in self._filestore_keys:
                reprint_header = True
                print('\n')
                print('%s is a non-scalar field. Computing the sum instead' %
                      key)
                key = 'sum(%s)' % key
                key = key[:self.data_field_width]
            new_names.append(key)
        self.field_column_names = new_names
        if reprint_header:
            print('\n\n')
            self.create_table()
            # self._print_table_header()

    def event(self, event_document):
        event_time = str(datetime.fromtimestamp(event_document['time']).time())
        row = [event_document['seq_num'], event_time]
        for field in self.fields:
            val = event_document['data'].get(field, '')
            if field in self._filestore_keys:
                val = fsapi.retrieve(val)
            if isinstance(val, np.ndarray) or isinstance(val, list):
                val = np.sum(np.asarray(val))
            try:
                val = format_num(val,
                                 max_len=self.data_field_width,
                                 pre=self.max_pre_decimal,
                                 post=self.max_post_decimal)
            except Exception:
                val = str(val)[:self.data_field_width]
            row.append(val)
        self.table.add_row(row)

        if self.rowwise:
            # Print the last row of data only.
            print(str(self.table).split('\n')[-2])  # [-1] is the bottom border
            # only print header intermittently for rowwise table printing
            if self.num_events_since_last_header >= self.print_header_interval:
                self._print_table_header()
                self.num_events_since_last_header = 0
            self.num_events_since_last_header += 1
        else:
            # print the whole table
            print(self.table)

        sys.stdout.flush()

    def stop(self, stop_document):
        """Print the last row of the table

        Parameters
        ----------
        stop_document : dict
            Not explicitly used in this function, other than to signal that
            the run has been completed
        """
        print(str(self.table).split('\n')[-1])
        # remove all data from the table
        self.table.clear_rows()
        # reset the filestore keys
        self._filestore_keys = set()
<|endoftext|>


<commit_before>import json
import socket
import uuid
from threading import Thread

import byte_utils

class SocketServer:
    def __init__(self, ip, port, motd, version_text, kick_message, samples, server_icon, logger, show_hostname):
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.ip = ip
        self.port = port
        self.motd = motd
        self.version_text = version_text
        self.kick_message = kick_message
        self.samples = samples
        self.server_icon = server_icon
        self.logger = logger
        self.show_hostname = show_hostname

    def on_new_client(self, client_socket, addr):
        data = client_socket.recv(1024)
        client_ip = addr[0]

        try:
            (length, i) = byte_utils.read_varint(data, 0)
            (packetID, i) = byte_utils.read_varint(data, i)

            if packetID == 0:
                (version, i) = byte_utils.read_varint(data, i)
                (ip, i) = byte_utils.read_utf(data, i)

                ip = ip.replace('\x00', '')
                is_using_fml = False

                if ip.endswith("FML"):
                    is_using_fml = True
                    ip = ip[:-3]

                (port, i) = byte_utils.read_ushort(data, i)
                (state, i) = byte_utils.read_varint(data, i)

                fqdn = socket.getfqdn(client_ip)
                if self.show_hostname and client_ip != fqdn:
                    client_ip = fqdn + "/" + client_ip

                if state == 1:
                    self.logger.info(("[%s:%s] Received client " + ("(using ForgeModLoader) " if is_using_fml else "") +
                                      "ping packet (%s:%s).") % (client_ip, addr[1], ip, port))
                    motd = {}
                    motd["version"] = {}
                    motd["version"]["name"] = self.version_text
                    motd["version"]["protocol"] = 2
                    motd["players"] = {}
                    motd["players"]["max"] = 0
                    motd["players"]["online"] = 0
                    motd["players"]["sample"] = []

                    for sample in self.samples:
                        motd["players"]["sample"].append({"name": sample, "id": str(uuid.uuid4())})

                    motd["description"] = {"text": self.motd}

                    if self.server_icon and len(self.server_icon) > 0:
                        motd["favicon"] = self.server_icon

                    self.write_response(client_socket, json.dumps(motd))
                elif state == 2:
                    name = ""
                    if len(data) != i:
                        (some_int, i) = byte_utils.read_varint(data, i)
                        (some_int, i) = byte_utils.read_varint(data, i)
                        (name, i) = byte_utils.read_utf(data, i)
                    self.logger.info(
                        ("[%s:%s] " + (name + " t" if len(name) > 0 else "T") + "ries to connect to the server " +
                         ("(using ForgeModLoader) " if is_using_fml else "") + "(%s:%s).")
                        % (client_ip, addr[1], ip, port))
                    self.write_response(client_socket, json.dumps({"text": self.kick_message}))
                else:
                    self.logger.info(
                        "[%s:%d] Tried to request a login/ping with an unknown state: %d" % (client_ip, addr[1], state))
            elif packetID == 1:
                (long, i) = byte_utils.read_long(data, i)
                response = bytearray()
                byte_utils.write_varint(response, 9)
                byte_utils.write_varint(response, 1)
                bytearray.append(long)
                client_socket.sendall(bytearray)
                self.logger.info("[%s:%d] Responded with pong packet." % (client_ip, addr[1]))
            else:
                self.logger.warning("[%s:%d] Sent an unexpected packet: %d" % (client_ip, addr[1], packetID))
        except (TypeError, IndexError):
            self.logger.warning("[%s:%s] Received invalid data (%s)" % (client_ip, addr[1], data))
            return

    def write_response(self, client_socket, response):
        response_array = bytearray()
        byte_utils.write_varint(response_array, 0)
        byte_utils.write_utf(response_array, response)
        length = bytearray()
        byte_utils.write_varint(length, len(response_array))
        client_socket.sendall(length)
        client_socket.sendall(response_array)

    def start(self):
        self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        self.sock.bind((self.ip, self.port))
        self.sock.settimeout(5000)
        self.sock.listen(30)
        self.logger.info("Server started on %s:%s! Waiting for incoming connections..." % (self.ip, self.port))
        while 1:
            (client, address) = self.sock.accept()
            Thread(target=self.on_new_client, daemon=True, args=(client, address,)).start()

    def close(self):
        self.sock.close()
<commit_msg>[~] Fixed not showing ip if invalid data was sent<commit_after>import json
import socket
import uuid
from threading import Thread

import byte_utils

class SocketServer:
    def __init__(self, ip, port, motd, version_text, kick_message, samples, server_icon, logger, show_hostname):
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.ip = ip
        self.port = port
        self.motd = motd
        self.version_text = version_text
        self.kick_message = kick_message
        self.samples = samples
        self.server_icon = server_icon
        self.logger = logger
        self.show_hostname = show_hostname

    def on_new_client(self, client_socket, addr):
        data = client_socket.recv(1024)
        client_ip = addr[0]

        fqdn = socket.getfqdn(client_ip)
        if self.show_hostname and client_ip != fqdn:
            client_ip = fqdn + "/" + client_ip

        try:
            (length, i) = byte_utils.read_varint(data, 0)
            (packetID, i) = byte_utils.read_varint(data, i)

            if packetID == 0:
                (version, i) = byte_utils.read_varint(data, i)
                (ip, i) = byte_utils.read_utf(data, i)

                ip = ip.replace('\x00', '')
                is_using_fml = False

                if ip.endswith("FML"):
                    is_using_fml = True
                    ip = ip[:-3]

                (port, i) = byte_utils.read_ushort(data, i)
                (state, i) = byte_utils.read_varint(data, i)

                if state == 1:
                    self.logger.info(("[%s:%s] Received client " + ("(using ForgeModLoader) " if is_using_fml else "") +
                                      "ping packet (%s:%s).") % (client_ip, addr[1], ip, port))
                    motd = {}
                    motd["version"] = {}
                    motd["version"]["name"] = self.version_text
                    motd["version"]["protocol"] = 2
                    motd["players"] = {}
                    motd["players"]["max"] = 0
                    motd["players"]["online"] = 0
                    motd["players"]["sample"] = []

                    for sample in self.samples:
                        motd["players"]["sample"].append({"name": sample, "id": str(uuid.uuid4())})

                    motd["description"] = {"text": self.motd}

                    if self.server_icon and len(self.server_icon) > 0:
                        motd["favicon"] = self.server_icon

                    self.write_response(client_socket, json.dumps(motd))
                elif state == 2:
                    name = ""
                    if len(data) != i:
                        (some_int, i) = byte_utils.read_varint(data, i)
                        (some_int, i) = byte_utils.read_varint(data, i)
                        (name, i) = byte_utils.read_utf(data, i)
                    self.logger.info(
                        ("[%s:%s] " + (name + " t" if len(name) > 0 else "T") + "ries to connect to the server " +
                         ("(using ForgeModLoader) " if is_using_fml else "") + "(%s:%s).")
                        % (client_ip, addr[1], ip, port))
                    self.write_response(client_socket, json.dumps({"text": self.kick_message}))
                else:
                    self.logger.info(
                        "[%s:%d] Tried to request a login/ping with an unknown state: %d" % (client_ip, addr[1], state))
            elif packetID == 1:
                (long, i) = byte_utils.read_long(data, i)
                response = bytearray()
                byte_utils.write_varint(response, 9)
                byte_utils.write_varint(response, 1)
                bytearray.append(long)
                client_socket.sendall(bytearray)
                self.logger.info("[%s:%d] Responded with pong packet." % (client_ip, addr[1]))
            else:
                self.logger.warning("[%s:%d] Sent an unexpected packet: %d" % (client_ip, addr[1], packetID))
        except (TypeError, IndexError):
            self.logger.warning("[%s:%s] Received invalid data (%s)" % (client_ip, addr[1], data))
            return

    def write_response(self, client_socket, response):
        response_array = bytearray()
        byte_utils.write_varint(response_array, 0)
        byte_utils.write_utf(response_array, response)
        length = bytearray()
        byte_utils.write_varint(length, len(response_array))
        client_socket.sendall(length)
        client_socket.sendall(response_array)

    def start(self):
        self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        self.sock.bind((self.ip, self.port))
        self.sock.settimeout(5000)
        self.sock.listen(30)
        self.logger.info("Server started on %s:%s! Waiting for incoming connections..." % (self.ip, self.port))
        while 1:
            (client, address) = self.sock.accept()
            Thread(target=self.on_new_client, daemon=True, args=(client, address,)).start()

    def close(self):
        self.sock.close()
<|endoftext|>


<commit_before># Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You may not
# use this file except in compliance with the License. A copy of the License
# is located at
#
#     http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is distributed on
# an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
# express or implied. See the License for the specific language governing
# permissions and limitations under the License.

"""
Simple Training CLI.
"""
import argparse
import json
import os
import pickle
import random
import shutil
import sys
from contextlib import ExitStack
from typing import Optional, Dict

import mxnet as mx
import numpy as np

import sockeye.arguments as arguments
import sockeye.attention
import sockeye.constants as C
import sockeye.data_io
import sockeye.decoder
import sockeye.encoder
import sockeye.initializer
import sockeye.lexicon
import sockeye.lr_scheduler
import sockeye.model
import sockeye.training
import sockeye.utils
import sockeye.vocab
from sockeye.log import setup_main_logger
from sockeye.utils import acquire_gpus, get_num_gpus, expand_requested_device_ids

def none_if_negative(val):
    return None if val < 0 else val

def _build_or_load_vocab(existing_vocab_path: Optional[str], data_path: str, num_words: int,
                         word_min_count: int) -> Dict:
    if existing_vocab_path is None:
        vocabulary = sockeye.vocab.build_from_path(data_path,
                                                   num_words=num_words,
                                                   min_count=word_min_count)
    else:
        vocabulary = sockeye.vocab.vocab_from_json(existing_vocab_path)
    return vocabulary

def _dict_difference(dict1: Dict, dict2: Dict):
    diffs = set()
    for k, v in dict1.items():
        if k not in dict2 or dict2[k] != v:
            diffs.add(k)
    return diffs

def main():
    params = argparse.ArgumentParser(description='CLI to train sockeye sequence-to-sequence models.')
    arguments.add_io_args(params)
    arguments.add_model_parameters(params)
    arguments.add_training_args(params)
    arguments.add_device_args(params)
    args = params.parse_args()

    # seed the RNGs
    np.random.seed(args.seed)
    random.seed(args.seed)
    mx.random.seed(args.seed)

    if args.use_fused_rnn:
        assert not args.use_cpu, "GPU required for FusedRNN cells"

    if args.rnn_residual_connections:
        assert args.rnn_num_layers > 2, "Residual connections require at least 3 RNN layers"

    assert args.optimized_metric == C.BLEU or args.optimized_metric in args.metrics, \
        "Must optimize either BLEU or one of tracked metrics (--metrics)"

    # Checking status of output folder, resumption, etc.
    # Create temporary logger to console only
    logger = setup_main_logger(__name__, file_logging=False, console=not args.quiet)
    output_folder = os.path.abspath(args.output)
    resume_training = False
    training_state_dir = os.path.join(output_folder, C.TRAINING_STATE_DIRNAME)
    if os.path.exists(output_folder):
        if args.overwrite_output:
            logger.info("Removing existing output folder %s.", output_folder)
            shutil.rmtree(output_folder)
            os.makedirs(output_folder)
        elif os.path.exists(training_state_dir):
            with open(os.path.join(output_folder, C.ARGS_STATE_NAME), "r") as fp:
                old_args = json.load(fp)
            arg_diffs = _dict_difference(vars(args), old_args) | _dict_difference(old_args, vars(args))
            # Remove args that may differ without affecting the training.
            arg_diffs -= set(C.ARGS_MAY_DIFFER)
            if not arg_diffs:
                resume_training = True
            else:
                # We do not have the logger yet
                logger.error("Mismatch in arguments for training continuation.")
                logger.error("Differing arguments: %s.", ", ".join(arg_diffs))
                sys.exit(1)
        else:
            logger.error("Refusing to overwrite existing output folder %s.", output_folder)
            sys.exit(1)
    else:
        os.makedirs(output_folder)

    logger = setup_main_logger(__name__,
                               file_logging=True,
                               console=not args.quiet, path=os.path.join(output_folder, C.LOG_NAME))

    logger.info("Command: %s", " ".join(sys.argv))
    logger.info("Arguments: %s", args)
    with open(os.path.join(output_folder, C.ARGS_STATE_NAME), "w") as fp:
        json.dump(vars(args), fp)

    with ExitStack() as exit_stack:
        # context
        if args.use_cpu:
            logger.info("Device: CPU")
            context = [mx.cpu()]
        else:
            num_gpus = get_num_gpus()
            assert num_gpus > 0, "No GPUs found, consider running on the CPU with --use-cpu " \
                                 "(note: check depends on nvidia-smi and this could also mean that the nvidia-smi " \
                                 "binary isn't on the path)."
            if args.disable_device_locking:
                context = expand_requested_device_ids(args.device_ids)
            else:
                context = exit_stack.enter_context(acquire_gpus(args.device_ids, lock_dir=args.lock_dir))
            logger.info("Device(s): GPU %s", context)
            context = [mx.gpu(gpu_id) for gpu_id in context]

        # create vocabs
        vocab_source = _build_or_load_vocab(args.source_vocab, args.source, args.num_words, args.word_min_count)
        sockeye.vocab.vocab_to_json(vocab_source, os.path.join(output_folder, C.VOCAB_SRC_NAME) + C.JSON_SUFFIX)

        vocab_target = _build_or_load_vocab(args.target_vocab, args.target, args.num_words, args.word_min_count)
        sockeye.vocab.vocab_to_json(vocab_target, os.path.join(output_folder, C.VOCAB_TRG_NAME) + C.JSON_SUFFIX)

        vocab_source_size = len(vocab_source)
        vocab_target_size = len(vocab_target)
        logger.info("Vocabulary sizes: source=%d target=%d", vocab_source_size, vocab_target_size)

        data_info = sockeye.data_io.DataInfo(os.path.abspath(args.source),
                                             os.path.abspath(args.target),
                                             os.path.abspath(args.validation_source),
                                             os.path.abspath(args.validation_target),
                                             args.source_vocab,
                                             args.target_vocab)

        # create data iterators
        train_iter, eval_iter = sockeye.data_io.get_training_data_iters(source=data_info.source,
                                                                        target=data_info.target,
                                                                        validation_source=data_info.validation_source,
                                                                        validation_target=data_info.validation_target,
                                                                        vocab_source=vocab_source,
                                                                        vocab_target=vocab_target,
                                                                        batch_size=args.batch_size,
                                                                        fill_up=args.fill_up,
                                                                        max_seq_len=args.max_seq_len,
                                                                        bucketing=not args.no_bucketing,
                                                                        bucket_width=args.bucket_width)

        # learning rate scheduling
        learning_rate_half_life = none_if_negative(args.learning_rate_half_life)
        # TODO: The loading for continuation of the scheduler is done separately from the other parts
        if not resume_training:
            lr_scheduler = sockeye.lr_scheduler.get_lr_scheduler(args.learning_rate_scheduler_type,
                                                                 args.checkpoint_frequency,
                                                                 learning_rate_half_life,
                                                                 args.learning_rate_reduce_factor,
                                                                 args.learning_rate_reduce_num_not_improved)
        else:
            with open(os.path.join(training_state_dir, C.SCHEDULER_STATE_NAME), "rb") as fp:
                lr_scheduler = pickle.load(fp)

        # model configuration
        num_embed_source = args.num_embed if args.num_embed_source is None else args.num_embed_source
        num_embed_target = args.num_embed if args.num_embed_target is None else args.num_embed_target
        attention_num_hidden = args.rnn_num_hidden if not args.attention_num_hidden else args.attention_num_hidden
        model_config = sockeye.model.ModelConfig(max_seq_len=args.max_seq_len,
                                                 vocab_source_size=vocab_source_size,
                                                 vocab_target_size=vocab_target_size,
                                                 num_embed_source=num_embed_source,
                                                 num_embed_target=num_embed_target,
                                                 attention_type=args.attention_type,
                                                 attention_num_hidden=attention_num_hidden,
                                                 attention_coverage_type=args.attention_coverage_type,
                                                 attention_coverage_num_hidden=args.attention_coverage_num_hidden,
                                                 attention_use_prev_word=args.attention_use_prev_word,
                                                 dropout=args.dropout,
                                                 rnn_cell_type=args.rnn_cell_type,
                                                 rnn_num_layers=args.rnn_num_layers,
                                                 rnn_num_hidden=args.rnn_num_hidden,
                                                 rnn_residual_connections=args.rnn_residual_connections,
                                                 weight_tying=args.weight_tying,
                                                 context_gating=args.context_gating,
                                                 lexical_bias=args.lexical_bias,
                                                 learn_lexical_bias=args.learn_lexical_bias,
                                                 data_info=data_info,
                                                 loss=args.loss,
                                                 normalize_loss=args.normalize_loss,
                                                 smoothed_cross_entropy_alpha=args.smoothed_cross_entropy_alpha)

        # create training model
        model = sockeye.training.TrainingModel(model_config=model_config,
                                               context=context,
                                               train_iter=train_iter,
                                               fused=args.use_fused_rnn,
                                               bucketing=not args.no_bucketing,
                                               lr_scheduler=lr_scheduler,
                                               rnn_forget_bias=args.rnn_forget_bias)

        # We may consider loading the params in TrainingModule, for consistency
        # with the training state saving
        if resume_training:
            logger.info("Found partial training in directory %s. Resuming from saved state.", training_state_dir)
            model.load_params_from_file(os.path.join(training_state_dir, C.TRAINING_STATE_PARAMS_NAME))
        elif args.params:
            logger.info("Training will initialize from parameters loaded from '%s'", args.params)
            model.load_params_from_file(args.params)

        lexicon = sockeye.lexicon.initialize_lexicon(args.lexical_bias,
                                                     vocab_source, vocab_target) if args.lexical_bias else None

        initializer = sockeye.initializer.get_initializer(args.rnn_h2h_init, lexicon=lexicon)

        optimizer = args.optimizer
        optimizer_params = {'wd': args.weight_decay,
                            "learning_rate": args.initial_learning_rate}
        if lr_scheduler is not None:
            optimizer_params["lr_scheduler"] = lr_scheduler
        clip_gradient = none_if_negative(args.clip_gradient)
        if clip_gradient is not None:
            optimizer_params["clip_gradient"] = clip_gradient
        if args.momentum is not None:
            optimizer_params["momentum"] = args.momentum
        if args.normalize_loss:
            # When normalize_loss is turned on we normalize by the number of non-PAD symbols in a batch which implicitly
            # already contains the number of sentences and therefore we need to disable rescale_grad.
            optimizer_params["rescale_grad"] = 1.0
        else:
            # Making MXNet module API's default scaling factor explicit
            optimizer_params["rescale_grad"] = 1.0 / args.batch_size
        logger.info("Optimizer: %s", optimizer)
        logger.info("Optimizer Parameters: %s", optimizer_params)

        model.fit(train_iter, eval_iter,
                  output_folder=output_folder,
                  metrics=args.metrics,
                  initializer=initializer,
                  max_updates=args.max_updates,
                  checkpoint_frequency=args.checkpoint_frequency,
                  optimizer=optimizer, optimizer_params=optimizer_params,
                  optimized_metric=args.optimized_metric,
                  max_num_not_improved=args.max_num_checkpoint_not_improved,
                  min_num_epochs=args.min_num_epochs,
                  monitor_bleu=args.monitor_bleu,
                  use_tensorboard=args.use_tensorboard)

if __name__ == "__main__":
    main()
<commit_msg>Logging the Sockeye version when training (#48)<commit_after># Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You may not
# use this file except in compliance with the License. A copy of the License
# is located at
#
#     http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is distributed on
# an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
# express or implied. See the License for the specific language governing
# permissions and limitations under the License.

"""
Simple Training CLI.
"""
import argparse
import json
import os
import pickle
import random
import shutil
import sys
from contextlib import ExitStack
from typing import Optional, Dict

import mxnet as mx
import numpy as np

import sockeye
import sockeye.arguments as arguments
import sockeye.attention
import sockeye.constants as C
import sockeye.data_io
import sockeye.decoder
import sockeye.encoder
import sockeye.initializer
import sockeye.lexicon
import sockeye.lr_scheduler
import sockeye.model
import sockeye.training
import sockeye.utils
import sockeye.vocab
from sockeye.log import setup_main_logger
from sockeye.utils import acquire_gpus, get_num_gpus, expand_requested_device_ids

def none_if_negative(val):
    return None if val < 0 else val

def _build_or_load_vocab(existing_vocab_path: Optional[str], data_path: str, num_words: int,
                         word_min_count: int) -> Dict:
    if existing_vocab_path is None:
        vocabulary = sockeye.vocab.build_from_path(data_path,
                                                   num_words=num_words,
                                                   min_count=word_min_count)
    else:
        vocabulary = sockeye.vocab.vocab_from_json(existing_vocab_path)
    return vocabulary

def _dict_difference(dict1: Dict, dict2: Dict):
    diffs = set()
    for k, v in dict1.items():
        if k not in dict2 or dict2[k] != v:
            diffs.add(k)
    return diffs

def main():
    params = argparse.ArgumentParser(description='CLI to train sockeye sequence-to-sequence models.')
    arguments.add_io_args(params)
    arguments.add_model_parameters(params)
    arguments.add_training_args(params)
    arguments.add_device_args(params)
    args = params.parse_args()

    # seed the RNGs
    np.random.seed(args.seed)
    random.seed(args.seed)
    mx.random.seed(args.seed)

    if args.use_fused_rnn:
        assert not args.use_cpu, "GPU required for FusedRNN cells"

    if args.rnn_residual_connections:
        assert args.rnn_num_layers > 2, "Residual connections require at least 3 RNN layers"

    assert args.optimized_metric == C.BLEU or args.optimized_metric in args.metrics, \
        "Must optimize either BLEU or one of tracked metrics (--metrics)"

    # Checking status of output folder, resumption, etc.
    # Create temporary logger to console only
    logger = setup_main_logger(__name__, file_logging=False, console=not args.quiet)
    output_folder = os.path.abspath(args.output)
    resume_training = False
    training_state_dir = os.path.join(output_folder, C.TRAINING_STATE_DIRNAME)
    if os.path.exists(output_folder):
        if args.overwrite_output:
            logger.info("Removing existing output folder %s.", output_folder)
            shutil.rmtree(output_folder)
            os.makedirs(output_folder)
        elif os.path.exists(training_state_dir):
            with open(os.path.join(output_folder, C.ARGS_STATE_NAME), "r") as fp:
                old_args = json.load(fp)
            arg_diffs = _dict_difference(vars(args), old_args) | _dict_difference(old_args, vars(args))
            # Remove args that may differ without affecting the training.
            arg_diffs -= set(C.ARGS_MAY_DIFFER)
            if not arg_diffs:
                resume_training = True
            else:
                # We do not have the logger yet
                logger.error("Mismatch in arguments for training continuation.")
                logger.error("Differing arguments: %s.", ", ".join(arg_diffs))
                sys.exit(1)
        else:
            logger.error("Refusing to overwrite existing output folder %s.", output_folder)
            sys.exit(1)
    else:
        os.makedirs(output_folder)

    logger = setup_main_logger(__name__,
                               file_logging=True,
                               console=not args.quiet, path=os.path.join(output_folder, C.LOG_NAME))
    logger.info("Sockeye version %s", sockeye.__version__)
    logger.info("Command: %s", " ".join(sys.argv))
    logger.info("Arguments: %s", args)
    with open(os.path.join(output_folder, C.ARGS_STATE_NAME), "w") as fp:
        json.dump(vars(args), fp)

    with ExitStack() as exit_stack:
        # context
        if args.use_cpu:
            logger.info("Device: CPU")
            context = [mx.cpu()]
        else:
            num_gpus = get_num_gpus()
            assert num_gpus > 0, "No GPUs found, consider running on the CPU with --use-cpu " \
                                 "(note: check depends on nvidia-smi and this could also mean that the nvidia-smi " \
                                 "binary isn't on the path)."
            if args.disable_device_locking:
                context = expand_requested_device_ids(args.device_ids)
            else:
                context = exit_stack.enter_context(acquire_gpus(args.device_ids, lock_dir=args.lock_dir))
            logger.info("Device(s): GPU %s", context)
            context = [mx.gpu(gpu_id) for gpu_id in context]

        # create vocabs
        vocab_source = _build_or_load_vocab(args.source_vocab, args.source, args.num_words, args.word_min_count)
        sockeye.vocab.vocab_to_json(vocab_source, os.path.join(output_folder, C.VOCAB_SRC_NAME) + C.JSON_SUFFIX)

        vocab_target = _build_or_load_vocab(args.target_vocab, args.target, args.num_words, args.word_min_count)
        sockeye.vocab.vocab_to_json(vocab_target, os.path.join(output_folder, C.VOCAB_TRG_NAME) + C.JSON_SUFFIX)

        vocab_source_size = len(vocab_source)
        vocab_target_size = len(vocab_target)
        logger.info("Vocabulary sizes: source=%d target=%d", vocab_source_size, vocab_target_size)

        data_info = sockeye.data_io.DataInfo(os.path.abspath(args.source),
                                             os.path.abspath(args.target),
                                             os.path.abspath(args.validation_source),
                                             os.path.abspath(args.validation_target),
                                             args.source_vocab,
                                             args.target_vocab)

        # create data iterators
        train_iter, eval_iter = sockeye.data_io.get_training_data_iters(source=data_info.source,
                                                                        target=data_info.target,
                                                                        validation_source=data_info.validation_source,
                                                                        validation_target=data_info.validation_target,
                                                                        vocab_source=vocab_source,
                                                                        vocab_target=vocab_target,
                                                                        batch_size=args.batch_size,
                                                                        fill_up=args.fill_up,
                                                                        max_seq_len=args.max_seq_len,
                                                                        bucketing=not args.no_bucketing,
                                                                        bucket_width=args.bucket_width)

        # learning rate scheduling
        learning_rate_half_life = none_if_negative(args.learning_rate_half_life)
        # TODO: The loading for continuation of the scheduler is done separately from the other parts
        if not resume_training:
            lr_scheduler = sockeye.lr_scheduler.get_lr_scheduler(args.learning_rate_scheduler_type,
                                                                 args.checkpoint_frequency,
                                                                 learning_rate_half_life,
                                                                 args.learning_rate_reduce_factor,
                                                                 args.learning_rate_reduce_num_not_improved)
        else:
            with open(os.path.join(training_state_dir, C.SCHEDULER_STATE_NAME), "rb") as fp:
                lr_scheduler = pickle.load(fp)

        # model configuration
        num_embed_source = args.num_embed if args.num_embed_source is None else args.num_embed_source
        num_embed_target = args.num_embed if args.num_embed_target is None else args.num_embed_target
        attention_num_hidden = args.rnn_num_hidden if not args.attention_num_hidden else args.attention_num_hidden
        model_config = sockeye.model.ModelConfig(max_seq_len=args.max_seq_len,
                                                 vocab_source_size=vocab_source_size,
                                                 vocab_target_size=vocab_target_size,
                                                 num_embed_source=num_embed_source,
                                                 num_embed_target=num_embed_target,
                                                 attention_type=args.attention_type,
                                                 attention_num_hidden=attention_num_hidden,
                                                 attention_coverage_type=args.attention_coverage_type,
                                                 attention_coverage_num_hidden=args.attention_coverage_num_hidden,
                                                 attention_use_prev_word=args.attention_use_prev_word,
                                                 dropout=args.dropout,
                                                 rnn_cell_type=args.rnn_cell_type,
                                                 rnn_num_layers=args.rnn_num_layers,
                                                 rnn_num_hidden=args.rnn_num_hidden,
                                                 rnn_residual_connections=args.rnn_residual_connections,
                                                 weight_tying=args.weight_tying,
                                                 context_gating=args.context_gating,
                                                 lexical_bias=args.lexical_bias,
                                                 learn_lexical_bias=args.learn_lexical_bias,
                                                 data_info=data_info,
                                                 loss=args.loss,
                                                 normalize_loss=args.normalize_loss,
                                                 smoothed_cross_entropy_alpha=args.smoothed_cross_entropy_alpha)

        # create training model
        model = sockeye.training.TrainingModel(model_config=model_config,
                                               context=context,
                                               train_iter=train_iter,
                                               fused=args.use_fused_rnn,
                                               bucketing=not args.no_bucketing,
                                               lr_scheduler=lr_scheduler,
                                               rnn_forget_bias=args.rnn_forget_bias)

        # We may consider loading the params in TrainingModule, for consistency
        # with the training state saving
        if resume_training:
            logger.info("Found partial training in directory %s. Resuming from saved state.", training_state_dir)
            model.load_params_from_file(os.path.join(training_state_dir, C.TRAINING_STATE_PARAMS_NAME))
        elif args.params:
            logger.info("Training will initialize from parameters loaded from '%s'", args.params)
            model.load_params_from_file(args.params)

        lexicon = sockeye.lexicon.initialize_lexicon(args.lexical_bias,
                                                     vocab_source, vocab_target) if args.lexical_bias else None

        initializer = sockeye.initializer.get_initializer(args.rnn_h2h_init, lexicon=lexicon)

        optimizer = args.optimizer
        optimizer_params = {'wd': args.weight_decay,
                            "learning_rate": args.initial_learning_rate}
        if lr_scheduler is not None:
            optimizer_params["lr_scheduler"] = lr_scheduler
        clip_gradient = none_if_negative(args.clip_gradient)
        if clip_gradient is not None:
            optimizer_params["clip_gradient"] = clip_gradient
        if args.momentum is not None:
            optimizer_params["momentum"] = args.momentum
        if args.normalize_loss:
            # When normalize_loss is turned on we normalize by the number of non-PAD symbols in a batch which implicitly
            # already contains the number of sentences and therefore we need to disable rescale_grad.
            optimizer_params["rescale_grad"] = 1.0
        else:
            # Making MXNet module API's default scaling factor explicit
            optimizer_params["rescale_grad"] = 1.0 / args.batch_size
        logger.info("Optimizer: %s", optimizer)
        logger.info("Optimizer Parameters: %s", optimizer_params)

        model.fit(train_iter, eval_iter,
                  output_folder=output_folder,
                  metrics=args.metrics,
                  initializer=initializer,
                  max_updates=args.max_updates,
                  checkpoint_frequency=args.checkpoint_frequency,
                  optimizer=optimizer, optimizer_params=optimizer_params,
                  optimized_metric=args.optimized_metric,
                  max_num_not_improved=args.max_num_checkpoint_not_improved,
                  min_num_epochs=args.min_num_epochs,
                  monitor_bleu=args.monitor_bleu,
                  use_tensorboard=args.use_tensorboard)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# coding: utf-8

from __future__ import (absolute_import, division,
                        print_function, unicode_literals)

"""
test_django-teryt
------------

Tests for `django-teryt` modules module.
"""
from django.utils import six
from django.test import TestCase
from django.db import models

from ..models import (RodzajMiejscowosci, JednostkaAdministracyjna,
                      Miejscowosc, Ulica)
from .factories import (RodzajMiejscowosciFactory,
                        JednostkaAdministracyjnaFactory,
                        MiejscowoscFactory, UlicaFactory)

class MixinTestObjectsManager(object):
    def test_obects_model_manager(self):
        self.assertIsInstance(JednostkaAdministracyjna.objects, models.Manager)

class TestRodzajMiejscowosci(TestCase, MixinTestObjectsManager):

    def test_str(self):
        rm = RodzajMiejscowosciFactory(id='96', nazwa='miasto')
        self.assertEqual(six.text_type(rm), '96: miasto')

    def test_set_val(self):
        rm = RodzajMiejscowosci()
        rm.set_val({'RM': '01',
                    'STAN_NA': '2013-02-28',
                    'NAZWA_RM': 'wieś'})
        self.assertEqual(rm.nazwa, 'wieś')
        self.assertEqual(rm.id, '01')
        self.assertEqual(rm.stan_na, '2013-02-28')

class TestJednostkaAdministracyjna(TestCase, MixinTestObjectsManager):
    def setUp(self):
        self.rm_miasto = RodzajMiejscowosciFactory(
            id='96',
            nazwa='miasto')

        self.gmina = JednostkaAdministracyjnaFactory(
            id='0201011',
            nazwa='Bolesławiec',
            nazwa_dod='gmina miejska')

        self.gmina_nowogrodziec = JednostkaAdministracyjnaFactory(
            id='0201044',
            nazwa='Nowogrodziec',
            nazwa_dod='miasto')

        self.powiat = JednostkaAdministracyjnaFactory(
            id='0201',
            nazwa='bolesławiecki',
            nazwa_dod='powiat')

        self.wojewodztwo = JednostkaAdministracyjnaFactory(
            id='02',
            nazwa='DOLNOŚLĄSKIE',
            nazwa_dod='wojewdztwo')

        self.boleslawiec = MiejscowoscFactory(
            symbol='0935989',
            jednostka=self.gmina,
            miejscowosc_nadrzedna=None,
            nazwa='Bolesławiec',
            rodzaj_miejscowosci=self.rm_miasto)

        self.nowogrodziec = MiejscowoscFactory(
            symbol='0936262',
            jednostka=self.gmina_nowogrodziec,
            miejscowosc_nadrzedna=None,
            nazwa='Nowogrodziec',
            rodzaj_miejscowosci=self.rm_miasto)

    def test_str(self):
        self.assertEqual(six.text_type(self.gmina), '0201011: Bolesławiec')

    def test_set_val(self):
        gmina = JednostkaAdministracyjna()
        gmina.set_val({
            'GMI': '01',
            'POW': '01',
            'STAN_NA': '2013-01-01',
            'NAZDOD': 'gmina miejska',
            'RODZ': '1',
            'NAZWA': 'Bolesławiec',
            'WOJ': '02'
        })

        wojewodztwo = JednostkaAdministracyjna()
        wojewodztwo.set_val({
            'GMI': None,
            'POW': None,
            'STAN_NA': '2013-01-01',
            'NAZDOD': 'województwo',
            'RODZ': None,
            'NAZWA': u'DOLNOŚLĄSKIE',
            'WOJ': '02'})
        # Common
        self.assertEqual(gmina.stan_na, '2013-01-01')
        self.assertEqual(gmina.aktywny, False)

        # JednostkaAdministracyjna - gmina
        self.assertEqual(gmina.id, '0201011')
        self.assertEqual(gmina.nazwa, 'Bolesławiec')
        self.assertEqual(gmina.nazwa_dod, 'gmina miejska')

        # JednostkaAdministracyjna - województwo
        self.assertEqual(wojewodztwo.id, '02')
        self.assertEqual(wojewodztwo.nazwa, 'dolnośląskie')
        self.assertEqual(wojewodztwo.nazwa_dod, 'województwo')

    def test_parents(self):

        self.assertEqual(self.gmina.powiat(), self.powiat)
        self.assertEqual(self.gmina.wojewodztwo(), self.wojewodztwo)
        self.assertEqual(self.powiat.wojewodztwo(), self.wojewodztwo)

    def test_managers_wojewodztwa(self):
        self.assertIsInstance(JednostkaAdministracyjna.wojewodztwa,
                              models.Manager)
        self.assertEqual(JednostkaAdministracyjna.wojewodztwa.count(), 1)
        JednostkaAdministracyjna.wojewodztwa.get(id='02')

    def test_managers_powiaty(self):
        self.assertIsInstance(JednostkaAdministracyjna.powiaty, models.Manager)
        self.assertEqual(JednostkaAdministracyjna.powiaty.count(), 1)
        JednostkaAdministracyjna.powiaty.get(id='0201')

    def test_managers_gminy(self):
        self.assertIsInstance(JednostkaAdministracyjna.gminy, models.Manager)
        self.assertEqual(JednostkaAdministracyjna.gminy.count(), 2)
        JednostkaAdministracyjna.gminy.get(id='0201011')

    def test_miejscowosci(self):
        self.assertEqual(self.gmina.miejscowosci().count(), 1)
        self.assertEqual(self.powiat.miejscowosci().count(), 2)
        self.assertEqual(self.wojewodztwo.miejscowosci().count(), 2)
        self.gmina.miejscowosci().get(symbol='0935989')

class TestMiejscowosc(TestCase, MixinTestObjectsManager):
    def setUp(self):
        self.miejscowosc = MiejscowoscFactory(
            symbol='0861110',
            miejscowosc_nadrzedna=None,
            nazwa='Strzygowska Kolonia',
            rodzaj_miejscowosci__id='02',
            rodzaj_miejscowosci__nazwa='kolonia')

        self.warszawa = MiejscowoscFactory(
            symbol='0918123',
            miejscowosc_nadrzedna=None,
            nazwa='Warszawa',
            rodzaj_miejscowosci__id='96',
            rodzaj_miejscowosci__nazwa='miasto')

        self.wies = MiejscowoscFactory(
            symbol='0005546',
            miejscowosc_nadrzedna=None,
            nazwa='Wolica',
            rodzaj_miejscowosci__id='01',
            rodzaj_miejscowosci__nazwa='wieś')

    def test_managers_miasta(self):
        self.assertIsInstance(Miejscowosc.miasta, models.Manager)
        self.assertEqual(Miejscowosc.miasta.count(), 1)
        Miejscowosc.miasta.get(symbol='0918123')

    def test_managers_wsie(self):
        self.assertIsInstance(Miejscowosc.wsie, models.Manager)
        self.assertEqual(Miejscowosc.wsie.count(), 1)
        Miejscowosc.wsie.get(symbol='0005546')

    def test_str(self):
        self.assertEqual(six.text_type(self.miejscowosc),
                         '0861110: Strzygowska Kolonia')

    def test_set_val(self):
        m_dict = {
            'GMI': '06',
            'RODZ_GMI': '5',
            'POW': '18',
            'STAN_NA': '2013-03-06',
            'SYM': '0861110',
            'NAZWA': 'Strzygowska Kolonia',
            'WOJ': '04',
            'RM': '02',
            'SYMPOD': '0861110',
            'MZ': '1'
        }
        miejscowosc = Miejscowosc()
        miejscowosc.set_val(m_dict)

        # Common
        self.assertEqual(miejscowosc.stan_na, '2013-03-06')
        self.assertEqual(miejscowosc.aktywny, False)

        # Miejscowosc
        self.assertIsNone(miejscowosc.miejscowosc_nadrzedna)
        self.assertEqual(miejscowosc.symbol, '0861110')
        self.assertEqual(miejscowosc.jednostka_id, '0418065')
        self.assertEqual(miejscowosc.nazwa, 'Strzygowska Kolonia')
        # RodzajMiejscowosci instance made in setUp()
        self.assertEqual(miejscowosc.rodzaj_miejscowosci.nazwa, 'kolonia')

        m_dict['SYMPOD'] = '1234567'
        miejscowosc2 = Miejscowosc()
        miejscowosc2.set_val(m_dict)
        self.assertEqual(miejscowosc2.miejscowosc_nadrzedna_id, '1234567')

class TestUlica(TestCase, MixinTestObjectsManager):
    def setUp(self):
        self.ulica1 = UlicaFactory(
            miejscowosc__miejscowosc_nadrzedna=None
        )

        self.ulica2 = UlicaFactory(
            cecha='pl.',
            nazwa_1='Hoffa',
            nazwa_2='Bogumiła',
            miejscowosc__miejscowosc_nadrzedna=None
        )

    def test_str(self):
        self.assertEqual(six.text_type(self.ulica1), 'ul. {} ({})'.format(
            self.ulica1.nazwa_1, self.ulica1.miejscowosc.nazwa))

        self.assertEqual(six.text_type(self.ulica2),
                         'pl. Bogumiła Hoffa ({})'.format(
                         self.ulica2.miejscowosc.nazwa))

    def test_set_val(self):
        m_dict = {
            'GMI': '03',
            'RODZ_GMI': '2',
            'NAZWA_1': 'Cicha',
            'NAZWA_2': None,
            'POW': '03',
            'STAN_NA': '2013-12-16',
            'SYM': '0185962',
            'CECHA': 'ul.',
            'WOJ': '08',
            'SYM_UL': '02974'
        }
        ulica = Ulica()
        ulica.set_val(m_dict)

        # Common
        self.assertEqual(ulica.stan_na, '2013-12-16')
        self.assertEqual(ulica.aktywny, False)

        # Ulica
        self.assertEqual(ulica.id, '018596202974')
        self.assertEqual(ulica.miejscowosc_id, '0185962')
        self.assertEqual(ulica.symbol_ulicy, '02974')
        self.assertEqual(ulica.cecha, 'ul.')
        self.assertEqual(ulica.nazwa_1, 'Cicha')
        self.assertIsNone(ulica.nazwa_2)
<commit_msg>Fix typo in teryt_models.py<commit_after>#!/usr/bin/env python
# coding: utf-8

from __future__ import (absolute_import, division,
                        print_function, unicode_literals)

"""
test_django-teryt
------------

Tests for `django-teryt` modules module.
"""
from django.utils import six
from django.test import TestCase
from django.db import models

from ..models import (RodzajMiejscowosci, JednostkaAdministracyjna,
                      Miejscowosc, Ulica)
from .factories import (RodzajMiejscowosciFactory,
                        JednostkaAdministracyjnaFactory,
                        MiejscowoscFactory, UlicaFactory)

class MixinTestObjectsManager(object):
    def test_objects_model_manager(self):
        self.assertIsInstance(JednostkaAdministracyjna.objects, models.Manager)

class TestRodzajMiejscowosci(TestCase, MixinTestObjectsManager):

    def test_str(self):
        rm = RodzajMiejscowosciFactory(id='96', nazwa='miasto')
        self.assertEqual(six.text_type(rm), '96: miasto')

    def test_set_val(self):
        rm = RodzajMiejscowosci()
        rm.set_val({'RM': '01',
                    'STAN_NA': '2013-02-28',
                    'NAZWA_RM': 'wieś'})
        self.assertEqual(rm.nazwa, 'wieś')
        self.assertEqual(rm.id, '01')
        self.assertEqual(rm.stan_na, '2013-02-28')

class TestJednostkaAdministracyjna(TestCase, MixinTestObjectsManager):
    def setUp(self):
        self.rm_miasto = RodzajMiejscowosciFactory(
            id='96',
            nazwa='miasto')

        self.gmina = JednostkaAdministracyjnaFactory(
            id='0201011',
            nazwa='Bolesławiec',
            nazwa_dod='gmina miejska')

        self.gmina_nowogrodziec = JednostkaAdministracyjnaFactory(
            id='0201044',
            nazwa='Nowogrodziec',
            nazwa_dod='miasto')

        self.powiat = JednostkaAdministracyjnaFactory(
            id='0201',
            nazwa='bolesławiecki',
            nazwa_dod='powiat')

        self.wojewodztwo = JednostkaAdministracyjnaFactory(
            id='02',
            nazwa='DOLNOŚLĄSKIE',
            nazwa_dod='wojewdztwo')

        self.boleslawiec = MiejscowoscFactory(
            symbol='0935989',
            jednostka=self.gmina,
            miejscowosc_nadrzedna=None,
            nazwa='Bolesławiec',
            rodzaj_miejscowosci=self.rm_miasto)

        self.nowogrodziec = MiejscowoscFactory(
            symbol='0936262',
            jednostka=self.gmina_nowogrodziec,
            miejscowosc_nadrzedna=None,
            nazwa='Nowogrodziec',
            rodzaj_miejscowosci=self.rm_miasto)

    def test_str(self):
        self.assertEqual(six.text_type(self.gmina), '0201011: Bolesławiec')

    def test_set_val(self):
        gmina = JednostkaAdministracyjna()
        gmina.set_val({
            'GMI': '01',
            'POW': '01',
            'STAN_NA': '2013-01-01',
            'NAZDOD': 'gmina miejska',
            'RODZ': '1',
            'NAZWA': 'Bolesławiec',
            'WOJ': '02'
        })

        wojewodztwo = JednostkaAdministracyjna()
        wojewodztwo.set_val({
            'GMI': None,
            'POW': None,
            'STAN_NA': '2013-01-01',
            'NAZDOD': 'województwo',
            'RODZ': None,
            'NAZWA': u'DOLNOŚLĄSKIE',
            'WOJ': '02'})
        # Common
        self.assertEqual(gmina.stan_na, '2013-01-01')
        self.assertEqual(gmina.aktywny, False)

        # JednostkaAdministracyjna - gmina
        self.assertEqual(gmina.id, '0201011')
        self.assertEqual(gmina.nazwa, 'Bolesławiec')
        self.assertEqual(gmina.nazwa_dod, 'gmina miejska')

        # JednostkaAdministracyjna - województwo
        self.assertEqual(wojewodztwo.id, '02')
        self.assertEqual(wojewodztwo.nazwa, 'dolnośląskie')
        self.assertEqual(wojewodztwo.nazwa_dod, 'województwo')

    def test_parents(self):

        self.assertEqual(self.gmina.powiat(), self.powiat)
        self.assertEqual(self.gmina.wojewodztwo(), self.wojewodztwo)
        self.assertEqual(self.powiat.wojewodztwo(), self.wojewodztwo)

    def test_managers_wojewodztwa(self):
        self.assertIsInstance(JednostkaAdministracyjna.wojewodztwa,
                              models.Manager)
        self.assertEqual(JednostkaAdministracyjna.wojewodztwa.count(), 1)
        JednostkaAdministracyjna.wojewodztwa.get(id='02')

    def test_managers_powiaty(self):
        self.assertIsInstance(JednostkaAdministracyjna.powiaty, models.Manager)
        self.assertEqual(JednostkaAdministracyjna.powiaty.count(), 1)
        JednostkaAdministracyjna.powiaty.get(id='0201')

    def test_managers_gminy(self):
        self.assertIsInstance(JednostkaAdministracyjna.gminy, models.Manager)
        self.assertEqual(JednostkaAdministracyjna.gminy.count(), 2)
        JednostkaAdministracyjna.gminy.get(id='0201011')

    def test_miejscowosci(self):
        self.assertEqual(self.gmina.miejscowosci().count(), 1)
        self.assertEqual(self.powiat.miejscowosci().count(), 2)
        self.assertEqual(self.wojewodztwo.miejscowosci().count(), 2)
        self.gmina.miejscowosci().get(symbol='0935989')

class TestMiejscowosc(TestCase, MixinTestObjectsManager):
    def setUp(self):
        self.miejscowosc = MiejscowoscFactory(
            symbol='0861110',
            miejscowosc_nadrzedna=None,
            nazwa='Strzygowska Kolonia',
            rodzaj_miejscowosci__id='02',
            rodzaj_miejscowosci__nazwa='kolonia')

        self.warszawa = MiejscowoscFactory(
            symbol='0918123',
            miejscowosc_nadrzedna=None,
            nazwa='Warszawa',
            rodzaj_miejscowosci__id='96',
            rodzaj_miejscowosci__nazwa='miasto')

        self.wies = MiejscowoscFactory(
            symbol='0005546',
            miejscowosc_nadrzedna=None,
            nazwa='Wolica',
            rodzaj_miejscowosci__id='01',
            rodzaj_miejscowosci__nazwa='wieś')

    def test_managers_miasta(self):
        self.assertIsInstance(Miejscowosc.miasta, models.Manager)
        self.assertEqual(Miejscowosc.miasta.count(), 1)
        Miejscowosc.miasta.get(symbol='0918123')

    def test_managers_wsie(self):
        self.assertIsInstance(Miejscowosc.wsie, models.Manager)
        self.assertEqual(Miejscowosc.wsie.count(), 1)
        Miejscowosc.wsie.get(symbol='0005546')

    def test_str(self):
        self.assertEqual(six.text_type(self.miejscowosc),
                         '0861110: Strzygowska Kolonia')

    def test_set_val(self):
        m_dict = {
            'GMI': '06',
            'RODZ_GMI': '5',
            'POW': '18',
            'STAN_NA': '2013-03-06',
            'SYM': '0861110',
            'NAZWA': 'Strzygowska Kolonia',
            'WOJ': '04',
            'RM': '02',
            'SYMPOD': '0861110',
            'MZ': '1'
        }
        miejscowosc = Miejscowosc()
        miejscowosc.set_val(m_dict)

        # Common
        self.assertEqual(miejscowosc.stan_na, '2013-03-06')
        self.assertEqual(miejscowosc.aktywny, False)

        # Miejscowosc
        self.assertIsNone(miejscowosc.miejscowosc_nadrzedna)
        self.assertEqual(miejscowosc.symbol, '0861110')
        self.assertEqual(miejscowosc.jednostka_id, '0418065')
        self.assertEqual(miejscowosc.nazwa, 'Strzygowska Kolonia')
        # RodzajMiejscowosci instance made in setUp()
        self.assertEqual(miejscowosc.rodzaj_miejscowosci.nazwa, 'kolonia')

        m_dict['SYMPOD'] = '1234567'
        miejscowosc2 = Miejscowosc()
        miejscowosc2.set_val(m_dict)
        self.assertEqual(miejscowosc2.miejscowosc_nadrzedna_id, '1234567')

class TestUlica(TestCase, MixinTestObjectsManager):
    def setUp(self):
        self.ulica1 = UlicaFactory(
            miejscowosc__miejscowosc_nadrzedna=None
        )

        self.ulica2 = UlicaFactory(
            cecha='pl.',
            nazwa_1='Hoffa',
            nazwa_2='Bogumiła',
            miejscowosc__miejscowosc_nadrzedna=None
        )

    def test_str(self):
        self.assertEqual(six.text_type(self.ulica1), 'ul. {} ({})'.format(
            self.ulica1.nazwa_1, self.ulica1.miejscowosc.nazwa))

        self.assertEqual(six.text_type(self.ulica2),
                         'pl. Bogumiła Hoffa ({})'.format(
                         self.ulica2.miejscowosc.nazwa))

    def test_set_val(self):
        m_dict = {
            'GMI': '03',
            'RODZ_GMI': '2',
            'NAZWA_1': 'Cicha',
            'NAZWA_2': None,
            'POW': '03',
            'STAN_NA': '2013-12-16',
            'SYM': '0185962',
            'CECHA': 'ul.',
            'WOJ': '08',
            'SYM_UL': '02974'
        }
        ulica = Ulica()
        ulica.set_val(m_dict)

        # Common
        self.assertEqual(ulica.stan_na, '2013-12-16')
        self.assertEqual(ulica.aktywny, False)

        # Ulica
        self.assertEqual(ulica.id, '018596202974')
        self.assertEqual(ulica.miejscowosc_id, '0185962')
        self.assertEqual(ulica.symbol_ulicy, '02974')
        self.assertEqual(ulica.cecha, 'ul.')
        self.assertEqual(ulica.nazwa_1, 'Cicha')
        self.assertIsNone(ulica.nazwa_2)
<|endoftext|>