<commit_before>"""Create a Graph class data structure."""
from stack import Stack

class Graph(object):
    """Graph object."""

    def __init__(self):
        """Graph things on initialization."""
        self._nodes = {}
        self._weights = {}

    def nodes(self):
        """List all nodes in the Graph."""
        return [key for key in self._nodes]

    def edges(self):
        """List all edges in the Graph."""
        return self._nodes

    def weights(self):
        """List all the edges weights in the Graph."""
        return self._weights

    def add_node(self, *args):
        """Add a new node to the Graph."""
        for arg in args:
            self._nodes.setdefault(arg, [])

    def add_edge(self, val1, val2):
        """Add a new edge to the Graph and connects the values.

        If either value doesn't exist they will be created.
        """
        self.add_node(val1, val2)
        if val2 in self._nodes[val1]:  # pragma: no cover
            pass
        else:
            weight = self.add_weight(val1, val2)
            self._nodes[val1].append(val2)
            self._weights['{}-{}'.format(val1, val2)] = weight

    def del_node(self, val):
        """Delete the node containing the given value."""
        try:
            del self._nodes[val]
            for key in self._nodes:
                if val in self._nodes[key]:
                    self._nodes[key].remove(val)
        except KeyError:
            raise KeyError("No such Node exists")

    def del_edge(self, val1, val2):
        """Delete the edge connecting the two values."""
        try:
            for node in self._nodes[val1]:
                if val2 == node:
                    self._nodes[val1].remove(node)
                    return
            raise IndexError("No connection between those two Nodes.")
        except KeyError:
            raise KeyError("No Node with given first value.")

    def has_node(self, val):
        """Return True is node containing value exists in Graph."""
        return True if val in self._nodes else False

    def neighbors(self, val):
        """Return list of all nodes connected to the node containing the value."""
        return self._nodes[val]

    def adjacent(self, val1, val2):
        """Return True if there is an edge connecting the two values."""
        return val2 in self._nodes[val1]

    def depth_first_traversal(self, start):
        """Perform a depth-first traversal and return full visited path."""
        if start not in self._nodes:
            raise KeyError("No such value")
        path = []
        visit = Stack()
        curr = start
        while True:
            try:
                neighbors = self.neighbors(curr)
                for n in neighbors:
                    visit.push(n)
                path.append(curr)
                curr = visit.pop()
            except IndexError:
                break
        return path

    def breadth_first_traversal(self, start):
        """Perform a breadth-first traversal and return full visited path."""
        if start not in self._nodes:
            raise KeyError("No such value")
        path, visit = [], []
        curr = start
        while True:
            try:
                path.append(curr)
                visit.extend(self.neighbors(curr))
                curr = visit[0]
                visit = visit[1:]
            except IndexError:
                break
        return path

    def add_weight(self, val1, val2):
        """."""
        if isinstance(val1, str) and isinstance(val2, str):
            return abs(ord(val1) - ord(val2))
        if isinstance(val1, int) and isinstance(val2, int):
            return abs(val1 - val2)
<commit_msg>minor text fix<commit_after>"""Create a Graph class data structure."""
from stack import Stack

class Graph(object):
    """Graph object."""

    def __init__(self):
        """Graph things on initialization."""
        self._nodes = {}
        self._weights = {}

    def nodes(self):
        """List all nodes in the Graph."""
        return [key for key in self._nodes]

    def edges(self):
        """List all edges in the Graph."""
        return "Edges: {} Weights: {}".format(self._nodes, self._weights)

    def weights(self):
        """List all the edges weights in the Graph."""
        return self._weights

    def add_node(self, *args):
        """Add a new node to the Graph."""
        for arg in args:
            self._nodes.setdefault(arg, [])

    def add_edge(self, val1, val2):
        """Add a new edge to the Graph and connects the values.

        If either value doesn't exist they will be created.
        """
        self.add_node(val1, val2)
        if val2 in self._nodes[val1]:  # pragma: no cover
            pass
        else:
            weight = self.add_weight(val1, val2)
            self._nodes[val1].append(val2)
            self._weights['{}-{}'.format(val1, val2)] = weight

    def del_node(self, val):
        """Delete the node containing the given value."""
        try:
            del self._nodes[val]
            for key in self._nodes:
                if val in self._nodes[key]:
                    self._nodes[key].remove(val)
        except KeyError:
            raise KeyError("No such Node exists")

    def del_edge(self, val1, val2):
        """Delete the edge connecting the two values."""
        try:
            for node in self._nodes[val1]:
                if val2 == node:
                    self._nodes[val1].remove(node)
                    del self._weights['{}-{}'.format(val1, val2)]
                    return
            raise IndexError("No connection between those two Nodes.")
        except KeyError:
            raise KeyError("No Node with given first value.")

    def has_node(self, val):
        """Return True is node containing value exists in Graph."""
        return True if val in self._nodes else False

    def neighbors(self, val):
        """Return list of all nodes connected to the node containing the value."""
        return self._nodes[val]

    def adjacent(self, val1, val2):
        """Return True if there is an edge connecting the two values."""
        return val2 in self._nodes[val1]

    def depth_first_traversal(self, start):
        """Perform a depth-first traversal and return full visited path."""
        if start not in self._nodes:
            raise KeyError("No such value")
        path = []
        visit = Stack()
        curr = start
        while True:
            try:
                neighbors = self.neighbors(curr)
                for n in neighbors:
                    visit.push(n)
                path.append(curr)
                curr = visit.pop()
            except IndexError:
                break
        return path

    def breadth_first_traversal(self, start):
        """Perform a breadth-first traversal and return full visited path."""
        if start not in self._nodes:
            raise KeyError("No such value")
        path, visit = [], []
        curr = start
        while True:
            try:
                path.append(curr)
                visit.extend(self.neighbors(curr))
                curr = visit[0]
                visit = visit[1:]
            except IndexError:
                break
        return path

    def add_weight(self, val1, val2):
        """."""
        if isinstance(val1, str) and isinstance(val2, str):
            return abs(ord(val1) - ord(val2))
        if isinstance(val1, int) and isinstance(val2, int):
            return abs(val1 - val2)
<|endoftext|>


<commit_before>
import pygame
from tile import Tile
from actor import Actor
from genie import Genie

class Level:
	def __init__(self, game, player, window, arrayOrFilename):
		self.window = window
		self.game = game
		self.player = player
		self.parallaxHeight = 0
		self.parallaxImageHeight = 480 # this should be the height of the image martin made
		if type(arrayOrFilename) == type("Jordan"):
			self.loadLevelFile(arrayOrFilename)
		else:
			self.levelArray = arrayOrFilename
		x = 0
		y = 0
		self.walls = []
		self.backgrounds = []
		self.levelWidth = 0
		self.levelHeight = 0

		enemyList = []
		genieList = []
		for row in self.levelArray:
			self.levelHeight += 1
			for item in row:
				if item == "O":
					enemyList += [Actor(self.window, self.game, x, y-64, "Ostrich")]
				elif item == "G":
					genieList += [Genie(self.window, self.game, self.player, x-64, y-87, self.genieMessage, self.genieWishes, False)] # THERE'S ONLY ONE GENIE PER LEVEL, IF THERE ISN'T YOU'RE FUCKED!
				elif item == "g":
					genieList += [Genie(self.window, self.game, self.player, x-64, y-87, self.genieMessage, self.genieWishes)]
				elif item == "P":
					if y==0 or self.levelArray[round((y/64)-1)][round(x/64)]!="P":
						if (round(x/64) < (len(row)) - 1 and self.levelArray[round(y/64)][round(x/64)+1]!="P"):
							tile = Tile(self.window, x, y, "Ground", self.game.groundRImage)
						elif (round(x/64) > 0 and self.levelArray[round(y/64)][round(x/64)-1]!="P"):
							tile = Tile(self.window, x, y, "Ground", self.game.groundLImage)
						else:
							tile = Tile(self.window, x, y, "Ground", self.game.groundImage)
					else:
						tile = Tile(self.window, x, y, "Ground", self.game.dirtImage)
					self.walls.append(tile)
				elif item == "S":
					if (round(x/64) < (len(row)) - 1 and self.levelArray[round(y/64)][round(x/64)+1]!="S" and round(x/64) > 0 and self.levelArray[round(y/64)][round(x/64)-1]!="S"):
						tile = Tile(self.window, x, y, "Ground", self.game.stoneCImage)
					elif (round(x/64) < (len(row)) - 1 and self.levelArray[round(y/64)][round(x/64)+1]!="S"):
						tile = Tile(self.window, x, y, "Ground", self.game.stoneRImage)
					elif (round(x/64) > 0 and self.levelArray[round(y/64)][round(x/64)-1]!="S"):
						tile = Tile(self.window, x, y, "Ground", self.game.stoneLImage)
					else:
						tile = Tile(self.window, x, y, "Ground", self.game.stoneImage)
					self.walls.append(tile)
				elif item == "|":
					if y==0 or self.levelArray[round((y/64)-1)][round(x/64)]!="|":
						tile = Tile(self.window, x, y, "Ground", self.game.pillarTImage)
					else:
						tile = Tile(self.window, x, y, "Ground", self.game.pillarImage)
					self.walls.append(tile)
				elif item == "^":
					tile = Tile(self.window, x, y, "Spike", self.game.spikeImage)
					self.backgrounds.append(tile)
				elif item == "H":
					tile = Tile(self.window, x, y, "Hidden", self.game.waterImage)
					self.walls.append(tile)
				elif item == "T":
					tile = Tile(self.window, x, y-64, "Torch", self.game.torchImage[self.game.animation])
					self.backgrounds.append(tile)
				elif item == ".":
					if y==0 or self.levelArray[round((y/64)-1)][round(x/64)]==" ":
						tile = Tile(self.window, x, y, "Wave", self.game.waveImage[self.game.animation])
					else:
						tile = Tile(self.window, x, y, "Water", self.game.waterImage)
					self.backgrounds.append(tile)
				x += 64
			y += 64
			x = 0
		self.levelWidth = len(self.levelArray[0])*64
		self.levelHeight = len(self.levelArray)*64
		self.staticTiles = pygame.Surface((self.levelWidth, self.levelHeight)) #(64*self.levelWidth, 64*self.levelHeight)
		self.belowGroundBackground = pygame.Surface((self.levelWidth, self.levelHeight))
		self.belowGroundBackground.fill((109, 99, 52))
		self.staticTiles.fill((255,255,255,0))
		self.staticTiles.set_colorkey((255,255,255,0))
		self.makeBackgroundImage()
		self.game.enemyList.append(enemyList)
		self.game.genieList.append(genieList)

	def loadLevelFile(self, filename):
		f = open(filename, "r")
		lines = f.readlines()
		firstLine = lines[0].split()
		self.parallaxHeight = float(firstLine[0])*64
		lengthOfMessage = int(lines[1])
		self.genieMessage = [line[:-1] for line in lines[2:2+lengthOfMessage]]
		self.genieWishes = [line[:-1] for line in lines[2+lengthOfMessage : 6+lengthOfMessage]]

		self.levelArray = [line[:-1] for line in lines[6+lengthOfMessage:]]
		f.close()

	def getLevelWidth(self):
		return self.levelWidth

	def getLevelHeight(self):
		return self.levelHeight

	def makeBackgroundImage(self):
		for tile in self.walls:
			tile.drawTo(0, 0, self.staticTiles)

	def getWalls(self):
		return self.walls

	def getBackgrounds(self):
		return self.backgrounds

	def drawParallax(self, cameraX, cameraY):
		parallaxWidth = self.game.screenWidth-32
		x = -((cameraX*self.game.far_parallax_scale)%(parallaxWidth))
		# do the background one
		self.window.blit(self.game.farParallax, (x, (self.parallaxHeight-self.game.screenHeight/2)-cameraY)) # -cameraY*self.game.close_parallax_scale
		self.window.blit(self.game.farParallax, (x+parallaxWidth, (self.parallaxHeight-self.game.screenHeight/2)-cameraY))
		self.window.blit(self.game.farParallax, (x+2*parallaxWidth, (self.parallaxHeight-self.game.screenHeight/2)-cameraY))
		x = -((cameraX*self.game.close_parallax_scale)%(parallaxWidth))
		# then do the foreground one
		self.window.blit(self.game.closeParallax, (x, (self.parallaxHeight-self.game.screenHeight/2)-cameraY)) # -cameraY*self.game.close_parallax_scale
		self.window.blit(self.game.closeParallax, (x+parallaxWidth, (self.parallaxHeight-self.game.screenHeight/2)-cameraY))
		self.window.blit(self.game.closeParallax, (x+2*parallaxWidth, (self.parallaxHeight-self.game.screenHeight/2)-cameraY))
		self.window.blit(self.belowGroundBackground, (0, (self.parallaxHeight-self.game.screenHeight/2)-cameraY+self.parallaxImageHeight, self.getLevelWidth(), self.getLevelHeight()))
		# backgroundColor = (109, 99, 52)
		# pygame.draw.rect(self.window, backgroundColor, (0, (self.parallaxHeight-self.game.screenHeight/2)-cameraY+self.parallaxImageHeight, self.getLevelWidth(), self.getLevelHeight()))

	def update(self, cameraX, cameraY):
		if self.player.rect.right<0:
			print("THIS SHOULDNT'T HAPPEN! THIS IS LEVEL UPDATING!")
			if self.game.levelCounter < len(self.game.levelList)-1:
				self.game.levelCounter+=1
				self.game.setCurrentLevel(self.game.levelList[self.game.levelCounter])

				self.player.rect.x=SCREENWIDTH-self.player.rect.width
				self.player.rect.y -= 15
				self.game.levelTitle = Word("LEVEL " + str(len(self.game.levelList)-self.game.levelCounter) + ": " + self.game.nameList[self.game.levelCounter], 20, 20, 30, [255, 255, 255])
			else:
				self.game.gameState = "ENDSCREEN"

		# for tile in self.walls:
		# 	tile.update(cameraX, cameraY)
		self.window.blit(self.staticTiles, (-cameraX, -cameraY))
		for tile in self.backgrounds:
			if tile.name=="Wave":
				tile.image = self.game.waveImage[self.game.animation]
			elif tile.name=="Torch":
				tile.image = self.game.torchImage[self.game.animation]

			tile.update(cameraX, cameraY)

<commit_msg>slight performance upgrade for parallax?<commit_after>
import pygame
from tile import Tile
from actor import Actor
from genie import Genie

class Level:
	def __init__(self, game, player, window, arrayOrFilename):
		self.window = window
		self.game = game
		self.player = player
		self.parallaxHeight = 0
		self.parallaxImageHeight = 480 # this should be the height of the image martin made
		if type(arrayOrFilename) == type("Jordan"):
			self.loadLevelFile(arrayOrFilename)
		else:
			self.levelArray = arrayOrFilename
		x = 0
		y = 0
		self.walls = []
		self.backgrounds = []
		self.levelWidth = 0
		self.levelHeight = 0

		enemyList = []
		genieList = []
		for row in self.levelArray:
			self.levelHeight += 1
			for item in row:
				if item == "O":
					enemyList += [Actor(self.window, self.game, x, y-64, "Ostrich")]
				elif item == "G":
					genieList += [Genie(self.window, self.game, self.player, x-64, y-87, self.genieMessage, self.genieWishes, False)] # THERE'S ONLY ONE GENIE PER LEVEL, IF THERE ISN'T YOU'RE FUCKED!
				elif item == "g":
					genieList += [Genie(self.window, self.game, self.player, x-64, y-87, self.genieMessage, self.genieWishes)]
				elif item == "P":
					if y==0 or self.levelArray[round((y/64)-1)][round(x/64)]!="P":
						if (round(x/64) < (len(row)) - 1 and self.levelArray[round(y/64)][round(x/64)+1]!="P"):
							tile = Tile(self.window, x, y, "Ground", self.game.groundRImage)
						elif (round(x/64) > 0 and self.levelArray[round(y/64)][round(x/64)-1]!="P"):
							tile = Tile(self.window, x, y, "Ground", self.game.groundLImage)
						else:
							tile = Tile(self.window, x, y, "Ground", self.game.groundImage)
					else:
						tile = Tile(self.window, x, y, "Ground", self.game.dirtImage)
					self.walls.append(tile)
				elif item == "S":
					if (round(x/64) < (len(row)) - 1 and self.levelArray[round(y/64)][round(x/64)+1]!="S" and round(x/64) > 0 and self.levelArray[round(y/64)][round(x/64)-1]!="S"):
						tile = Tile(self.window, x, y, "Ground", self.game.stoneCImage)
					elif (round(x/64) < (len(row)) - 1 and self.levelArray[round(y/64)][round(x/64)+1]!="S"):
						tile = Tile(self.window, x, y, "Ground", self.game.stoneRImage)
					elif (round(x/64) > 0 and self.levelArray[round(y/64)][round(x/64)-1]!="S"):
						tile = Tile(self.window, x, y, "Ground", self.game.stoneLImage)
					else:
						tile = Tile(self.window, x, y, "Ground", self.game.stoneImage)
					self.walls.append(tile)
				elif item == "|":
					if y==0 or self.levelArray[round((y/64)-1)][round(x/64)]!="|":
						tile = Tile(self.window, x, y, "Ground", self.game.pillarTImage)
					else:
						tile = Tile(self.window, x, y, "Ground", self.game.pillarImage)
					self.walls.append(tile)
				elif item == "^":
					tile = Tile(self.window, x, y, "Spike", self.game.spikeImage)
					self.backgrounds.append(tile)
				elif item == "H":
					tile = Tile(self.window, x, y, "Hidden", self.game.waterImage)
					self.walls.append(tile)
				elif item == "T":
					tile = Tile(self.window, x, y-64, "Torch", self.game.torchImage[self.game.animation])
					self.backgrounds.append(tile)
				elif item == ".":
					if y==0 or self.levelArray[round((y/64)-1)][round(x/64)]==" ":
						tile = Tile(self.window, x, y, "Wave", self.game.waveImage[self.game.animation])
					else:
						tile = Tile(self.window, x, y, "Water", self.game.waterImage)
					self.backgrounds.append(tile)
				x += 64
			y += 64
			x = 0
		self.levelWidth = len(self.levelArray[0])*64
		self.levelHeight = len(self.levelArray)*64
		self.staticTiles = pygame.Surface((self.levelWidth, self.levelHeight)) #(64*self.levelWidth, 64*self.levelHeight)
		self.belowGroundBackground = pygame.Surface((self.game.screenWidth, self.game.screenHeight))
		self.belowGroundBackground.fill((109, 99, 52))
		self.staticTiles.fill((255,255,255,0))
		self.staticTiles.set_colorkey((255,255,255,0))
		self.makeBackgroundImage()
		self.game.enemyList.append(enemyList)
		self.game.genieList.append(genieList)

	def loadLevelFile(self, filename):
		f = open(filename, "r")
		lines = f.readlines()
		firstLine = lines[0].split()
		self.parallaxHeight = float(firstLine[0])*64
		lengthOfMessage = int(lines[1])
		self.genieMessage = [line[:-1] for line in lines[2:2+lengthOfMessage]]
		self.genieWishes = [line[:-1] for line in lines[2+lengthOfMessage : 6+lengthOfMessage]]

		self.levelArray = [line[:-1] for line in lines[6+lengthOfMessage:]]
		f.close()

	def getLevelWidth(self):
		return self.levelWidth

	def getLevelHeight(self):
		return self.levelHeight

	def makeBackgroundImage(self):
		for tile in self.walls:
			tile.drawTo(0, 0, self.staticTiles)

	def getWalls(self):
		return self.walls

	def getBackgrounds(self):
		return self.backgrounds

	def drawParallax(self, cameraX, cameraY):
		parallaxWidth = self.game.screenWidth-32
		x = -((cameraX*self.game.far_parallax_scale)%(parallaxWidth))
		# do the background one
		self.window.blit(self.game.farParallax, (x, (self.parallaxHeight-self.game.screenHeight/2)-cameraY)) # -cameraY*self.game.close_parallax_scale
		self.window.blit(self.game.farParallax, (x+parallaxWidth, (self.parallaxHeight-self.game.screenHeight/2)-cameraY))
		self.window.blit(self.game.farParallax, (x+2*parallaxWidth, (self.parallaxHeight-self.game.screenHeight/2)-cameraY))
		x = -((cameraX*self.game.close_parallax_scale)%(parallaxWidth))
		# then do the foreground one
		self.window.blit(self.game.closeParallax, (x, (self.parallaxHeight-self.game.screenHeight/2)-cameraY)) # -cameraY*self.game.close_parallax_scale
		self.window.blit(self.game.closeParallax, (x+parallaxWidth, (self.parallaxHeight-self.game.screenHeight/2)-cameraY))
		self.window.blit(self.game.closeParallax, (x+2*parallaxWidth, (self.parallaxHeight-self.game.screenHeight/2)-cameraY))
		self.window.blit(self.belowGroundBackground, (0, (self.parallaxHeight-self.game.screenHeight/2)-cameraY+self.parallaxImageHeight))
		# backgroundColor = (109, 99, 52)
		# pygame.draw.rect(self.window, backgroundColor, (0, (self.parallaxHeight-self.game.screenHeight/2)-cameraY+self.parallaxImageHeight, self.getLevelWidth(), self.getLevelHeight()))

	def update(self, cameraX, cameraY):
		if self.player.rect.right<0:
			print("THIS SHOULDNT'T HAPPEN! THIS IS LEVEL UPDATING!")
			if self.game.levelCounter < len(self.game.levelList)-1:
				self.game.levelCounter+=1
				self.game.setCurrentLevel(self.game.levelList[self.game.levelCounter])

				self.player.rect.x=SCREENWIDTH-self.player.rect.width
				self.player.rect.y -= 15
				self.game.levelTitle = Word("LEVEL " + str(len(self.game.levelList)-self.game.levelCounter) + ": " + self.game.nameList[self.game.levelCounter], 20, 20, 30, [255, 255, 255])
			else:
				self.game.gameState = "ENDSCREEN"

		# for tile in self.walls:
		# 	tile.update(cameraX, cameraY)
		self.window.blit(self.staticTiles, (-cameraX, -cameraY))
		for tile in self.backgrounds:
			if tile.name=="Wave":
				tile.image = self.game.waveImage[self.game.animation]
			elif tile.name=="Torch":
				tile.image = self.game.torchImage[self.game.animation]

			tile.update(cameraX, cameraY)

<|endoftext|>


<commit_before>import os.path
import socket
import re
import errno
import struct
import traceback
import logging
from math import ceil
from functools import partial
from collections import namedtuple, deque

from zorro import channel, Lock, gethub, Condition, Future
from zorro.util import setcloexec

from .auth import read_auth

log = logging.getLogger(__name__)

class XError(Exception):

    def __init__(self, typ, params):
        self.typ = typ
        self.params = dict(params)

    def __str__(self):
        return '{}{!r}'.format(self.typ.name, self.params)

class Channel(channel.PipelinedReqChannel):
    MAJOR_VERSION = 11
    MINOR_VERSION = 0
    BUFSIZE = 4096

    def __init__(self, *, host=None, port=None,
                 unixsock, event_dispatcher, proto):
        super().__init__()
        self.unixsock = unixsock
        self.request_id = 0
        self.last_seq = 0
        self.epoch = 0
        self.event_dispatcher = event_dispatcher
        self.proto = proto
        self.errors = proto.subprotos['xproto'].errors_by_num.copy()
        if unixsock:
            self._sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
        else:
            self._sock = socket.socket(socket.AF_INET,
                socket.SOCK_STREAM, socket.IPPROTO_TCP)
        setcloexec(self._sock)
        self._sock.setblocking(0)
        try:
            if unixsock:
                self._sock.connect(unixsock)
            else:
                self._sock.connect((host, port))
        except socket.error as e:
            if e.errno == errno.EINPROGRESS:
                gethub().do_write(self._sock)
            else:
                raise
        self._start()

    def connect(self, auth_type, auth_key):
        buf = bytearray()
        buf.extend(struct.pack('<BxHHHH2x',
            0o154, #little endian
            self.MAJOR_VERSION,
            self.MINOR_VERSION,
            len(auth_type),
            len(auth_key)))
        if isinstance(auth_type, str):
            auth_type = auth_type.encode('ascii')
        buf.extend(auth_type)
        buf.extend(b'\x00'*(4 - len(auth_type) % 4))
        buf.extend(auth_key)
        return self.request(buf, None).get()

    def request(self, input, reply):
        if not self._alive:
            raise channel.PipeError()
        val = Future()
        self._pending.append((input, val, reply))
        self._cond.notify()
        return val

    def push(self, input, ignore_error=False):
        """For requests which do not need an answer"""
        if ignore_error:
            tb = None
        else:
            tb = traceback.extract_stack(limit=7)[:-2]
        self._pending.append((input, None, tb))

        self._cond.notify()

    def sender(self):
        buf = bytearray()

        add_chunk = buf.extend
        wait_write = gethub().do_write

        while True:
            if not buf:
                self.wait_requests()
            if not self._alive:
                return
            wait_write(self._sock)
            for inp, fut, tb in self.get_pending_requests():
                self._producing.append((self.request_id, fut, tb))
                self.request_id += 1
                add_chunk(inp)
            try:
                bytes = self._sock.send(buf)
            except socket.error as e:
                if e.errno in (errno.EAGAIN, errno.EINTR):
                    continue
                else:
                    raise
            if not bytes:
                raise EOFError()
            del buf[:bytes]

    def produce(self, seq, value):
        if not self._alive:
            raise ShutdownException()
        if seq < self.last_seq:
            self.epoch += 65536
        self.last_seq = seq
        seq += self.epoch
        assert seq <= self.request_id
        request_id, fut, reply = self._producing.popleft()
        while request_id < seq:
            if fut is not None:
                fut.throw(RuntimeError("Request ignored"))
            assert fut is None
            request_id, fut, reply = self._producing.popleft()
        assert seq == request_id, (seq, request_id)
        if fut is not None:
            if value[0] == 0:
                value = self.parse_error(value)
            else:
                if reply is not None:
                    value = self.parse_reply(reply, value)
            fut.set(value)
        elif reply:  # traceback here, if reply is None error should be ignored
            assert value[0] == 0
            err = self.parse_error(value)
            lst = traceback.format_list(reply)
            lst.extend(traceback.format_exception_only(
                err.__class__, err))
            log.error("Error in asynchronous request\n%s", ''.join(lst))

    def parse_reply(self, reply, buf):
        assert buf[0] == 1
        val, pos = reply.read_from(buf, 1)
        assert max(ceil((pos-2)/4)*4+2, 26) == len(buf), (pos, len(buf))
        return val

    def parse_error(self, buf):
        # TODO(tailhook) parse extension errors
        typ = self.errors[buf[1]]
        err, pos = typ.read_from(buf, 6)
        assert len(buf) == max(pos, 30), (len(buf), pos, buf)
        return XError(typ, err)

    def register_error(self, code, proto):
        for i, v in proto.errors_by_num.items():
            self.errors[code+i] = v

    def _stop_producing(self):
        prod = self._producing
        del self._producing
        for rid, fut, tb in prod:
            if fut is not None:
                fut.throw(channel.PipeError())

    def receiver(self):
        buf = bytearray()

        sock = self._sock
        wait_read = gethub().do_read
        add_chunk = buf.extend
        pos = 0

        while True:
            wait_read(sock)
            try:
                bytes = sock.recv(self.BUFSIZE)
                if not bytes:
                    raise EOFError()
                add_chunk(bytes)
            except socket.error as e:
                if e.errno in (errno.EAGAIN, errno.EINTR):
                    continue
                else:
                    raise
            if len(buf)-pos >= 8:
                res, maj, min, ln = struct.unpack_from('<BxHHH', buf, pos)
                ln = ln*4+8
                if len(buf)-pos < ln:
                    break
                self._producing.popleft()[1].set(buf[pos:pos+ln])
                pos += ln
                break

        while True:
            if pos*2 > len(buf):
                del buf[:pos]
                pos = 0
            wait_read(sock)
            try:
                bytes = sock.recv(self.BUFSIZE)
                if not bytes:
                    raise EOFError()
                add_chunk(bytes)
            except socket.error as e:
                if e.errno in (errno.EAGAIN, errno.EINTR):
                    continue
                else:
                    raise
            while len(buf)-pos >= 8:
                opcode, seq, ln = struct.unpack_from('<BxHL', buf, pos)
                if opcode != 1:
                    if len(buf) - pos < 32:
                        break
                    val = buf[pos:pos+2] + buf[pos+4:pos+32]
                    pos += 32
                    if opcode > 1:
                        self.event_dispatcher(seq, buf[pos-32:pos])
                        continue
                else:
                    ln = ln*4+32
                    if len(buf)-pos < ln:
                        break
                    val = buf[pos:pos+2] + buf[pos+8:pos+ln]
                    pos += ln
                self.produce(seq, val)

class Connection(object):

    def __init__(self, proto, display=None,
        auth_file=None, auth_type=None, auth_key=None):
        self.proto = proto
        if display is None:
            display = os.environ.get('DISPLAY', ':0')
        if auth_file is None and auth_type is None:
            auth_file = os.environ.get('XAUTHORITY')
            if auth_file is None:
                auth_file = os.path.expanduser('~/.XAuthority')
        host, port = display.split(':')
        if '.' in port:
            maj, min = map(int, port.split('.'))
        else:
            maj = int(port)
            min = 0
        assert min == 0, 'Subdisplays are not not supported so far'
        if auth_type is None:
            if host != "":
                haddr = socket.inet_aton(socket.gethostbyname(host))
            for auth in read_auth():
                if host == "":
                    if auth.family == 1 and maj == auth.number:
                        auth_type = auth.name
                        auth_key = auth.data
                        break
                else:
                    if auth.family == 0 and auth.address == haddr:
                        auth_type = auth.name
                        auth_key = auth.data
                        break
            else:
                raise RuntimeError("Can't find X auth type")
        self.unixsock = '/tmp/.X11-unix/X{:d}'.format(maj)
        self.auth_type = auth_type
        self.auth_key = auth_key
        self._channel = None
        self._channel_lock = Lock()
        self._condition = Condition()
        self.events = deque()

    def connection(self):
        if self._channel is None:
            with self._channel_lock:
                if self._channel is None:
                    chan = Channel(unixsock=self.unixsock,
                                   proto=self.proto,
                                   event_dispatcher=self.event_dispatcher)
                    data = chan.connect(self.auth_type, self.auth_key)
                    core = self.proto.subprotos['xproto']
                    value, pos = core.types['Setup'].read_from(data)
                    assert pos == len(data)
                    self.init_data = value
                    assert self.init_data['status'] == 1
                    assert self.init_data['protocol_major_version'] == 11
                    self._init_values()
                    self._channel = chan
                    self._eventreg = core.events_by_num.copy()
        return self._channel

    def _init_values(self):
        d = self.init_data
        base = self.init_data["resource_id_base"]
        mask = self.init_data["resource_id_mask"]
        inc = mask & -mask
        self.xid_generator = iter(range(base, base | mask, inc))

    def query_extension(self, name):
        sub = self.proto.subprotos[name]
        conn = self.connection()
        res = self.do_request(
            self.proto.subprotos['xproto'].requests['QueryExtension'],
            name=sub.xname)
        if res['present']:
            if res['first_event']:
                self.register_event(res['first_event'], sub)
            if res['first_error']:
                conn.register_error(res['first_error'], sub)
        return res

    def do_request(self, rtype, *, _opcode=None, _ignore_error=False, **kw):
        conn = self.connection()
        for i in list(kw):
            n = i + '_len'
            if n in rtype.items and n not in kw:
                kw[n] = len(kw[i])

        buf = bytearray()
        rtype.write_to(buf, kw)
        ln = int(ceil((len(buf)+3)/4))
        if _opcode is None:
            buf.insert(0, rtype.opcode)
        else:
            buf.insert(0, _opcode)
            buf.insert(1, rtype.opcode)
        buf[2:2] = struct.pack('<H', ln)
        buf += b'\x00'*(ln*4 - len(buf))

        if rtype.reply:
            res = conn.request(buf, rtype.reply).get()
            if isinstance(res, XError):
                raise res
            else:
                return res
        else:
            conn.push(buf, ignore_error=_ignore_error)

    def new_xid(self):
        return next(self.xid_generator)

    def register_event(self, code, subpro):
        for ev in subpro.events.values():
            self._eventreg[code + ev.number] = ev

    def event_dispatcher(self, seq, buf):
        etype = self._eventreg[buf[0] & 127]
        if not etype.no_seq:
            seq = None
            buf = buf[:2] + buf[4:]
        ev, pos = etype.read_from(buf, 1)
        assert pos <= 32
        self.events.append(etype.type(seq, **ev))
        self._condition.notify()

    def get_events(self):
        while True:
            while self.events:
                yield self.events.popleft()
            self._condition.wait()

<commit_msg>Fixed bug in protocol handling which leads to WM freeze on start<commit_after>import os.path
import socket
import re
import errno
import struct
import traceback
import logging
from math import ceil
from functools import partial
from collections import namedtuple, deque

from zorro import channel, Lock, gethub, Condition, Future
from zorro.util import setcloexec

from .auth import read_auth

log = logging.getLogger(__name__)

class XError(Exception):

    def __init__(self, typ, params):
        self.typ = typ
        self.params = dict(params)

    def __str__(self):
        return '{}{!r}'.format(self.typ.name, self.params)

class Channel(channel.PipelinedReqChannel):
    MAJOR_VERSION = 11
    MINOR_VERSION = 0
    BUFSIZE = 4096

    def __init__(self, *, host=None, port=None,
                 unixsock, event_dispatcher, proto):
        super().__init__()
        self.unixsock = unixsock
        self.request_id = 0
        self.last_seq = 0
        self.epoch = 0
        self.event_dispatcher = event_dispatcher
        self.proto = proto
        self.errors = proto.subprotos['xproto'].errors_by_num.copy()
        if unixsock:
            self._sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
        else:
            self._sock = socket.socket(socket.AF_INET,
                socket.SOCK_STREAM, socket.IPPROTO_TCP)
        setcloexec(self._sock)
        self._sock.setblocking(0)
        try:
            if unixsock:
                self._sock.connect(unixsock)
            else:
                self._sock.connect((host, port))
        except socket.error as e:
            if e.errno == errno.EINPROGRESS:
                gethub().do_write(self._sock)
            else:
                raise
        self._start()

    def connect(self, auth_type, auth_key):
        buf = bytearray()
        buf.extend(struct.pack('<BxHHHH2x',
            0o154, #little endian
            self.MAJOR_VERSION,
            self.MINOR_VERSION,
            len(auth_type),
            len(auth_key)))
        if isinstance(auth_type, str):
            auth_type = auth_type.encode('ascii')
        buf.extend(auth_type)
        buf.extend(b'\x00'*(4 - len(auth_type) % 4))
        buf.extend(auth_key)
        return self.request(buf, None).get()

    def request(self, input, reply):
        if not self._alive:
            raise channel.PipeError()
        val = Future()
        self._pending.append((input, val, reply))
        self._cond.notify()
        return val

    def push(self, input, ignore_error=False):
        """For requests which do not need an answer"""
        if ignore_error:
            tb = None
        else:
            tb = traceback.extract_stack(limit=7)[:-2]
        self._pending.append((input, None, tb))

        self._cond.notify()

    def sender(self):
        buf = bytearray()

        add_chunk = buf.extend
        wait_write = gethub().do_write

        while True:
            if not buf:
                self.wait_requests()
            if not self._alive:
                return
            wait_write(self._sock)
            for inp, fut, tb in self.get_pending_requests():
                self._producing.append((self.request_id, fut, tb))
                self.request_id += 1
                add_chunk(inp)
            try:
                bytes = self._sock.send(buf)
            except socket.error as e:
                if e.errno in (errno.EAGAIN, errno.EINTR):
                    continue
                else:
                    raise
            if not bytes:
                raise EOFError()
            del buf[:bytes]

    def produce(self, seq, value):
        if not self._alive:
            raise ShutdownException()
        if seq < self.last_seq:
            self.epoch += 65536
        self.last_seq = seq
        seq += self.epoch
        assert seq <= self.request_id
        request_id, fut, reply = self._producing.popleft()
        while request_id < seq:
            if fut is not None:
                fut.throw(RuntimeError("Request ignored"))
            assert fut is None
            request_id, fut, reply = self._producing.popleft()
        assert seq == request_id, (seq, request_id)
        if fut is not None:
            if value[0] == 0:
                value = self.parse_error(value)
            else:
                if reply is not None:
                    value = self.parse_reply(reply, value)
            fut.set(value)
        elif reply:  # traceback here, if reply is None error should be ignored
            assert value[0] == 0
            err = self.parse_error(value)
            lst = traceback.format_list(reply)
            lst.extend(traceback.format_exception_only(
                err.__class__, err))
            log.error("Error in asynchronous request\n%s", ''.join(lst))

    def parse_reply(self, reply, buf):
        assert buf[0] == 1
        val, pos = reply.read_from(buf, 1)
        assert max(ceil((pos-2)/4)*4+2, 26) == len(buf), (pos, len(buf))
        return val

    def parse_error(self, buf):
        # TODO(tailhook) parse extension errors
        typ = self.errors[buf[1]]
        err, pos = typ.read_from(buf, 6)
        assert len(buf) == max(pos, 30), (len(buf), pos, buf)
        return XError(typ, err)

    def register_error(self, code, proto):
        for i, v in proto.errors_by_num.items():
            self.errors[code+i] = v

    def _stop_producing(self):
        prod = self._producing
        del self._producing
        for rid, fut, tb in prod:
            if fut is not None:
                fut.throw(channel.PipeError())

    def receiver(self):
        buf = bytearray()

        sock = self._sock
        wait_read = gethub().do_read
        add_chunk = buf.extend
        pos = 0

        while True:
            wait_read(sock)
            try:
                bytes = sock.recv(self.BUFSIZE)
                if not bytes:
                    raise EOFError()
                add_chunk(bytes)
            except socket.error as e:
                if e.errno in (errno.EAGAIN, errno.EINTR):
                    continue
                else:
                    raise
            if len(buf)-pos >= 8:
                res, maj, min, ln = struct.unpack_from('<BxHHH', buf, pos)
                ln = ln*4+8
                if len(buf)-pos < ln:
                    continue
                self._producing.popleft()[1].set(buf[pos:pos+ln])
                pos += ln
                break

        while True:
            if pos*2 > len(buf):
                del buf[:pos]
                pos = 0
            wait_read(sock)
            try:
                bytes = sock.recv(self.BUFSIZE)
                if not bytes:
                    raise EOFError()
                add_chunk(bytes)
            except socket.error as e:
                if e.errno in (errno.EAGAIN, errno.EINTR):
                    continue
                else:
                    raise
            while len(buf)-pos >= 8:
                opcode, seq, ln = struct.unpack_from('<BxHL', buf, pos)
                if opcode != 1:
                    if len(buf) - pos < 32:
                        break
                    val = buf[pos:pos+2] + buf[pos+4:pos+32]
                    pos += 32
                    if opcode > 1:
                        self.event_dispatcher(seq, buf[pos-32:pos])
                        continue
                else:
                    ln = ln*4+32
                    if len(buf)-pos < ln:
                        break
                    val = buf[pos:pos+2] + buf[pos+8:pos+ln]
                    pos += ln
                self.produce(seq, val)

class Connection(object):

    def __init__(self, proto, display=None,
        auth_file=None, auth_type=None, auth_key=None):
        self.proto = proto
        if display is None:
            display = os.environ.get('DISPLAY', ':0')
        if auth_file is None and auth_type is None:
            auth_file = os.environ.get('XAUTHORITY')
            if auth_file is None:
                auth_file = os.path.expanduser('~/.XAuthority')
        host, port = display.split(':')
        if '.' in port:
            maj, min = map(int, port.split('.'))
        else:
            maj = int(port)
            min = 0
        assert min == 0, 'Subdisplays are not not supported so far'
        if auth_type is None:
            if host != "":
                haddr = socket.inet_aton(socket.gethostbyname(host))
            for auth in read_auth():
                if host == "":
                    if auth.family == 1 and maj == auth.number:
                        auth_type = auth.name
                        auth_key = auth.data
                        break
                else:
                    if auth.family == 0 and auth.address == haddr:
                        auth_type = auth.name
                        auth_key = auth.data
                        break
            else:
                raise RuntimeError("Can't find X auth type")
        self.unixsock = '/tmp/.X11-unix/X{:d}'.format(maj)
        self.auth_type = auth_type
        self.auth_key = auth_key
        self._channel = None
        self._channel_lock = Lock()
        self._condition = Condition()
        self.events = deque()

    def connection(self):
        if self._channel is None:
            with self._channel_lock:
                if self._channel is None:
                    chan = Channel(unixsock=self.unixsock,
                                   proto=self.proto,
                                   event_dispatcher=self.event_dispatcher)
                    data = chan.connect(self.auth_type, self.auth_key)
                    core = self.proto.subprotos['xproto']
                    value, pos = core.types['Setup'].read_from(data)
                    assert pos == len(data)
                    self.init_data = value
                    assert self.init_data['status'] == 1
                    assert self.init_data['protocol_major_version'] == 11
                    self._init_values()
                    self._channel = chan
                    self._eventreg = core.events_by_num.copy()
        return self._channel

    def _init_values(self):
        d = self.init_data
        base = self.init_data["resource_id_base"]
        mask = self.init_data["resource_id_mask"]
        inc = mask & -mask
        self.xid_generator = iter(range(base, base | mask, inc))

    def query_extension(self, name):
        sub = self.proto.subprotos[name]
        conn = self.connection()
        res = self.do_request(
            self.proto.subprotos['xproto'].requests['QueryExtension'],
            name=sub.xname)
        if res['present']:
            if res['first_event']:
                self.register_event(res['first_event'], sub)
            if res['first_error']:
                conn.register_error(res['first_error'], sub)
        return res

    def do_request(self, rtype, *, _opcode=None, _ignore_error=False, **kw):
        conn = self.connection()
        for i in list(kw):
            n = i + '_len'
            if n in rtype.items and n not in kw:
                kw[n] = len(kw[i])

        buf = bytearray()
        rtype.write_to(buf, kw)
        ln = int(ceil((len(buf)+3)/4))
        if _opcode is None:
            buf.insert(0, rtype.opcode)
        else:
            buf.insert(0, _opcode)
            buf.insert(1, rtype.opcode)
        buf[2:2] = struct.pack('<H', ln)
        buf += b'\x00'*(ln*4 - len(buf))

        if rtype.reply:
            res = conn.request(buf, rtype.reply).get()
            if isinstance(res, XError):
                raise res
            else:
                return res
        else:
            conn.push(buf, ignore_error=_ignore_error)

    def new_xid(self):
        return next(self.xid_generator)

    def register_event(self, code, subpro):
        for ev in subpro.events.values():
            self._eventreg[code + ev.number] = ev

    def event_dispatcher(self, seq, buf):
        etype = self._eventreg[buf[0] & 127]
        if not etype.no_seq:
            seq = None
            buf = buf[:2] + buf[4:]
        ev, pos = etype.read_from(buf, 1)
        assert pos <= 32
        self.events.append(etype.type(seq, **ev))
        self._condition.notify()

    def get_events(self):
        while True:
            while self.events:
                yield self.events.popleft()
            self._condition.wait()

<|endoftext|>


<commit_before>from collections import namedtuple
from cStringIO import StringIO
from shapely.geometry import MultiPolygon
from shapely import geometry
from shapely.wkb import loads
from tilequeue.config import create_query_bounds_pad_fn
from tilequeue.tile import calc_meters_per_pixel_dim
from tilequeue.tile import coord_to_mercator_bounds
from tilequeue.tile import normalize_geometry_type
from tilequeue.transform import mercator_point_to_lnglat
from tilequeue.transform import transform_feature_layers_shape
from zope.dottedname.resolve import resolve
from sys import getsizeof

def make_transform_fn(transform_fns):
    if not transform_fns:
        return None

    def transform_fn(shape, properties, fid, zoom):
        for fn in transform_fns:
            shape, properties, fid = fn(shape, properties, fid, zoom)
        return shape, properties, fid
    return transform_fn

def resolve_transform_fns(fn_dotted_names):
    if not fn_dotted_names:
        return None
    return map(resolve, fn_dotted_names)

def _sizeof(val):
    size = 0

    if isinstance(val, dict):
        for k, v in val.items():
            size += len(k) + _sizeof(v)
    elif isinstance(val, list):
        for v in val:
            size += _sizeof(v)
    elif isinstance(val, (str, bytes, unicode)):
        size += len(val)
    else:
        size += getsizeof(val)

    return size

def _preprocess_data(feature_layers):
    preproc_feature_layers = []
    extra_data = dict(size={})

    for feature_layer in feature_layers:
        layer_datum = feature_layer['layer_datum']
        geometry_types = layer_datum['geometry_types']
        padded_bounds = feature_layer['padded_bounds']

        features = []
        features_size = 0
        for row in feature_layer['features']:
            wkb = row.pop('__geometry__')
            shape = loads(wkb)

            if shape.is_empty:
                continue

            if not shape.is_valid:
                continue

            if geometry_types is not None:
                if shape.type not in geometry_types:
                    continue

            # since a bounding box intersection is used, we
            # perform a more accurate check here to filter out
            # any extra features
            # the formatter specific transformations will take
            # care of any additional filtering
            geom_type_bounds = padded_bounds[
                normalize_geometry_type(shape.type)]
            shape_padded_bounds = geometry.box(*geom_type_bounds)
            if not shape_padded_bounds.intersects(shape):
                continue

            feature_id = row.pop('__id__')
            props = dict()
            feature_size = getsizeof(feature_id) + len(wkb)
            for k, v in row.iteritems():
                if k == 'mz_properties':
                    for output_key, output_val in v.items():
                        if output_val is not None:
                            # all other tags are utf8 encoded, encode
                            # these the same way to be consistent
                            if isinstance(output_key, unicode):
                                output_key = output_key.encode('utf-8')
                            if isinstance(output_val, unicode):
                                output_val = output_val.encode('utf-8')
                            props[output_key] = output_val
                            feature_size += len(output_key) + \
                                _sizeof(output_val)
                else:
                    props[k] = v
                    feature_size += len(k) + _sizeof(v)

            feature = shape, props, feature_id
            features.append(feature)
            features_size += feature_size

        extra_data['size'][layer_datum['name']] = features_size

        preproc_feature_layer = dict(
            name=layer_datum['name'],
            layer_datum=layer_datum,
            features=features,
            padded_bounds=padded_bounds,
        )
        preproc_feature_layers.append(preproc_feature_layer)

    return preproc_feature_layers, extra_data

# shared context for all the post-processor functions. this single object can
# be passed around rather than needing all the parameters to be explicit.
Context = namedtuple('Context', [
    'feature_layers',    # the feature layers list
    'nominal_zoom',      # the zoom level to use for styling (display scale)
    'unpadded_bounds',   # the latlon bounds of the tile
    'params',            # user configuration parameters
    'resources',         # resources declared in config
])

# post-process all the layers simultaneously, which allows new
# layers to be created from processing existing ones (e.g: for
# computed centroids) or modifying layers based on the contents
# of other layers (e.g: projecting attributes, deleting hidden
# features, etc...)
def _postprocess_data(
        feature_layers, post_process_data, nominal_zoom, unpadded_bounds):

    for step in post_process_data:
        fn = resolve(step['fn_name'])

        ctx = Context(
            feature_layers=feature_layers,
            nominal_zoom=nominal_zoom,
            unpadded_bounds=unpadded_bounds,
            params=step['params'],
            resources=step['resources'],
        )

        layer = fn(ctx)
        feature_layers = ctx.feature_layers
        if layer is not None:
            for index, feature_layer in enumerate(feature_layers):
                layer_datum = feature_layer['layer_datum']
                layer_name = layer_datum['name']
                if layer_name == layer['layer_datum']['name']:
                    feature_layers[index] = layer
                    layer = None
                    break
            # if this layer isn't replacing an old layer, then
            # append it.
            if layer is not None:
                feature_layers.append(layer)

    return feature_layers

def _cut_coord(
        feature_layers, unpadded_bounds, meters_per_pixel_dim, buffer_cfg):
    cut_feature_layers = []
    for feature_layer in feature_layers:
        features = feature_layer['features']
        padded_bounds_fn = create_query_bounds_pad_fn(
            buffer_cfg, feature_layer['name'])
        padded_bounds = padded_bounds_fn(unpadded_bounds, meters_per_pixel_dim)

        cut_features = []
        for feature in features:
            shape, props, feature_id = feature

            geom_type_bounds = padded_bounds[
                normalize_geometry_type(shape.type)]
            shape_padded_bounds = geometry.box(*geom_type_bounds)
            if not shape_padded_bounds.intersects(shape):
                continue
            props_copy = props.copy()
            cut_feature = shape, props_copy, feature_id

            cut_features.append(cut_feature)

        cut_feature_layer = dict(
            name=feature_layer['name'],
            layer_datum=feature_layer['layer_datum'],
            features=cut_features,
            padded_bounds=padded_bounds,
        )
        cut_feature_layers.append(cut_feature_layer)

    return cut_feature_layers

def _make_valid_if_necessary(shape):
    """
    attempt to correct invalid shapes if necessary

    After simplification, even when preserving topology, invalid
    shapes can be returned. This appears to only occur with polygon
    types. As an optimization, we only check if the polygon types are
    valid.
    """
    if shape.type in ('Polygon', 'MultiPolygon') and not shape.is_valid:
        shape = shape.buffer(0)

        # return value from buffer is usually valid, but it's
        # not clear from the docs whether this is guaranteed,
        # so return None if not.
        if not shape.is_valid:
            return None

    return shape

# returns none if the shape is not visible given the number of meters
# per pixel. for multipolygons, will filter out any subpolygons that
# should not be visible, which means that the shape could have been
# altered
def _visible_shape(shape, area_threshold_meters):
    if shape is None:
        return None
    elif shape.type == 'MultiPolygon':
        visible_shapes = []
        for subshape in shape.geoms:
            subshape = _visible_shape(subshape, area_threshold_meters)
            if subshape:
                visible_shapes.append(subshape)
        if visible_shapes:
            return MultiPolygon(visible_shapes)
        else:
            return None
    elif shape.type == 'Polygon':
        shape_meters = shape.area
        return shape if shape_meters >= area_threshold_meters else None
    else:
        return shape

def _create_formatted_tile(
        feature_layers, format, scale, unpadded_bounds, unpadded_bounds_lnglat,
        coord, layer, meters_per_pixel_dim, buffer_cfg):

    # perform format specific transformations
    transformed_feature_layers = transform_feature_layers_shape(
        feature_layers, format, scale, unpadded_bounds,
        meters_per_pixel_dim, buffer_cfg)

    # use the formatter to generate the tile
    tile_data_file = StringIO()
    format.format_tile(tile_data_file, transformed_feature_layers, coord.zoom,
                       unpadded_bounds, unpadded_bounds_lnglat)
    tile = tile_data_file.getvalue()

    formatted_tile = dict(format=format, tile=tile, coord=coord, layer=layer)
    return formatted_tile

def _process_feature_layers(
        feature_layers, coord, post_process_data, unpadded_bounds):

    # the nominal zoom is the "display scale" zoom, which may not correspond
    # to actual tile coordinates in future versions of the code. it just
    # becomes a measure of the scale between tile features and intended
    # display size.
    nominal_zoom = coord.zoom

    processed_feature_layers = []
    # filter, and then transform each layer as necessary
    for feature_layer in feature_layers:
        layer_datum = feature_layer['layer_datum']
        layer_name = layer_datum['name']
        features = feature_layer['features']

        transform_fn_names = layer_datum['transform_fn_names']
        if transform_fn_names:
            transform_fns = resolve_transform_fns(transform_fn_names)
            layer_transform_fn = make_transform_fn(transform_fns)
        else:
            layer_transform_fn = None

        # perform any specific layer transformations
        if layer_transform_fn is None:
            processed_features = features
        else:
            processed_features = []
            for feature in features:
                shape, props, feature_id = feature
                shape, props, feature_id = layer_transform_fn(
                    shape, props, feature_id, nominal_zoom)
                transformed_feature = shape, props, feature_id
                processed_features.append(transformed_feature)

        sort_fn_name = layer_datum['sort_fn_name']
        if sort_fn_name:
            sort_fn = resolve(sort_fn_name)
            processed_features = sort_fn(processed_features, nominal_zoom)

        feature_layer = dict(
            name=layer_name,
            features=processed_features,
            layer_datum=layer_datum,
            padded_bounds=feature_layer['padded_bounds'],
        )
        processed_feature_layers.append(feature_layer)

    # post-process data here, before it gets formatted
    processed_feature_layers = _postprocess_data(
        processed_feature_layers, post_process_data, nominal_zoom,
        unpadded_bounds)

    return processed_feature_layers

def _format_feature_layers(
        processed_feature_layers, coord, formats, unpadded_bounds, scale,
        buffer_cfg):

    # the nominal zoom is the "display scale" zoom, which may not correspond
    # to actual tile coordinates in future versions of the code. it just
    # becomes a measure of the scale between tile features and intended
    # display size.
    nominal_zoom = coord.zoom

    meters_per_pixel_dim = calc_meters_per_pixel_dim(nominal_zoom)

    # topojson formatter expects bounds to be in lnglat
    unpadded_bounds_lnglat = (
        mercator_point_to_lnglat(unpadded_bounds[0], unpadded_bounds[1]) +
        mercator_point_to_lnglat(unpadded_bounds[2], unpadded_bounds[3]))

    # now, perform the format specific transformations
    # and format the tile itself
    formatted_tiles = []
    layer = 'all'
    for format in formats:
        formatted_tile = _create_formatted_tile(
            processed_feature_layers, format, scale, unpadded_bounds,
            unpadded_bounds_lnglat, coord, layer, meters_per_pixel_dim,
            buffer_cfg)
        formatted_tiles.append(formatted_tile)

    return formatted_tiles

def _cut_child_tiles(
        feature_layers, cut_coord, formats, scale, buffer_cfg):

    unpadded_cut_bounds = coord_to_mercator_bounds(cut_coord)
    meters_per_pixel_dim = calc_meters_per_pixel_dim(cut_coord.zoom)

    cut_feature_layers = _cut_coord(
        feature_layers, unpadded_cut_bounds, meters_per_pixel_dim, buffer_cfg)

    return _format_feature_layers(
        cut_feature_layers, cut_coord, formats, unpadded_cut_bounds, scale,
        buffer_cfg)

# given a coord and the raw feature layers results from the database,
# filter, transform, sort, post-process and then format according to
# each formatter. this is the entry point from the worker process
def process_coord(coord, feature_layers, post_process_data, formats,
                  unpadded_bounds, cut_coords, buffer_cfg, scale=4096):
    feature_layers, extra_data = _preprocess_data(feature_layers)

    processed_feature_layers = _process_feature_layers(
        feature_layers, coord, post_process_data, unpadded_bounds)

    coord_formatted_tiles = _format_feature_layers(
        processed_feature_layers, coord, formats, unpadded_bounds, scale,
        buffer_cfg)

    children_formatted_tiles = []
    if cut_coords:
        for cut_coord in cut_coords:
            child_tiles = _cut_child_tiles(
                processed_feature_layers, cut_coord, formats, scale,
                buffer_cfg)
            children_formatted_tiles.extend(child_tiles)

    all_formatted_tiles = coord_formatted_tiles + children_formatted_tiles
    return all_formatted_tiles, extra_data
<commit_msg>Pass nominal zoom instead of coordinate to processing function.<commit_after>from collections import namedtuple
from cStringIO import StringIO
from shapely.geometry import MultiPolygon
from shapely import geometry
from shapely.wkb import loads
from tilequeue.config import create_query_bounds_pad_fn
from tilequeue.tile import calc_meters_per_pixel_dim
from tilequeue.tile import coord_to_mercator_bounds
from tilequeue.tile import normalize_geometry_type
from tilequeue.transform import mercator_point_to_lnglat
from tilequeue.transform import transform_feature_layers_shape
from zope.dottedname.resolve import resolve
from sys import getsizeof

def make_transform_fn(transform_fns):
    if not transform_fns:
        return None

    def transform_fn(shape, properties, fid, zoom):
        for fn in transform_fns:
            shape, properties, fid = fn(shape, properties, fid, zoom)
        return shape, properties, fid
    return transform_fn

def resolve_transform_fns(fn_dotted_names):
    if not fn_dotted_names:
        return None
    return map(resolve, fn_dotted_names)

def _sizeof(val):
    size = 0

    if isinstance(val, dict):
        for k, v in val.items():
            size += len(k) + _sizeof(v)
    elif isinstance(val, list):
        for v in val:
            size += _sizeof(v)
    elif isinstance(val, (str, bytes, unicode)):
        size += len(val)
    else:
        size += getsizeof(val)

    return size

def _preprocess_data(feature_layers):
    preproc_feature_layers = []
    extra_data = dict(size={})

    for feature_layer in feature_layers:
        layer_datum = feature_layer['layer_datum']
        geometry_types = layer_datum['geometry_types']
        padded_bounds = feature_layer['padded_bounds']

        features = []
        features_size = 0
        for row in feature_layer['features']:
            wkb = row.pop('__geometry__')
            shape = loads(wkb)

            if shape.is_empty:
                continue

            if not shape.is_valid:
                continue

            if geometry_types is not None:
                if shape.type not in geometry_types:
                    continue

            # since a bounding box intersection is used, we
            # perform a more accurate check here to filter out
            # any extra features
            # the formatter specific transformations will take
            # care of any additional filtering
            geom_type_bounds = padded_bounds[
                normalize_geometry_type(shape.type)]
            shape_padded_bounds = geometry.box(*geom_type_bounds)
            if not shape_padded_bounds.intersects(shape):
                continue

            feature_id = row.pop('__id__')
            props = dict()
            feature_size = getsizeof(feature_id) + len(wkb)
            for k, v in row.iteritems():
                if k == 'mz_properties':
                    for output_key, output_val in v.items():
                        if output_val is not None:
                            # all other tags are utf8 encoded, encode
                            # these the same way to be consistent
                            if isinstance(output_key, unicode):
                                output_key = output_key.encode('utf-8')
                            if isinstance(output_val, unicode):
                                output_val = output_val.encode('utf-8')
                            props[output_key] = output_val
                            feature_size += len(output_key) + \
                                _sizeof(output_val)
                else:
                    props[k] = v
                    feature_size += len(k) + _sizeof(v)

            feature = shape, props, feature_id
            features.append(feature)
            features_size += feature_size

        extra_data['size'][layer_datum['name']] = features_size

        preproc_feature_layer = dict(
            name=layer_datum['name'],
            layer_datum=layer_datum,
            features=features,
            padded_bounds=padded_bounds,
        )
        preproc_feature_layers.append(preproc_feature_layer)

    return preproc_feature_layers, extra_data

# shared context for all the post-processor functions. this single object can
# be passed around rather than needing all the parameters to be explicit.
Context = namedtuple('Context', [
    'feature_layers',    # the feature layers list
    'nominal_zoom',      # the zoom level to use for styling (display scale)
    'unpadded_bounds',   # the latlon bounds of the tile
    'params',            # user configuration parameters
    'resources',         # resources declared in config
])

# post-process all the layers simultaneously, which allows new
# layers to be created from processing existing ones (e.g: for
# computed centroids) or modifying layers based on the contents
# of other layers (e.g: projecting attributes, deleting hidden
# features, etc...)
def _postprocess_data(
        feature_layers, post_process_data, nominal_zoom, unpadded_bounds):

    for step in post_process_data:
        fn = resolve(step['fn_name'])

        ctx = Context(
            feature_layers=feature_layers,
            nominal_zoom=nominal_zoom,
            unpadded_bounds=unpadded_bounds,
            params=step['params'],
            resources=step['resources'],
        )

        layer = fn(ctx)
        feature_layers = ctx.feature_layers
        if layer is not None:
            for index, feature_layer in enumerate(feature_layers):
                layer_datum = feature_layer['layer_datum']
                layer_name = layer_datum['name']
                if layer_name == layer['layer_datum']['name']:
                    feature_layers[index] = layer
                    layer = None
                    break
            # if this layer isn't replacing an old layer, then
            # append it.
            if layer is not None:
                feature_layers.append(layer)

    return feature_layers

def _cut_coord(
        feature_layers, unpadded_bounds, meters_per_pixel_dim, buffer_cfg):
    cut_feature_layers = []
    for feature_layer in feature_layers:
        features = feature_layer['features']
        padded_bounds_fn = create_query_bounds_pad_fn(
            buffer_cfg, feature_layer['name'])
        padded_bounds = padded_bounds_fn(unpadded_bounds, meters_per_pixel_dim)

        cut_features = []
        for feature in features:
            shape, props, feature_id = feature

            geom_type_bounds = padded_bounds[
                normalize_geometry_type(shape.type)]
            shape_padded_bounds = geometry.box(*geom_type_bounds)
            if not shape_padded_bounds.intersects(shape):
                continue
            props_copy = props.copy()
            cut_feature = shape, props_copy, feature_id

            cut_features.append(cut_feature)

        cut_feature_layer = dict(
            name=feature_layer['name'],
            layer_datum=feature_layer['layer_datum'],
            features=cut_features,
            padded_bounds=padded_bounds,
        )
        cut_feature_layers.append(cut_feature_layer)

    return cut_feature_layers

def _make_valid_if_necessary(shape):
    """
    attempt to correct invalid shapes if necessary

    After simplification, even when preserving topology, invalid
    shapes can be returned. This appears to only occur with polygon
    types. As an optimization, we only check if the polygon types are
    valid.
    """
    if shape.type in ('Polygon', 'MultiPolygon') and not shape.is_valid:
        shape = shape.buffer(0)

        # return value from buffer is usually valid, but it's
        # not clear from the docs whether this is guaranteed,
        # so return None if not.
        if not shape.is_valid:
            return None

    return shape

# returns none if the shape is not visible given the number of meters
# per pixel. for multipolygons, will filter out any subpolygons that
# should not be visible, which means that the shape could have been
# altered
def _visible_shape(shape, area_threshold_meters):
    if shape is None:
        return None
    elif shape.type == 'MultiPolygon':
        visible_shapes = []
        for subshape in shape.geoms:
            subshape = _visible_shape(subshape, area_threshold_meters)
            if subshape:
                visible_shapes.append(subshape)
        if visible_shapes:
            return MultiPolygon(visible_shapes)
        else:
            return None
    elif shape.type == 'Polygon':
        shape_meters = shape.area
        return shape if shape_meters >= area_threshold_meters else None
    else:
        return shape

def _create_formatted_tile(
        feature_layers, format, scale, unpadded_bounds, unpadded_bounds_lnglat,
        coord, layer, meters_per_pixel_dim, buffer_cfg):

    # perform format specific transformations
    transformed_feature_layers = transform_feature_layers_shape(
        feature_layers, format, scale, unpadded_bounds,
        meters_per_pixel_dim, buffer_cfg)

    # use the formatter to generate the tile
    tile_data_file = StringIO()
    format.format_tile(tile_data_file, transformed_feature_layers, coord.zoom,
                       unpadded_bounds, unpadded_bounds_lnglat)
    tile = tile_data_file.getvalue()

    formatted_tile = dict(format=format, tile=tile, coord=coord, layer=layer)
    return formatted_tile

def _process_feature_layers(
        feature_layers, nominal_zoom, post_process_data, unpadded_bounds):

    processed_feature_layers = []
    # filter, and then transform each layer as necessary
    for feature_layer in feature_layers:
        layer_datum = feature_layer['layer_datum']
        layer_name = layer_datum['name']
        features = feature_layer['features']

        transform_fn_names = layer_datum['transform_fn_names']
        if transform_fn_names:
            transform_fns = resolve_transform_fns(transform_fn_names)
            layer_transform_fn = make_transform_fn(transform_fns)
        else:
            layer_transform_fn = None

        # perform any specific layer transformations
        if layer_transform_fn is None:
            processed_features = features
        else:
            processed_features = []
            for feature in features:
                shape, props, feature_id = feature
                shape, props, feature_id = layer_transform_fn(
                    shape, props, feature_id, nominal_zoom)
                transformed_feature = shape, props, feature_id
                processed_features.append(transformed_feature)

        sort_fn_name = layer_datum['sort_fn_name']
        if sort_fn_name:
            sort_fn = resolve(sort_fn_name)
            processed_features = sort_fn(processed_features, nominal_zoom)

        feature_layer = dict(
            name=layer_name,
            features=processed_features,
            layer_datum=layer_datum,
            padded_bounds=feature_layer['padded_bounds'],
        )
        processed_feature_layers.append(feature_layer)

    # post-process data here, before it gets formatted
    processed_feature_layers = _postprocess_data(
        processed_feature_layers, post_process_data, nominal_zoom,
        unpadded_bounds)

    return processed_feature_layers

def _format_feature_layers(
        processed_feature_layers, coord, formats, unpadded_bounds, scale,
        buffer_cfg):

    # the nominal zoom is the "display scale" zoom, which may not correspond
    # to actual tile coordinates in future versions of the code. it just
    # becomes a measure of the scale between tile features and intended
    # display size.
    nominal_zoom = coord.zoom

    meters_per_pixel_dim = calc_meters_per_pixel_dim(nominal_zoom)

    # topojson formatter expects bounds to be in lnglat
    unpadded_bounds_lnglat = (
        mercator_point_to_lnglat(unpadded_bounds[0], unpadded_bounds[1]) +
        mercator_point_to_lnglat(unpadded_bounds[2], unpadded_bounds[3]))

    # now, perform the format specific transformations
    # and format the tile itself
    formatted_tiles = []
    layer = 'all'
    for format in formats:
        formatted_tile = _create_formatted_tile(
            processed_feature_layers, format, scale, unpadded_bounds,
            unpadded_bounds_lnglat, coord, layer, meters_per_pixel_dim,
            buffer_cfg)
        formatted_tiles.append(formatted_tile)

    return formatted_tiles

def _cut_child_tiles(
        feature_layers, cut_coord, formats, scale, buffer_cfg):

    unpadded_cut_bounds = coord_to_mercator_bounds(cut_coord)
    meters_per_pixel_dim = calc_meters_per_pixel_dim(cut_coord.zoom)

    cut_feature_layers = _cut_coord(
        feature_layers, unpadded_cut_bounds, meters_per_pixel_dim, buffer_cfg)

    return _format_feature_layers(
        cut_feature_layers, cut_coord, formats, unpadded_cut_bounds, scale,
        buffer_cfg)

# given a coord and the raw feature layers results from the database,
# filter, transform, sort, post-process and then format according to
# each formatter. this is the entry point from the worker process
def process_coord(coord, feature_layers, post_process_data, formats,
                  unpadded_bounds, cut_coords, buffer_cfg, scale=4096):
    # the nominal zoom is the "display scale" zoom, which may not correspond
    # to actual tile coordinates in future versions of the code. it just
    # becomes a measure of the scale between tile features and intended
    # display size.
    nominal_zoom = coord.zoom

    feature_layers, extra_data = _preprocess_data(feature_layers)

    processed_feature_layers = _process_feature_layers(
        feature_layers, nominal_zoom, post_process_data, unpadded_bounds)

    coord_formatted_tiles = _format_feature_layers(
        processed_feature_layers, coord, formats, unpadded_bounds, scale,
        buffer_cfg)

    children_formatted_tiles = []
    if cut_coords:
        for cut_coord in cut_coords:
            child_tiles = _cut_child_tiles(
                processed_feature_layers, cut_coord, formats, scale,
                buffer_cfg)
            children_formatted_tiles.extend(child_tiles)

    all_formatted_tiles = coord_formatted_tiles + children_formatted_tiles
    return all_formatted_tiles, extra_data
<|endoftext|>


<commit_before>global inputfolder
inputfolder =  '../data'

global sparqloutput
sparqloutput = '../output/sparqls.txt'

global malformedsparql
malformedsparql = '../output/malformed-sparql.txt'

global extractionLog
extractionLog = '../output/extraction.log'

global analysisLog
analysisLog = '../output/analysis.log'

global bgpfile
bgpfile = '../output/bgp.txt'

global bgpanalysis
bgpanalysis = '../output/bgp-analysis.csv'

global bgpanalysisLog
bgpanalysisLog = '../output/bgp-analysis.log'<commit_msg>included super folder variable for all files<commit_after>superdir = ""

global inputfolder
inputfolder =  superdir +'/data'

global sparqloutput
sparqloutput = superdir +'/output/sparqls.txt'

global malformedsparql
malformedsparql = superdir +'/output/malformed-sparql.txt'

global extractionLog
extractionLog = superdir +'/output/extraction.log'

global analysisLog
analysisLog = superdir +'/output/analysis.log'

global bgpfile
bgpfile = superdir +'/output/bgp.txt'

global bgpanalysis
bgpanalysis = superdir +'/output/bgp-analysis.csv'

global bgpanalysisLog
bgpanalysisLog = superdir +'/output/bgp-analysis.log'
<|endoftext|>


<commit_before>from . import layers
from . import placeholders
<commit_msg>Updating init for timeflow qith features and trainer<commit_after>from . import layers
from . import placeholders
from . import trainer
from . import features
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from core import *
from metadata import Metadata
<commit_msg>revert module loading<commit_after># -*- coding: utf-8 -*-

from core import *
from metadata import Metadata
import decoder
import encoder
import grapher
import analyzer
<|endoftext|>


<commit_before>import datetime

from django.http import HttpResponse, Http404
from django.shortcuts import render_to_response
from django.template import RequestContext

from tracker.models import TrackingEntry, Tbluser
from utils.calendar_utils import gen_calendar
from tracker.forms import entry_form, add_form

def index(request):
    return render_to_response('base.html',
                              {},
                              RequestContext(request))

def view_calendar(request,
             year=datetime.date.today().year,
             month=datetime.date.today().month,
             day=datetime.date.today().day,
             ):

    """
    Generates a calendar based on the URL it receives.
    site.com/calendar/2012/02/, also takes a day
    just in case you want to add a particular view for a day, for example.

    Even the generated HTML is aligned and pretty printed.
    """

    calendar_table = gen_calendar(year, month, day,
                                  user='aaron.france@hp.com')

    return render_to_response(
        'calendar.html',
        {
         'calendar': calendar_table,
         'changeform': entry_form(),
         'addform' : add_form()
        },
        RequestContext(request)
        )

def process_change_request(request):

    """
    Processes a change into the database from the calendar page
    """
    pass

<commit_msg>Changed docstring to better match what I meant - views.view_calendar<commit_after>import datetime

from django.http import HttpResponse, Http404
from django.shortcuts import render_to_response
from django.template import RequestContext

from tracker.models import TrackingEntry, Tbluser
from utils.calendar_utils import gen_calendar
from tracker.forms import entry_form, add_form

def index(request):
    return render_to_response('base.html',
                              {},
                              RequestContext(request))

def view_calendar(request,
             year=datetime.date.today().year,
             month=datetime.date.today().month,
             day=datetime.date.today().day,
             ):

    """
    Generates a calendar based on the URL it receives.
    site.com/calendar/2012/02/, also takes a day
    just in case you want to add a particular view for a day, for example.

    The generated HTML is pretty printed
    """

    calendar_table = gen_calendar(year, month, day,
                                  user='aaron.france@hp.com')

    return render_to_response(
        'calendar.html',
        {
         'calendar': calendar_table,
         'changeform': entry_form(),
         'addform' : add_form()
        },
        RequestContext(request)
        )

def process_change_request(request):

    """
    Processes a change into the database from the calendar page
    """
    pass

<|endoftext|>


<commit_before>"""
Contains the KitList class
"""

from collections import namedtuple
import sys
import yaml
try:
    from yaml import CLoader as Loader, CDumper as Dumper
except ImportError:
    from yaml import Loader, Dumper
import readers
import testers

Reader = namedtuple('Reader', ['name', 'func', 'args'])
Tester = namedtuple('Tester', ['name', 'func', 'args'])

class AuditJob(object):  # pylint: disable=R0903
    """
    The audit of a single tool
    """

    def __init__(self, name, path, reader, tester=None, version=None,
                 checksum=None, output_checksum=None):
        self.name = name
        self.path = path
        self.reader = reader
        self.tester = tester
        self.version = version
        self.checksum = checksum
        self.output_checksum = output_checksum

    def __repr__(self):
        r = "{0}({1!r}, {2!r}, {3!r}, {4!r}, {5!r}, {6!r}, {7!r}, {8!r})"
        return r.format(
            'AuditJob',
            self.name,
            self.path,
            self.reader,
            self.tester,
            self.version,
            self.checksum,
            self.output_checksum
        )

    def as_dict(self):
        """
        Convert to a dict

        Returns
        -------
        dict : AuditJob
            This AuditJob as a dictionary
        """
        reader_dict = {'name': self.reader.name}
        for k, v in self.reader.args.iteritems():
            reader_dict[k] = v
        if self.tester:
            tester_dict = {'name': self.tester.name}
            for k, v in self.tester.args.iteritems():
                tester_dict[k] = v
        else:
            tester_dict = None
        return {
            'name': self.name,
            'path': self.path,
            'reader': reader_dict,
            'tester': tester_dict,
            'version': self.version,
            'checksum': self.checksum,
            'output_checksum': self.output_checksum
        }

class KitList(object):
    """
    Represents a list of applications used in a pipeline and the procedure
    used to check each applications version
    """

    reader_functions = {
        'command_line': readers.command_line,
        'line_in_file': readers.line_in_file,
        'manual': readers.manual
    }

    tester_functions = {
        'stdout': testers.stdout,
        'file': testers.fileout
    }

    def __init__(self):
        self.tools = []

    @classmethod
    def from_file(cls, path):
        """
        Read the KitList description at *path* using a new instance of
        :class:`KitList` and return the resulting instance.

        Parameters
        ----------
        path : str
            The path to read from

        Returns
        -------
        :class:`KitList` : kitlist
            An instance with data loaded
        """

        instance = cls()
        instance.read(path)
        return instance

    def read(self, path):
        """
        Read a saved KitList in YAML format

        Parameters
        ----------
        path : str
            Path to a saved KitList
        """

        with open(path, 'r') as f:
            yaml_data = yaml.load(f, Loader=Loader)
        if 'tools' not in yaml_data:
            raise(KeyError(
                'The kitlist provided does not contain a tools element'
            ))
        tools = []
        for tool in yaml_data['tools']:
            tool_name = tool['name']
            tool_path = tool['path']
            reader = self._parse_reader_element(tool['reader'])
            if 'test' in tool:
                tester = self._parse_tester_element(tool['test'])
            else:
                tester = None
            audit_job = AuditJob(
                tool_name, tool_path,
                reader,
                tester,
                tool.get('version', None),
                tool.get('checksum', None),
                tool.get('output_checksum', None)
            )
            tools.append(audit_job)
        self.tools = tools

    @classmethod
    def _parse_tester_element(cls, element):
        """
        Take a tester element read from a KitList file and convert it to a
        Tester namedtuple
        """
        tester_name = element['name']
        if tester_name not in cls.tester_functions:
            raise(KeyError(
                'Unknown tester function {}'.format(tester_name)
            ))
        args = [
            (k, v) for k, v in element.iteritems() if k != 'name'
        ]
        tester_func = cls.tester_functions[tester_name]
        tester_args = dict(args)
        tester = Tester(tester_name, tester_func, tester_args)
        return tester

    @classmethod
    def _parse_reader_element(cls, element):
        """
        Take a reader element read from a KitList file and convert it to a
        Reader namedtuple
        """
        reader_name = element['name']
        if reader_name not in cls.reader_functions:
            raise(KeyError(
                'Unknown reader function {}'.format(reader_name)
            ))
        args = [
            (k, v) for k, v in element.iteritems() if k != 'name'
        ]
        reader_func = cls.reader_functions[reader_name]
        reader_args = dict(args)
        if 'regex' in reader_args:
            reader_args['regex'] = cls._fixup_regex(reader_args['regex'])
        reader = Reader(reader_name, reader_func, reader_args)
        return reader

    def get_tool(self, tool_name):
        """
        Get a single tool identifed by name

        Parameters
        ----------
        tool_name : str
            The name of the tool to retrieve

        Returns
        -------
        tool : class:`AuditJob` or None
            The requested tool or None if not found
        """
        for t in self.tools:
            if t.name == tool_name:
                return t

    def save(self, path):
        """
        Output the kitlist to YAML document.

        Parameters
        ----------
        path : str
            The path to write to
        """

        to_save = {'tools': [t.as_dict() for t in self.tools]}
        with open(path, 'w') as f:
            yaml.dump(to_save, f, explicit_start=True, Dumper=Dumper)

    def to_stdout(self):
        """
        Output the kitlist to stdout.
        """

        to_save = {'tools': [t.as_dict() for t in self.tools]}
        print >> sys.stdout, yaml.dump(
            to_save, explicit_start=True, Dumper=Dumper)

    @classmethod
    def _fixup_regex(cls, regex):
        """
        Replace double backslashes in a string with single backslashes.

        PyYAML needs backslashes in strings to be escaped but they need
        to be unescaped when used with :module:`re`.

        Parameters
        ----------
        regex : str
            A regular expression string in which backslashes are escaped

        Returns
        -------
        fixed_regex : str
            The input string with backslashes unescaped
        """

        return regex.replace('\\\\', '\\')
<commit_msg>Absolute imports, iteritems and print<commit_after>"""
Contains the KitList class
"""

from __future__ import print_function
from collections import namedtuple
import sys
import yaml
try:
    from yaml import CLoader as Loader, CDumper as Dumper
except ImportError:
    from yaml import Loader, Dumper
from . import readers
from . import testers
from six import iteritems

Reader = namedtuple('Reader', ['name', 'func', 'args'])
Tester = namedtuple('Tester', ['name', 'func', 'args'])

class AuditJob(object):  # pylint: disable=R0903
    """
    The audit of a single tool
    """

    def __init__(self, name, path, reader, tester=None, version=None,
                 checksum=None, output_checksum=None):
        self.name = name
        self.path = path
        self.reader = reader
        self.tester = tester
        self.version = version
        self.checksum = checksum
        self.output_checksum = output_checksum

    def __repr__(self):
        r = "{0}({1!r}, {2!r}, {3!r}, {4!r}, {5!r}, {6!r}, {7!r}, {8!r})"
        return r.format(
            'AuditJob',
            self.name,
            self.path,
            self.reader,
            self.tester,
            self.version,
            self.checksum,
            self.output_checksum
        )

    def as_dict(self):
        """
        Convert to a dict

        Returns
        -------
        dict : AuditJob
            This AuditJob as a dictionary
        """
        reader_dict = {'name': self.reader.name}
        for k, v in iteritems(self.reader.args):
            reader_dict[k] = v
        if self.tester:
            tester_dict = {'name': self.tester.name}
            for k, v in iteritems(self.tester.args):
                tester_dict[k] = v
        else:
            tester_dict = None
        return {
            'name': self.name,
            'path': self.path,
            'reader': reader_dict,
            'tester': tester_dict,
            'version': self.version,
            'checksum': self.checksum,
            'output_checksum': self.output_checksum
        }

class KitList(object):
    """
    Represents a list of applications used in a pipeline and the procedure
    used to check each applications version
    """

    reader_functions = {
        'command_line': readers.command_line,
        'line_in_file': readers.line_in_file,
        'manual': readers.manual
    }

    tester_functions = {
        'stdout': testers.stdout,
        'file': testers.fileout
    }

    def __init__(self):
        self.tools = []

    @classmethod
    def from_file(cls, path):
        """
        Read the KitList description at *path* using a new instance of
        :class:`KitList` and return the resulting instance.

        Parameters
        ----------
        path : str
            The path to read from

        Returns
        -------
        :class:`KitList` : kitlist
            An instance with data loaded
        """

        instance = cls()
        instance.read(path)
        return instance

    def read(self, path):
        """
        Read a saved KitList in YAML format

        Parameters
        ----------
        path : str
            Path to a saved KitList
        """

        with open(path, 'r') as f:
            yaml_data = yaml.load(f, Loader=Loader)
        if 'tools' not in yaml_data:
            raise(KeyError(
                'The kitlist provided does not contain a tools element'
            ))
        tools = []
        for tool in yaml_data['tools']:
            tool_name = tool['name']
            tool_path = tool['path']
            reader = self._parse_reader_element(tool['reader'])
            if 'test' in tool:
                tester = self._parse_tester_element(tool['test'])
            else:
                tester = None
            audit_job = AuditJob(
                tool_name, tool_path,
                reader,
                tester,
                tool.get('version', None),
                tool.get('checksum', None),
                tool.get('output_checksum', None)
            )
            tools.append(audit_job)
        self.tools = tools

    @classmethod
    def _parse_tester_element(cls, element):
        """
        Take a tester element read from a KitList file and convert it to a
        Tester namedtuple
        """
        tester_name = element['name']
        if tester_name not in cls.tester_functions:
            raise(KeyError(
                'Unknown tester function {}'.format(tester_name)
            ))
        args = [
            (k, v) for k, v in iteritems(element) if k != 'name'
        ]
        tester_func = cls.tester_functions[tester_name]
        tester_args = dict(args)
        tester = Tester(tester_name, tester_func, tester_args)
        return tester

    @classmethod
    def _parse_reader_element(cls, element):
        """
        Take a reader element read from a KitList file and convert it to a
        Reader namedtuple
        """
        reader_name = element['name']
        if reader_name not in cls.reader_functions:
            raise(KeyError(
                'Unknown reader function {}'.format(reader_name)
            ))
        args = [
            (k, v) for k, v in iteritems(element) if k != 'name'
        ]
        reader_func = cls.reader_functions[reader_name]
        reader_args = dict(args)
        if 'regex' in reader_args:
            reader_args['regex'] = cls._fixup_regex(reader_args['regex'])
        reader = Reader(reader_name, reader_func, reader_args)
        return reader

    def get_tool(self, tool_name):
        """
        Get a single tool identifed by name

        Parameters
        ----------
        tool_name : str
            The name of the tool to retrieve

        Returns
        -------
        tool : class:`AuditJob` or None
            The requested tool or None if not found
        """
        for t in self.tools:
            if t.name == tool_name:
                return t

    def save(self, path):
        """
        Output the kitlist to YAML document.

        Parameters
        ----------
        path : str
            The path to write to
        """

        to_save = {'tools': [t.as_dict() for t in self.tools]}
        with open(path, 'w') as f:
            yaml.dump(to_save, f, explicit_start=True, Dumper=Dumper)

    def to_stdout(self):
        """
        Output the kitlist to stdout.
        """

        to_save = {'tools': [t.as_dict() for t in self.tools]}
        print(
            yaml.dump(to_save, explicit_start=True, Dumper=Dumper),
            file=sys.stdout
        )

    @classmethod
    def _fixup_regex(cls, regex):
        """
        Replace double backslashes in a string with single backslashes.

        PyYAML needs backslashes in strings to be escaped but they need
        to be unescaped when used with :module:`re`.

        Parameters
        ----------
        regex : str
            A regular expression string in which backslashes are escaped

        Returns
        -------
        fixed_regex : str
            The input string with backslashes unescaped
        """

        return regex.replace('\\\\', '\\')
<|endoftext|>