<commit_before>import os, re, sublime, sublime_plugin, colorsys

#	RGBA	rgba(RRR, GGG, BBB, A.A)
# rgb(50, 150, 200)
# rgba(50, 150, 200, 1)
#	ARGB	argb(AAA, RRR, GGG, BBB)
# argb(255, 50, 150, 200)
#	HEX		#RRGGBBAA
# #A98
# #FA98
# #AA9988
# #FFAA9988
#	INT		0xAARRGGBB
# 0x5599FF
# 0xFFAA443F
#	HSLA	hsla(HHH, SSS, LLL, A.A)
# hsl(100, 50, 50)
# hsla(100, 50%, 50%, 0.5)
#
class Color():

	rgb_regex = re.compile('a?rgba?\( *(\d{1,3}) *, *(\d{1,3}) *, *(\d{1,3}) *(?:, *(\d{0,3}(?:\.\d+)?) *)?\)')
	hsl_regex = re.compile('hsla?\( *(\d{1,3}) *, *(\d{1,3})%? *, *(\d{1,3})%? *(?:, *(\d?(?:\.\d+)?) *)?\)')
	r = 0
	g = 0
	b = 0
	a = 255
	valid = True

	def __init__(self, color):
		if color.find("rgb") == 0:
			s = re.search(self.rgb_regex, color)
			(r, g, b, a) = s.groups()
			self.r = int('0'+ r)
			self.g = int('0'+ g)
			self.b = int('0'+ b)
			if a == None:
				self.a = 255
			else:
				self.a = int(float('0'+ a) * 255)
		elif color.find("argb") == 0:
			s = re.search(self.rgb_regex, color)
			(a, r, g, b) = s.groups()
			if b == None:
				self.valid = False
			else:
				self.r = int('0'+ r)
				self.g = int('0'+ g)
				self.b = int('0'+ b)
				self.a = int('0'+ a)
		elif color.find("hsl") == 0:
			s = re.search(self.hsl_regex, color)
			(h, s, l, a) = s.groups()
			rgb = colorsys.hls_to_rgb(float('0'+ h) / 360, float('0'+ s) / 100, float('0'+ l) / 100)
			self.r = int(rgb[0] * 255)
			self.g = int(rgb[1] * 255)
			self.b = int(rgb[2] * 255)
			if a == None:
				self.a = 255
			else:
				self.a = int(float('0'+ a) * 255)
		elif color.find("0x") == 0:
			length = len(color) - 2
			if length == 6:
				(a, r, g, b) = ("00", color[2:4], color[4:6], color[6:8])
			elif length == 8:
				(a, r, g, b) = (color[2:4], color[4:6], color[6:8], color[8:])
			else:
				self.valid = False
				return
			self.r = self._htoi(r)
			self.g = self._htoi(g)
			self.b = self._htoi(b)
			self.a = self._htoi(a)
		elif color.find("#") == 0:
			length = len(color) - 1
			if length == 3 or length == 4:
				if length == 3:
					(r, g, b, a) = (color[1:2], color[2:3], color[3:4], "F")
				else:
					(r, g, b, a) = (color[1:2], color[2:3], color[3:4], color[4])
				r += r
				g += g
				b += b
				a += a
			elif length == 6:
				(r, g, b, a) = (color[1:3], color[3:5], color[5:7], "FF")
			elif length == 8:
				(r, g, b, a) = (color[1:3], color[3:5], color[5:7], color[7:])
			else:
				self.valid = False
				return
			self.r = self._htoi(r)
			self.g = self._htoi(g)
			self.b = self._htoi(b)
			self.a = self._htoi(a)

	def _htoi(self, hex):
		return int(hex, 16)

	def _itoh(self, n):
		return (('0'+ hex(n)[2:])[-2:]).upper()

	def to_hsl(self):
		c = colorsys.rgb_to_hls(self.r / 255, self.g / 255, self.b / 255)
		hsl = str(round(c[0] * 360)) +", "+ str(round(c[1] * 100)) +"%, "+ str(round(c[2] * 100)) +"%"
		if self.a == 255:
			return "hsl("+ hsl +")"
		else:
			return "hsla("+ hsl +", "+ str(round(self.a / 255, 2)) +")"

	def to_int(self):
		i = "0x"
		if self.a != 0:
			i += self._itoh(self.a)
		return i + self._itoh(self.r) + self._itoh(self.g) + self._itoh(self.b)

	def to_hex(self):
		h = "#"+ self._itoh(self.r) + self._itoh(self.g) + self._itoh(self.b)
		if self.a != 255:
			h += self._itoh(self.a)
		return h

	def to_rgb(self):
		rgb = str(self.r) +", "+ str(self.g) +", "+ str(self.b)
		if self.a == 255:
			return "rgb("+ rgb +")"
		else:
			if self.a == 0:
				a = "0"
			else:
				a = str(round(self.a / 255, 2))
			return "rgba("+ rgb +", "+ a +")"

	def sublime_hex(self):
		h = "#"+ self._itoh(self.r) + self._itoh(self.g) + self._itoh(self.b)
		if self.a == 0:
			h += "01"
		else:
			h += self._itoh(self.a)
		return h

	def contrasted_hex(self):
		if self.r < 10 and self.g < 10 and self.b < 10:
			return "#FFFFFFFF"
		else:
			return "#000000FF"

lastColor = None
lastColorRegion = None

class JulooColorConvert(sublime_plugin.TextCommand):

	def run(self, edit, **args):
		global lastColorRegion, lastColor
		if lastColorRegion != None:
			arg = args['to']
			if arg == 'hex':
				self.view.replace(edit, lastColorRegion, lastColor.to_hex())
			elif arg == 'rgb':
				self.view.replace(edit, lastColorRegion, lastColor.to_rgb())
			elif arg == 'hsl':
				self.view.replace(edit, lastColorRegion, lastColor.to_hsl())
			else:
				self.view.replace(edit, lastColorRegion, lastColor.to_int())
			print(arg)

class JulooColorHighlight(sublime_plugin.EventListener):

	color_regex = '(#|0x)[0-9a-fA-F]{8}|(#|0x)[0-9a-fA-F]{6}|#[0-9a-fA-F]{3,4}|(?:a?rgba?\( *\d{1,3} *, *\d{1,3} *, *\d{1,3} *(?:, *\d{0,3}(?:\.\d+)? *)?\))|(?:hsla?\( *\d{1,3} *, *\d{1,3}%? *, *\d{1,3}%? *(?:, *\d{0,3}(?:\.\d+)? *)?\))'
	#xml_location = "Packages/User/JulooColorHighlight/"
	xml_location = "Packages/JulooColorHighlight/tmThemeCache/"
	xml_template = """	<dict>
			<key>name</key>
			<string>colorhighlight</string>
			<key>scope</key>
			<string>juloo.color</string>
			<key>settings</key>
			<dict>
				<key>background</key>
				<string>#000000</string>
				<key>foreground</key>
				<string>{0}</string>
			</dict>
		</dict>
		<dict>
			<key>name</key>
			<string>colorhighlighttext</string>
			<key>scope</key>
			<string>juloo.colortext</string>
			<key>settings</key>
			<dict>
				<key>fontStyle</key>
				<string>normal</string>
				<key>background</key>
				<string>{0}</string>
				<key>foreground</key>
				<string>{1}</string>
			</dict>
		</dict>
	"""
	tmp_sel = None

	def get_xml_path(self, id):
		return self.xml_location + str(id) +".tmTheme"

	def get_full_path(self, theme_path):
		return os.path.join(sublime.packages_path(), os.path.normpath(theme_path[9:]))

	def get_colored_region(self, view):
		if len(view.sel()) == 1:
			sel = view.sel()[0]
			line = view.line(sel)
			m = view.line(sel)
			m.a = m.begin()
			m.b = m.begin()
			max_iteration = 5
			while max_iteration > 0:
				m = view.find(self.color_regex, m.end())
				if m == None or m.end() > line.end():
					break
				if m.contains(sel):
					return m
				max_iteration -= 1
		return None

	def on_close(self, view):
		if view.settings().has("old_color_scheme"):
			old_color_scheme = view.settings().get("old_color_scheme")
			view.settings().set("color_scheme", old_color_scheme)
			view.settings().erase("old_color_scheme")
		full_path = self.get_full_path(self.get_xml_path(view.id()))
		if os.path.exists(full_path):
			os.remove(full_path)

	def on_selection_modified_async(self, view):
		global lastColorRegion, lastColor
		if len(view.sel()) == 0 or view.sel()[0] == self.tmp_sel:
			return;
		else:
			self.tmp_sel = view.sel()[0]
		region = self.get_colored_region(view)
		if region == None:
			view.erase_status("color_juloo")
			view.erase_regions("colorhightlight")
			view.erase_regions("texthightlight")
			if view.settings().has("old_color_scheme"):
				view.settings().erase("old_color_scheme")
				view.settings().erase("color_scheme")
				full_path = self.get_full_path(self.get_xml_path(view.id()))
				if os.path.exists(full_path):
					os.remove(full_path)
		else:
			lastColorRegion = region
			color = Color(view.substr(region))
			lastColor = color
			if color.valid:
				status = "[ Color: "+ color.to_hex() +", "+ color.to_rgb() +", "+ color.to_int() +", "+ color.to_hsl() +" ]"
			else:
				status = "[ Invalid color ]"
			view.set_status("color_juloo", status)
			if not color.valid:
				return;
			if view.settings().has("old_color_scheme"):
				color_scheme = view.settings().get("old_color_scheme")
			else:
				color_scheme = view.settings().get("color_scheme")
				view.settings().set("old_color_scheme", color_scheme)
			data = sublime.load_resource(color_scheme)
			index = data.find("</array>")
			xml = self.xml_template.format(color.sublime_hex(), color.contrasted_hex())
			data = data[:index] + xml + data[index:]
			if not os.path.exists(self.get_full_path(self.xml_location)):
				os.mkdir(self.get_full_path(self.xml_location))
			f = open(self.get_full_path(self.get_xml_path(view.id())), "wb")
			f.write(data.encode("utf-8"))
			f.close()
			view.settings().set("color_scheme", self.get_xml_path(view.id()))
			view.add_regions("colorhightlight", [region], "juloo.color", "circle", sublime.DRAW_NO_OUTLINE)
			view.add_regions("texthightlight", [region], "juloo.colortext", "", sublime.DRAW_NO_OUTLINE)
<commit_msg>Change cache location, optimise color finding and fix region bug<commit_after>import os, re, sublime, sublime_plugin, colorsys

#	RGBA	rgba(RRR, GGG, BBB, A.A)
# rgb(50, 150, 200)
# rgba(50, 150, 200, 0.5)
#	ARGB	argb(AAA, RRR, GGG, BBB)
# argb(255, 50, 150, 200)
#	HEX		#RRGGBBAA
# #A98
# #A98F
# #AA9988
# #AA9988FF
#	INT		0xAARRGGBB
# 0x5599FF
# 0xFFAA443F
#	HSLA	hsla(HHH, SSS, LLL, A.A)
# hsl(100, 50, 50)
# hsla(100, 50%, 50%, 0.5)
#
class Color():

	rgb_regex = re.compile('a?rgba?\( *(\d{1,3}) *, *(\d{1,3}) *, *(\d{1,3}) *(?:, *(\d{0,3}(?:\.\d+)?) *)?\)')
	hsl_regex = re.compile('hsla?\( *(\d{1,3}) *, *(\d{1,3})%? *, *(\d{1,3})%? *(?:, *(\d?(?:\.\d+)?) *)?\)')
	r = 0
	g = 0
	b = 0
	a = 255
	valid = True

	def __init__(self, color):
		if color.find("rgb") == 0:
			s = re.search(self.rgb_regex, color)
			(r, g, b, a) = s.groups()
			self.r = int('0'+ r)
			self.g = int('0'+ g)
			self.b = int('0'+ b)
			if a == None:
				self.a = 255
			else:
				self.a = int(float('0'+ a) * 255)
		elif color.find("argb") == 0:
			s = re.search(self.rgb_regex, color)
			(a, r, g, b) = s.groups()
			if b == None:
				self.valid = False
			else:
				self.r = int('0'+ r)
				self.g = int('0'+ g)
				self.b = int('0'+ b)
				self.a = int('0'+ a)
		elif color.find("hsl") == 0:
			s = re.search(self.hsl_regex, color)
			(h, s, l, a) = s.groups()
			rgb = colorsys.hls_to_rgb(float('0'+ h) / 360, float('0'+ s) / 100, float('0'+ l) / 100)
			self.r = int(rgb[0] * 255)
			self.g = int(rgb[1] * 255)
			self.b = int(rgb[2] * 255)
			if a == None:
				self.a = 255
			else:
				self.a = int(float('0'+ a) * 255)
		elif color.find("0x") == 0:
			length = len(color) - 2
			if length == 6:
				(a, r, g, b) = ("00", color[2:4], color[4:6], color[6:8])
			elif length == 8:
				(a, r, g, b) = (color[2:4], color[4:6], color[6:8], color[8:])
			else:
				self.valid = False
				return
			self.r = self._htoi(r)
			self.g = self._htoi(g)
			self.b = self._htoi(b)
			self.a = self._htoi(a)
		elif color.find("#") == 0:
			length = len(color) - 1
			if length == 3 or length == 4:
				if length == 3:
					(r, g, b, a) = (color[1:2], color[2:3], color[3:4], "F")
				else:
					(r, g, b, a) = (color[1:2], color[2:3], color[3:4], color[4])
				r += r
				g += g
				b += b
				a += a
			elif length == 6:
				(r, g, b, a) = (color[1:3], color[3:5], color[5:7], "FF")
			elif length == 8:
				(r, g, b, a) = (color[1:3], color[3:5], color[5:7], color[7:])
			else:
				self.valid = False
				return
			self.r = self._htoi(r)
			self.g = self._htoi(g)
			self.b = self._htoi(b)
			self.a = self._htoi(a)

	def _htoi(self, hex):
		return int(hex, 16)

	def _itoh(self, n):
		return (('0'+ hex(n)[2:])[-2:]).upper()

	def to_hsl(self):
		c = colorsys.rgb_to_hls(self.r / 255, self.g / 255, self.b / 255)
		hsl = str(round(c[0] * 360)) +", "+ str(round(c[1] * 100)) +"%, "+ str(round(c[2] * 100)) +"%"
		if self.a == 255:
			return "hsl("+ hsl +")"
		else:
			return "hsla("+ hsl +", "+ str(round(self.a / 255, 2)) +")"

	def to_int(self):
		i = "0x"
		if self.a != 0:
			i += self._itoh(self.a)
		return i + self._itoh(self.r) + self._itoh(self.g) + self._itoh(self.b)

	def to_hex(self):
		h = "#"+ self._itoh(self.r) + self._itoh(self.g) + self._itoh(self.b)
		if self.a != 255:
			h += self._itoh(self.a)
		return h

	def to_rgb(self):
		rgb = str(self.r) +", "+ str(self.g) +", "+ str(self.b)
		if self.a == 255:
			return "rgb("+ rgb +")"
		else:
			if self.a == 0:
				a = "0"
			else:
				a = str(round(self.a / 255, 2))
			return "rgba("+ rgb +", "+ a +")"

	def sublime_hex(self):
		h = "#"+ self._itoh(self.r) + self._itoh(self.g) + self._itoh(self.b)
		if self.a == 0:
			h += "01"
		else:
			h += self._itoh(self.a)
		return h

	def contrasted_hex(self):
		if self.r < 10 and self.g < 10 and self.b < 10:
			return "#FFFFFFFF"
		else:
			return "#000000FF"

lastColor = None
lastColorRegion = None

class JulooColorConvert(sublime_plugin.TextCommand):

	def run(self, edit, **args):
		global lastColorRegion, lastColor
		if lastColorRegion != None:
			arg = args['to']
			if arg == 'hex':
				self.view.replace(edit, lastColorRegion, lastColor.to_hex())
			elif arg == 'rgb':
				self.view.replace(edit, lastColorRegion, lastColor.to_rgb())
			elif arg == 'hsl':
				self.view.replace(edit, lastColorRegion, lastColor.to_hsl())
			else:
				self.view.replace(edit, lastColorRegion, lastColor.to_int())
			print(arg)

class JulooColorHighlight(sublime_plugin.EventListener):

	color_regex = '(#|0x)[0-9a-fA-F]{8}|(#|0x)[0-9a-fA-F]{6}|#[0-9a-fA-F]{3,4}|(?:a?rgba?\( *\d{1,3} *, *\d{1,3} *, *\d{1,3} *(?:, *\d{0,3}(?:\.\d+)? *)?\))|(?:hsla?\( *\d{1,3} *, *\d{1,3}%? *, *\d{1,3}%? *(?:, *\d{0,3}(?:\.\d+)? *)?\))'
	xml_location = "Packages/julooColorHighlightCache/"
	xml_template = """	<dict>
			<key>name</key>
			<string>colorhighlight</string>
			<key>scope</key>
			<string>juloo.color</string>
			<key>settings</key>
			<dict>
				<key>background</key>
				<string>#000000</string>
				<key>foreground</key>
				<string>{0}</string>
			</dict>
		</dict>
		<dict>
			<key>name</key>
			<string>colorhighlighttext</string>
			<key>scope</key>
			<string>juloo.colortext</string>
			<key>settings</key>
			<dict>
				<key>fontStyle</key>
				<string>normal</string>
				<key>background</key>
				<string>{0}</string>
				<key>foreground</key>
				<string>{1}</string>
			</dict>
		</dict>
	"""
	tmp_sel = None

	def get_xml_path(self, id):
		return self.xml_location + str(id) +".tmTheme"

	def get_full_path(self, theme_path):
		return os.path.join(sublime.packages_path(), os.path.normpath(theme_path[9:]))

	def get_colored_region(self, view):
		if len(view.sel()) == 1:
			sel = view.sel()[0]
			sel = sublime.Region(sel.end(), sel.end())
			line = view.line(sel)
			startPos = max(line.begin(), sel.begin() - 30)
			endPos = min(sel.end() + 30, line.end())
			m = sublime.Region(startPos, startPos)
			max_iteration = 5
			while max_iteration > 0:
				m = view.find(self.color_regex, m.end())
				if m == None or m.end() > endPos:
					break
				if m.contains(sel):
					return m
				max_iteration -= 1
		return None

	def on_close(self, view):
		if view.settings().has("old_color_scheme"):
			old_color_scheme = view.settings().get("old_color_scheme")
			view.settings().set("color_scheme", old_color_scheme)
			view.settings().erase("old_color_scheme")
		full_path = self.get_full_path(self.get_xml_path(view.id()))
		if os.path.exists(full_path):
			os.remove(full_path)

	def on_selection_modified_async(self, view):
		global lastColorRegion, lastColor
		if len(view.sel()) == 0 or view.sel()[0] == self.tmp_sel:
			return;
		else:
			self.tmp_sel = view.sel()[0]
		region = self.get_colored_region(view)
		if region == None:
			view.erase_status("color_juloo")
			view.erase_regions("colorhightlight")
			view.erase_regions("texthightlight")
			if view.settings().has("old_color_scheme"):
				view.settings().erase("old_color_scheme")
				view.settings().erase("color_scheme")
				full_path = self.get_full_path(self.get_xml_path(view.id()))
				if os.path.exists(full_path):
					os.remove(full_path)
		else:
			lastColorRegion = region
			color = Color(view.substr(region))
			lastColor = color
			if color.valid:
				status = "[ Color: "+ color.to_hex() +", "+ color.to_rgb() +", "+ color.to_int() +", "+ color.to_hsl() +" ]"
			else:
				status = "[ Invalid color ]"
			view.set_status("color_juloo", status)
			if not color.valid:
				return;
			if view.settings().has("old_color_scheme"):
				color_scheme = view.settings().get("old_color_scheme")
			else:
				color_scheme = view.settings().get("color_scheme")
				view.settings().set("old_color_scheme", color_scheme)
			data = sublime.load_resource(color_scheme)
			index = data.find("</array>")
			xml = self.xml_template.format(color.sublime_hex(), color.contrasted_hex())
			data = data[:index] + xml + data[index:]
			if not os.path.exists(self.get_full_path(self.xml_location)):
				os.mkdir(self.get_full_path(self.xml_location))
			f = open(self.get_full_path(self.get_xml_path(view.id())), "wb")
			f.write(data.encode("utf-8"))
			f.close()
			view.settings().set("color_scheme", self.get_xml_path(view.id()))
			view.add_regions("colorhightlight", [region], "juloo.color", "circle", sublime.HIDDEN)
			view.add_regions("texthightlight", [region], "juloo.colortext", "", sublime.DRAW_NO_OUTLINE)
<|endoftext|>


<commit_before># This file is part of LiSE, a framework for life simulation games.
# Copyright (c) Zachary Spector,  zacharyspector@gmail.com
"""Sickle cell anemia vs. malaria, a classic example of population genetics.

This script will initialize LiSEworld.db and the game code libraries to run the
simulation described. To view it, run ELiDE from the same directory
as you ran this script from.

"""

import networkx as nx
from LiSE import Engine
from os import remove

def sickle_cell_test(
        engine,
        n_creatures=5,
        n_sickles=3,
        malaria_chance=.05,
        mate_chance=.05,
        mapsize=(1, 1),
        startpos=(0, 0),
        turns=100
):
    """Natural Selection on Sickle Cell Anemia

    If anyone carries a pair of sickle betaglobin genes, they die of
    sickle cell anemia.

    Individuals with 1x betaglobin, 1x sickle betaglobin are immune to
    malaria.

    """
    initmap = nx.grid_2d_graph(*mapsize)
    phys = engine.new_character("physical", data=initmap)
    species = engine.new_character(
        "species",
        mate_chance=mate_chance,
        malaria_chance=malaria_chance,
        n_creatures=n_creatures,
    )
    for n in range(0, n_creatures):
        name = "critter" + str(n)
        phys.add_thing(
            name=name,
            location=startpos,
            sickle_a=(n < n_sickles),
            sickle_b=False,
            male=engine.coinflip(),
            last_mate_turn=-1
        )
        assert name in phys.thing
        assert name not in phys.place
        assert name in phys.node
        assert hasattr(phys.node[name], 'location')
        species.add_avatar("physical", name)
        assert hasattr(species.avatar['physical'][name], 'location')

# putting dieoff earlier in the code than mate means that dieoff will
# be followed before mate is
    @species.avatar.rule
    def dieoff(critter):
        critter.delete()
        assert(critter.name not in critter.character.node)
        if critter['from_malaria']:
            return 'malaria'
        else:
            return 'anemia'

    @species.avatar.rule
    def mate(critter):
        """If I share my location with another critter, attempt to mate"""
        suitors = list(
            oc for oc in critter.location.contents()
            if oc['male'] != critter['male']
        )
        assert(len(suitors) > 0)
        other_critter = critter.engine.choice(suitors)
        sickles = [
            critter['sickle_a'],
            critter['sickle_b'],
            other_critter['sickle_a'],
            other_critter['sickle_b']
        ]
        engine.shuffle(sickles)
        name = "critter" + str(species.stat["n_creatures"])
        species.stat["n_creatures"] += 1
        engine.character["physical"].add_thing(
            name,
            critter["location"],
            sickle_a=sickles.pop(),
            sickle_b=sickles.pop(),
            male=engine.coinflip(),
            last_mate_turn=engine.turn
        )
        species.add_avatar("physical", name)
        critter['last_mate_turn'] = other_critter['last_mate_turn'] =\
            engine.turn
        return 'mated'

    @mate.prereq
    def once_per_turn(critter):
        return critter['last_mate_turn'] < critter.engine.turn

    @mate.prereq
    def mate_present(critter):
        for oc in critter.location.contents():
            if oc['male'] != critter['male']:
                return True
        return False

    @mate.trigger
    def in_the_mood(critter):
        return critter.engine.random() < critter.user.stat['mate_chance']

    @dieoff.trigger
    def sickle2(critter):
        r = critter['sickle_a'] and critter['sickle_b']
        if r:
            critter['from_malaria'] = False
        return r

    @dieoff.trigger
    def malaria(critter):
        r = (
            critter.engine.random() < critter.user.stat['malaria_chance'] and not
            (critter['sickle_a'] or critter['sickle_b'])
        )
        if r:
            critter['from_malaria'] = True
        return r

    # it would make more sense to keep using species.avatar.rule, this
    # is just a test
    @phys.thing.rule
    def wander(critter):
        dest = critter.engine.choice(list(critter.character.place.keys()))
        critter.travel_to(dest)

    @wander.trigger
    def not_travelling(critter):
        return critter['next_location'] is None

    print(
        "Starting with {} creatures, of which {} have "
        "at least one sickle betaglobin.".format(
            len(species.avatar['physical']),
            sum(
                1 for critter in species.avatar['physical'].values()
                if critter['sickle_a'] or critter['sickle_b']
            )
        )
    )

    for i in range(0, turns):
        malaria_dead = 0
        anemia_dead = 0
        born = 0
        while engine.turn < i:
            r = engine.next_turn()
            if not r:
                continue
            r = r[0]
            if 'malaria' in r:
                malaria_dead += 1
            if 'anemia' in r:
                anemia_dead += 1
            if 'mated' in r:
                born += 1
        print("On tick {}, {} critters were born; "
              "{} died of malaria, and {} of sickle cell anemia, "
              "leaving {} alive.".format(
                  i, born, malaria_dead, anemia_dead,
                  len(engine.character['species'].avatar['physical'])
              ))
    print(
        "Of the remaining {} creatures, {} have a sickle betaglobin.".format(
            len(species.avatar['physical']),
            sum(
                1 for critter in species.avatar['physical'].values()
                if critter['sickle_a'] or critter['sickle_b']
            )
        )
    )

try:
    remove('LiSEworld.db')
except OSError:
    pass
with Engine('LiSEworld.db', random_seed=69105) as engine:
    sickle_cell_test(engine)
<commit_msg>tick -> turn<commit_after># This file is part of LiSE, a framework for life simulation games.
# Copyright (c) Zachary Spector,  zacharyspector@gmail.com
"""Sickle cell anemia vs. malaria, a classic example of population genetics.

This script will initialize LiSEworld.db and the game code libraries to run the
simulation described. To view it, run ELiDE from the same directory
as you ran this script from.

"""

import networkx as nx
from LiSE import Engine
from os import remove

def sickle_cell_test(
        engine,
        n_creatures=5,
        n_sickles=3,
        malaria_chance=.05,
        mate_chance=.05,
        mapsize=(1, 1),
        startpos=(0, 0),
        turns=100
):
    """Natural Selection on Sickle Cell Anemia

    If anyone carries a pair of sickle betaglobin genes, they die of
    sickle cell anemia.

    Individuals with 1x betaglobin, 1x sickle betaglobin are immune to
    malaria.

    """
    initmap = nx.grid_2d_graph(*mapsize)
    phys = engine.new_character("physical", data=initmap)
    species = engine.new_character(
        "species",
        mate_chance=mate_chance,
        malaria_chance=malaria_chance,
        n_creatures=n_creatures,
    )
    for n in range(0, n_creatures):
        name = "critter" + str(n)
        phys.add_thing(
            name=name,
            location=startpos,
            sickle_a=(n < n_sickles),
            sickle_b=False,
            male=engine.coinflip(),
            last_mate_turn=-1
        )
        assert name in phys.thing
        assert name not in phys.place
        assert name in phys.node
        assert hasattr(phys.node[name], 'location')
        species.add_avatar("physical", name)
        assert hasattr(species.avatar['physical'][name], 'location')

# putting dieoff earlier in the code than mate means that dieoff will
# be followed before mate is
    @species.avatar.rule
    def dieoff(critter):
        critter.delete()
        assert(critter.name not in critter.character.node)
        if critter['from_malaria']:
            return 'malaria'
        else:
            return 'anemia'

    @species.avatar.rule
    def mate(critter):
        """If I share my location with another critter, attempt to mate"""
        suitors = list(
            oc for oc in critter.location.contents()
            if oc['male'] != critter['male']
        )
        assert(len(suitors) > 0)
        other_critter = critter.engine.choice(suitors)
        sickles = [
            critter['sickle_a'],
            critter['sickle_b'],
            other_critter['sickle_a'],
            other_critter['sickle_b']
        ]
        engine.shuffle(sickles)
        name = "critter" + str(species.stat["n_creatures"])
        species.stat["n_creatures"] += 1
        engine.character["physical"].add_thing(
            name,
            critter["location"],
            sickle_a=sickles.pop(),
            sickle_b=sickles.pop(),
            male=engine.coinflip(),
            last_mate_turn=engine.turn
        )
        species.add_avatar("physical", name)
        critter['last_mate_turn'] = other_critter['last_mate_turn'] =\
            engine.turn
        return 'mated'

    @mate.prereq
    def once_per_turn(critter):
        return critter['last_mate_turn'] < critter.engine.turn

    @mate.prereq
    def mate_present(critter):
        for oc in critter.location.contents():
            if oc['male'] != critter['male']:
                return True
        return False

    @mate.trigger
    def in_the_mood(critter):
        return critter.engine.random() < critter.user.stat['mate_chance']

    @dieoff.trigger
    def sickle2(critter):
        r = critter['sickle_a'] and critter['sickle_b']
        if r:
            critter['from_malaria'] = False
        return r

    @dieoff.trigger
    def malaria(critter):
        r = (
            critter.engine.random() < critter.user.stat['malaria_chance'] and not
            (critter['sickle_a'] or critter['sickle_b'])
        )
        if r:
            critter['from_malaria'] = True
        return r

    # it would make more sense to keep using species.avatar.rule, this
    # is just a test
    @phys.thing.rule
    def wander(critter):
        dest = critter.engine.choice(list(critter.character.place.keys()))
        critter.travel_to(dest)

    @wander.trigger
    def not_travelling(critter):
        return critter['next_location'] is None

    print(
        "Starting with {} creatures, of which {} have "
        "at least one sickle betaglobin.".format(
            len(species.avatar['physical']),
            sum(
                1 for critter in species.avatar['physical'].values()
                if critter['sickle_a'] or critter['sickle_b']
            )
        )
    )

    for i in range(0, turns):
        malaria_dead = 0
        anemia_dead = 0
        born = 0
        while engine.turn < i:
            r = engine.next_turn()
            if not r:
                continue
            r = r[0]
            if 'malaria' in r:
                malaria_dead += 1
            if 'anemia' in r:
                anemia_dead += 1
            if 'mated' in r:
                born += 1
        print("On turn {}, {} critters were born; "
              "{} died of malaria, and {} of sickle cell anemia, "
              "leaving {} alive.".format(
                  i, born, malaria_dead, anemia_dead,
                  len(engine.character['species'].avatar['physical'])
              ))
    print(
        "Of the remaining {} creatures, {} have a sickle betaglobin.".format(
            len(species.avatar['physical']),
            sum(
                1 for critter in species.avatar['physical'].values()
                if critter['sickle_a'] or critter['sickle_b']
            )
        )
    )

try:
    remove('LiSEworld.db')
except OSError:
    pass
with Engine('LiSEworld.db', random_seed=69105) as engine:
    sickle_cell_test(engine)
<|endoftext|>


<commit_before># Copyright 2015 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function, division, absolute_import

import glob
import logging
import os
import plistlib
import re
import tempfile

from booleanOperations import BooleanOperationManager
from cu2qu.pens import ReverseContourPen
from cu2qu.ufo import font_to_quadratic, fonts_to_quadratic
from defcon import Font
from fontTools import subset
from fontTools.misc.loggingTools import configLogger, Timer
from fontTools.misc.transform import Transform
from fontTools.pens.transformPen import TransformPen
from fontTools.ttLib import TTFont
from glyphsLib import build_masters, build_instances
from mutatorMath.ufo import build as build_designspace
from mutatorMath.ufo.document import DesignSpaceDocumentReader
from ufo2ft import compileOTF, compileTTF
from ufo2ft.makeotfParts import FeatureOTFCompiler

from fontmake.ttfautohint import ttfautohint

timer = Timer(logging.getLogger('fontmake'), level=logging.DEBUG)

PUBLIC_PREFIX = 'public.'
GLYPHS_PREFIX = 'com.schriftgestaltung.'

class FontProject:
    """Provides methods for building fonts."""

    @staticmethod
    def preprocess(glyphs_path):
        """Return Glyphs source with illegal glyph/class names changed."""

        with open(glyphs_path) as fp:
            text = fp.read()
        names = set(re.findall('\n(?:glyph)?name = "(.+-.+)";\n', text))

        if names:
            num_names = len(names)
            printed_names = sorted(names)[:5]
            if num_names > 5:
                printed_names.append('...')
            print('Found %s glyph names containing hyphens: %s' % (
                num_names, ', '.join(printed_names)))
            print('Replacing all hyphens with periods.')

        for old_name in names:
            new_name = old_name.replace('-', '.')
            text = text.replace(old_name, new_name)
        return text

    def __init__(self, timing=False):
        if timing:
            configLogger(logger=timer.logger, level=logging.DEBUG)

    @timer()
    def build_ufos(
            self, glyphs_path, interpolate=False, masters_as_instances=False,
            family_name=None):
        """Build UFOs from Glyphs source."""

        master_dir = self._output_dir('ufo')
        instance_dir = self._output_dir('ufo', is_instance=True)
        ufos = []
        if not interpolate or masters_as_instances:
            ufos.extend(build_masters(
                glyphs_path, master_dir, designspace_instance_dir=instance_dir,
                family_name=family_name))
        if interpolate:
            ufos.extend(build_instances(
                glyphs_path, master_dir, instance_dir, family_name=family_name))
        return ufos

    @timer()
    def remove_overlaps(self, ufos):
        """Remove overlaps in UFOs' glyphs' contours."""

        manager = BooleanOperationManager()
        for ufo in ufos:
            print('>> Removing overlaps for ' + self._font_name(ufo))
            for glyph in ufo:
                contours = list(glyph)
                glyph.clearContours()
                manager.union(contours, glyph.getPointPen())

    @timer()
    def decompose_glyphs(self, ufos):
        """Move components of UFOs' glyphs to their outlines."""

        for ufo in ufos:
            print('>> Decomposing glyphs for ' + self._font_name(ufo))
            for glyph in ufo:
                self._deep_copy_contours(ufo, glyph, glyph, Transform())
                glyph.clearComponents()

    def _deep_copy_contours(self, ufo, parent, component, transformation):
        """Copy contours from component to parent, including nested components."""

        for nested in component.components:
            self._deep_copy_contours(
                ufo, parent, ufo[nested.baseGlyph],
                transformation.transform(nested.transformation))

        if component != parent:
            pen = TransformPen(parent.getPen(), transformation)

            # if the transformation has a negative determinant, it will reverse
            # the contour direction of the component
            xx, xy, yx, yy = transformation[:4]
            if xx * yy - xy * yx < 0:
                pen = ReverseContourPen(pen)

            component.draw(pen)

    @timer()
    def convert_curves(self, ufos, compatible=False, reverse_direction=True):
        if compatible:
            fonts_to_quadratic(ufos, reverse_direction=reverse_direction,
                               dump_stats=True)
        else:
            for ufo in ufos:
                print('>> Converting curves for ' + self._font_name(ufo))
                font_to_quadratic(ufo, reverse_direction=reverse_direction,
                                  dump_stats=True)

    def build_otfs(self, ufos, remove_overlaps=True, **kwargs):
        """Build OpenType binaries with CFF outlines."""

        print('\n>> Building OTFs')

        self.decompose_glyphs(ufos)
        if remove_overlaps:
            self.remove_overlaps(ufos)
        del kwargs['reverse_direction']
        self.save_otfs(ufos, **kwargs)

    def build_ttfs(self, ufos, remove_overlaps=True, **kwargs):
        """Build OpenType binaries with TrueType outlines."""

        print('\n>> Building TTFs')

        if remove_overlaps:
            self.remove_overlaps(ufos)
        reverse_direction = kwargs.pop('reverse_direction')
        self.convert_curves(ufos, reverse_direction=reverse_direction)
        self.save_otfs(ufos, ttf=True, **kwargs)

    def build_interpolatable_ttfs(self, ufos, **kwargs):
        """Build OpenType binaries with interpolatable TrueType outlines."""

        print('\n>> Building interpolation-compatible TTFs')

        reverse_direction = kwargs.pop('reverse_direction')
        self.convert_curves(ufos, compatible=True,
                            reverse_direction=reverse_direction)
        self.save_otfs(ufos, ttf=True, interpolatable=True, **kwargs)

    @timer()
    def save_otfs(
            self, ufos, ttf=False, interpolatable=False, mti_paths=None,
            is_instance=False, use_afdko=False, autohint=None, subset=True,
            use_production_names=None):
        """Write OpenType binaries."""

        ext = 'ttf' if ttf else 'otf'
        fea_compiler = FDKFeatureCompiler if use_afdko else FeatureOTFCompiler
        otf_compiler = compileTTF if ttf else compileOTF

        for ufo in ufos:
            name = self._font_name(ufo)
            print('>> Saving %s for %s' % (ext.upper(), name))

            otf_path = self._output_path(ufo, ext, is_instance, interpolatable)
            if use_production_names is None:
                use_production_names = not ufo.lib.get(
                    GLYPHS_PREFIX + "Don't use Production Names")
            otf = otf_compiler(
                ufo, featureCompilerClass=fea_compiler,
                mtiFeaFiles=mti_paths[name] if mti_paths is not None else None,
                glyphOrder=ufo.lib[PUBLIC_PREFIX + 'glyphOrder'],
                useProductionNames=use_production_names)
            otf.save(otf_path)

            if subset:
                self.subset_otf_from_ufo(otf_path, ufo)

            if ttf and autohint is not None:
                hinted_otf_path = self._output_path(
                    ufo, ext, is_instance, interpolatable, autohinted=True)
                ttfautohint(otf_path, hinted_otf_path, args=autohint)

    def subset_otf_from_ufo(self, otf_path, ufo):
        """Subset a font using export flags set by glyphsLib."""

        keep_glyphs = set(ufo.lib.get(GLYPHS_PREFIX + 'Keep Glyphs', []))

        include = []
        ufo_order = [glyph_name
                     for glyph_name in ufo.lib[PUBLIC_PREFIX + 'glyphOrder']
                     if glyph_name in ufo]
        for old_name, new_name in zip(
                ufo_order,
                TTFont(otf_path).getGlyphOrder()):
            glyph = ufo[old_name]
            if ((keep_glyphs and old_name not in keep_glyphs) or
                not glyph.lib.get(GLYPHS_PREFIX + 'Glyphs.Export', True)):
                continue
            include.append(new_name)

        # copied from nototools.subset
        opt = subset.Options()
        opt.name_IDs = ['*']
        opt.name_legacy = True
        opt.name_languages = ['*']
        opt.layout_features = ['*']
        opt.notdef_outline = True
        opt.recalc_bounds = True
        opt.recalc_timestamp = True
        opt.canonical_order = True

        opt.glyph_names = True

        font = subset.load_font(otf_path, opt, lazy=False)
        subsetter = subset.Subsetter(options=opt)
        subsetter.populate(glyphs=include)
        subsetter.subset(font)
        subset.save_font(font, otf_path, opt)

    def run_from_glyphs(
            self, glyphs_path, preprocess=True, interpolate=False,
            masters_as_instances=False, family_name=None, **kwargs):
        """Run toolchain from Glyphs source to OpenType binaries."""

        if preprocess:
            print('>> Checking Glyphs source for illegal glyph names')
            glyphs_source = self.preprocess(glyphs_path)
            tmp_glyphs_file = tempfile.NamedTemporaryFile()
            glyphs_path = tmp_glyphs_file.name
            tmp_glyphs_file.write(glyphs_source)
            tmp_glyphs_file.seek(0)

        print('>> Building UFOs from Glyphs source')
        ufos = self.build_ufos(
            glyphs_path, interpolate, masters_as_instances, family_name)
        self.run_from_ufos(
            ufos, is_instance=(interpolate or masters_as_instances), **kwargs)

    def run_from_designspace(
            self, designspace_path, interpolate=False,
            masters_as_instances=False, **kwargs):
        """Run toolchain from a MutatorMath design space document to OpenType
        binaries.
        """

        ufos = []
        if not interpolate or masters_as_instances:
            reader = DesignSpaceDocumentReader(designspace_path, ufoVersion=3)
            ufos.extend(reader.getSourcePaths())
        if interpolate:
            print('>> Interpolating master UFOs from design space')
            results = build_designspace(
                designspace_path, outputUFOFormatVersion=3)
            for result in results:
                ufos.extend(result.values())
        self.run_from_ufos(
            ufos, is_instance=(interpolate or masters_as_instances), **kwargs)

    def run_from_ufos(self, ufos, output=(), mti_source=None, **kwargs):
        """Run toolchain from UFO sources to OpenType binaries."""

        if set(output) == set(['ufo']):
            return
        if hasattr(ufos[0], 'path'):
            ufo_paths = [ufo.path for ufo in ufos]
        else:
            if isinstance(ufos, str):
                ufo_paths = glob.glob(ufos)
            else:
                ufo_paths = ufos
            ufos = [Font(path) for path in ufo_paths]

        mti_paths = None
        if mti_source:
            mti_paths = {}
            mti_paths = plistlib.readPlist(mti_source)
            src_dir = os.path.dirname(mti_source)
            for paths in mti_paths.values():
                for tag in paths.keys():
                    paths[tag] = os.path.join(src_dir, paths[tag])

        need_reload = False
        if 'otf' in output:
            self.build_otfs(ufos, mti_paths=mti_paths, **kwargs)
            need_reload = True

        if 'ttf' in output:
            if need_reload:
                ufos = [Font(path) for path in ufo_paths]
            self.build_ttfs(ufos, mti_paths=mti_paths, **kwargs)
            need_reload = True

        if 'ttf-interpolatable' in output:
            if need_reload:
                ufos = [Font(path) for path in ufo_paths]
            self.build_interpolatable_ttfs(ufos, mti_paths=mti_paths, **kwargs)

    def _font_name(self, ufo):
        return '%s-%s' % (ufo.info.familyName.replace(' ', ''),
                          ufo.info.styleName.replace(' ', ''))

    def _output_dir(self, ext, is_instance=False, interpolatable=False,
                    autohinted=False):
        """Generate an output directory."""

        dir_prefix = 'instance_' if is_instance else 'master_'
        dir_suffix = '_interpolatable' if interpolatable else ''
        output_dir = dir_prefix + ext + dir_suffix
        if autohinted:
            output_dir = os.path.join('autohinted', output_dir)
        return output_dir

    def _output_path(self, ufo, ext, is_instance=False, interpolatable=False,
                     autohinted=False):
        """Generate output path for a font file with given extension."""

        out_dir = self._output_dir(ext, is_instance, interpolatable, autohinted)
        if not os.path.exists(out_dir):
            os.makedirs(out_dir)
        return os.path.join(out_dir, '%s.%s' % (self._font_name(ufo), ext))

class FDKFeatureCompiler(FeatureOTFCompiler):
    """An OTF compiler which uses the AFDKO to compile feature syntax."""

    def setupFile_featureTables(self):
        if self.mtiFeaFiles is not None:
            super(FDKFeatureCompiler, self).setupFile_featureTables()

        elif not self.features.strip():
            return

        import subprocess
        from fontTools.misc.py23 import tostr

        fd, outline_path = tempfile.mkstemp()
        os.close(fd)
        self.outline.save(outline_path)

        fd, feasrc_path = tempfile.mkstemp()
        os.close(fd)

        fd, fea_path = tempfile.mkstemp()
        os.write(fd, self.features)
        os.close(fd)

        report = tostr(subprocess.check_output([
            "makeotf", "-o", feasrc_path, "-f", outline_path,
            "-ff", fea_path]))
        os.remove(outline_path)
        os.remove(fea_path)

        print(report)
        success = "Done." in report
        if success:
            feasrc = TTFont(feasrc_path)
            for table in ["GDEF", "GPOS", "GSUB"]:
                if table in feasrc:
                    self.outline[table] = feasrc[table]
            feasrc.close()

        os.remove(feasrc_path)
        if not success:
            raise ValueError("Feature syntax compilation failed.")
<commit_msg>font_project: move booleanOperations import inside 'remove_overalap' method<commit_after># Copyright 2015 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function, division, absolute_import

import glob
import logging
import os
import plistlib
import re
import tempfile

from cu2qu.pens import ReverseContourPen
from cu2qu.ufo import font_to_quadratic, fonts_to_quadratic
from defcon import Font
from fontTools import subset
from fontTools.misc.loggingTools import configLogger, Timer
from fontTools.misc.transform import Transform
from fontTools.pens.transformPen import TransformPen
from fontTools.ttLib import TTFont
from glyphsLib import build_masters, build_instances
from mutatorMath.ufo import build as build_designspace
from mutatorMath.ufo.document import DesignSpaceDocumentReader
from ufo2ft import compileOTF, compileTTF
from ufo2ft.makeotfParts import FeatureOTFCompiler

from fontmake.ttfautohint import ttfautohint

timer = Timer(logging.getLogger('fontmake'), level=logging.DEBUG)

PUBLIC_PREFIX = 'public.'
GLYPHS_PREFIX = 'com.schriftgestaltung.'

class FontProject:
    """Provides methods for building fonts."""

    @staticmethod
    def preprocess(glyphs_path):
        """Return Glyphs source with illegal glyph/class names changed."""

        with open(glyphs_path) as fp:
            text = fp.read()
        names = set(re.findall('\n(?:glyph)?name = "(.+-.+)";\n', text))

        if names:
            num_names = len(names)
            printed_names = sorted(names)[:5]
            if num_names > 5:
                printed_names.append('...')
            print('Found %s glyph names containing hyphens: %s' % (
                num_names, ', '.join(printed_names)))
            print('Replacing all hyphens with periods.')

        for old_name in names:
            new_name = old_name.replace('-', '.')
            text = text.replace(old_name, new_name)
        return text

    def __init__(self, timing=False):
        if timing:
            configLogger(logger=timer.logger, level=logging.DEBUG)

    @timer()
    def build_ufos(
            self, glyphs_path, interpolate=False, masters_as_instances=False,
            family_name=None):
        """Build UFOs from Glyphs source."""

        master_dir = self._output_dir('ufo')
        instance_dir = self._output_dir('ufo', is_instance=True)
        ufos = []
        if not interpolate or masters_as_instances:
            ufos.extend(build_masters(
                glyphs_path, master_dir, designspace_instance_dir=instance_dir,
                family_name=family_name))
        if interpolate:
            ufos.extend(build_instances(
                glyphs_path, master_dir, instance_dir, family_name=family_name))
        return ufos

    @timer()
    def remove_overlaps(self, ufos):
        """Remove overlaps in UFOs' glyphs' contours."""
        from booleanOperations import BooleanOperationManager

        manager = BooleanOperationManager()
        for ufo in ufos:
            print('>> Removing overlaps for ' + self._font_name(ufo))
            for glyph in ufo:
                contours = list(glyph)
                glyph.clearContours()
                manager.union(contours, glyph.getPointPen())

    @timer()
    def decompose_glyphs(self, ufos):
        """Move components of UFOs' glyphs to their outlines."""

        for ufo in ufos:
            print('>> Decomposing glyphs for ' + self._font_name(ufo))
            for glyph in ufo:
                self._deep_copy_contours(ufo, glyph, glyph, Transform())
                glyph.clearComponents()

    def _deep_copy_contours(self, ufo, parent, component, transformation):
        """Copy contours from component to parent, including nested components."""

        for nested in component.components:
            self._deep_copy_contours(
                ufo, parent, ufo[nested.baseGlyph],
                transformation.transform(nested.transformation))

        if component != parent:
            pen = TransformPen(parent.getPen(), transformation)

            # if the transformation has a negative determinant, it will reverse
            # the contour direction of the component
            xx, xy, yx, yy = transformation[:4]
            if xx * yy - xy * yx < 0:
                pen = ReverseContourPen(pen)

            component.draw(pen)

    @timer()
    def convert_curves(self, ufos, compatible=False, reverse_direction=True):
        if compatible:
            fonts_to_quadratic(ufos, reverse_direction=reverse_direction,
                               dump_stats=True)
        else:
            for ufo in ufos:
                print('>> Converting curves for ' + self._font_name(ufo))
                font_to_quadratic(ufo, reverse_direction=reverse_direction,
                                  dump_stats=True)

    def build_otfs(self, ufos, remove_overlaps=True, **kwargs):
        """Build OpenType binaries with CFF outlines."""

        print('\n>> Building OTFs')

        self.decompose_glyphs(ufos)
        if remove_overlaps:
            self.remove_overlaps(ufos)
        del kwargs['reverse_direction']
        self.save_otfs(ufos, **kwargs)

    def build_ttfs(self, ufos, remove_overlaps=True, **kwargs):
        """Build OpenType binaries with TrueType outlines."""

        print('\n>> Building TTFs')

        if remove_overlaps:
            self.remove_overlaps(ufos)
        reverse_direction = kwargs.pop('reverse_direction')
        self.convert_curves(ufos, reverse_direction=reverse_direction)
        self.save_otfs(ufos, ttf=True, **kwargs)

    def build_interpolatable_ttfs(self, ufos, **kwargs):
        """Build OpenType binaries with interpolatable TrueType outlines."""

        print('\n>> Building interpolation-compatible TTFs')

        reverse_direction = kwargs.pop('reverse_direction')
        self.convert_curves(ufos, compatible=True,
                            reverse_direction=reverse_direction)
        self.save_otfs(ufos, ttf=True, interpolatable=True, **kwargs)

    @timer()
    def save_otfs(
            self, ufos, ttf=False, interpolatable=False, mti_paths=None,
            is_instance=False, use_afdko=False, autohint=None, subset=True,
            use_production_names=None):
        """Write OpenType binaries."""

        ext = 'ttf' if ttf else 'otf'
        fea_compiler = FDKFeatureCompiler if use_afdko else FeatureOTFCompiler
        otf_compiler = compileTTF if ttf else compileOTF

        for ufo in ufos:
            name = self._font_name(ufo)
            print('>> Saving %s for %s' % (ext.upper(), name))

            otf_path = self._output_path(ufo, ext, is_instance, interpolatable)
            if use_production_names is None:
                use_production_names = not ufo.lib.get(
                    GLYPHS_PREFIX + "Don't use Production Names")
            otf = otf_compiler(
                ufo, featureCompilerClass=fea_compiler,
                mtiFeaFiles=mti_paths[name] if mti_paths is not None else None,
                glyphOrder=ufo.lib[PUBLIC_PREFIX + 'glyphOrder'],
                useProductionNames=use_production_names)
            otf.save(otf_path)

            if subset:
                self.subset_otf_from_ufo(otf_path, ufo)

            if ttf and autohint is not None:
                hinted_otf_path = self._output_path(
                    ufo, ext, is_instance, interpolatable, autohinted=True)
                ttfautohint(otf_path, hinted_otf_path, args=autohint)

    def subset_otf_from_ufo(self, otf_path, ufo):
        """Subset a font using export flags set by glyphsLib."""

        keep_glyphs = set(ufo.lib.get(GLYPHS_PREFIX + 'Keep Glyphs', []))

        include = []
        ufo_order = [glyph_name
                     for glyph_name in ufo.lib[PUBLIC_PREFIX + 'glyphOrder']
                     if glyph_name in ufo]
        for old_name, new_name in zip(
                ufo_order,
                TTFont(otf_path).getGlyphOrder()):
            glyph = ufo[old_name]
            if ((keep_glyphs and old_name not in keep_glyphs) or
                not glyph.lib.get(GLYPHS_PREFIX + 'Glyphs.Export', True)):
                continue
            include.append(new_name)

        # copied from nototools.subset
        opt = subset.Options()
        opt.name_IDs = ['*']
        opt.name_legacy = True
        opt.name_languages = ['*']
        opt.layout_features = ['*']
        opt.notdef_outline = True
        opt.recalc_bounds = True
        opt.recalc_timestamp = True
        opt.canonical_order = True

        opt.glyph_names = True

        font = subset.load_font(otf_path, opt, lazy=False)
        subsetter = subset.Subsetter(options=opt)
        subsetter.populate(glyphs=include)
        subsetter.subset(font)
        subset.save_font(font, otf_path, opt)

    def run_from_glyphs(
            self, glyphs_path, preprocess=True, interpolate=False,
            masters_as_instances=False, family_name=None, **kwargs):
        """Run toolchain from Glyphs source to OpenType binaries."""

        if preprocess:
            print('>> Checking Glyphs source for illegal glyph names')
            glyphs_source = self.preprocess(glyphs_path)
            tmp_glyphs_file = tempfile.NamedTemporaryFile()
            glyphs_path = tmp_glyphs_file.name
            tmp_glyphs_file.write(glyphs_source)
            tmp_glyphs_file.seek(0)

        print('>> Building UFOs from Glyphs source')
        ufos = self.build_ufos(
            glyphs_path, interpolate, masters_as_instances, family_name)
        self.run_from_ufos(
            ufos, is_instance=(interpolate or masters_as_instances), **kwargs)

    def run_from_designspace(
            self, designspace_path, interpolate=False,
            masters_as_instances=False, **kwargs):
        """Run toolchain from a MutatorMath design space document to OpenType
        binaries.
        """

        ufos = []
        if not interpolate or masters_as_instances:
            reader = DesignSpaceDocumentReader(designspace_path, ufoVersion=3)
            ufos.extend(reader.getSourcePaths())
        if interpolate:
            print('>> Interpolating master UFOs from design space')
            results = build_designspace(
                designspace_path, outputUFOFormatVersion=3)
            for result in results:
                ufos.extend(result.values())
        self.run_from_ufos(
            ufos, is_instance=(interpolate or masters_as_instances), **kwargs)

    def run_from_ufos(self, ufos, output=(), mti_source=None, **kwargs):
        """Run toolchain from UFO sources to OpenType binaries."""

        if set(output) == set(['ufo']):
            return
        if hasattr(ufos[0], 'path'):
            ufo_paths = [ufo.path for ufo in ufos]
        else:
            if isinstance(ufos, str):
                ufo_paths = glob.glob(ufos)
            else:
                ufo_paths = ufos
            ufos = [Font(path) for path in ufo_paths]

        mti_paths = None
        if mti_source:
            mti_paths = {}
            mti_paths = plistlib.readPlist(mti_source)
            src_dir = os.path.dirname(mti_source)
            for paths in mti_paths.values():
                for tag in paths.keys():
                    paths[tag] = os.path.join(src_dir, paths[tag])

        need_reload = False
        if 'otf' in output:
            self.build_otfs(ufos, mti_paths=mti_paths, **kwargs)
            need_reload = True

        if 'ttf' in output:
            if need_reload:
                ufos = [Font(path) for path in ufo_paths]
            self.build_ttfs(ufos, mti_paths=mti_paths, **kwargs)
            need_reload = True

        if 'ttf-interpolatable' in output:
            if need_reload:
                ufos = [Font(path) for path in ufo_paths]
            self.build_interpolatable_ttfs(ufos, mti_paths=mti_paths, **kwargs)

    def _font_name(self, ufo):
        return '%s-%s' % (ufo.info.familyName.replace(' ', ''),
                          ufo.info.styleName.replace(' ', ''))

    def _output_dir(self, ext, is_instance=False, interpolatable=False,
                    autohinted=False):
        """Generate an output directory."""

        dir_prefix = 'instance_' if is_instance else 'master_'
        dir_suffix = '_interpolatable' if interpolatable else ''
        output_dir = dir_prefix + ext + dir_suffix
        if autohinted:
            output_dir = os.path.join('autohinted', output_dir)
        return output_dir

    def _output_path(self, ufo, ext, is_instance=False, interpolatable=False,
                     autohinted=False):
        """Generate output path for a font file with given extension."""

        out_dir = self._output_dir(ext, is_instance, interpolatable, autohinted)
        if not os.path.exists(out_dir):
            os.makedirs(out_dir)
        return os.path.join(out_dir, '%s.%s' % (self._font_name(ufo), ext))

class FDKFeatureCompiler(FeatureOTFCompiler):
    """An OTF compiler which uses the AFDKO to compile feature syntax."""

    def setupFile_featureTables(self):
        if self.mtiFeaFiles is not None:
            super(FDKFeatureCompiler, self).setupFile_featureTables()

        elif not self.features.strip():
            return

        import subprocess
        from fontTools.misc.py23 import tostr

        fd, outline_path = tempfile.mkstemp()
        os.close(fd)
        self.outline.save(outline_path)

        fd, feasrc_path = tempfile.mkstemp()
        os.close(fd)

        fd, fea_path = tempfile.mkstemp()
        os.write(fd, self.features)
        os.close(fd)

        report = tostr(subprocess.check_output([
            "makeotf", "-o", feasrc_path, "-f", outline_path,
            "-ff", fea_path]))
        os.remove(outline_path)
        os.remove(fea_path)

        print(report)
        success = "Done." in report
        if success:
            feasrc = TTFont(feasrc_path)
            for table in ["GDEF", "GPOS", "GSUB"]:
                if table in feasrc:
                    self.outline[table] = feasrc[table]
            feasrc.close()

        os.remove(feasrc_path)
        if not success:
            raise ValueError("Feature syntax compilation failed.")
<|endoftext|>


<commit_before># Copyright 2015 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import re
from time import time

from booleanOperations import BooleanOperationManager
from cu2qu.rf import fonts_to_quadratic
from glyphs2ufo.glyphslib import build_masters, build_instances
from robofab.world import OpenFont
from ufo2ft import compileOTF, compileTTF

class FontProject:
    """Provides methods for building fonts."""

    def preprocess(self, glyphs_path):
        """Return Glyphs source with illegal glyph names changed."""

        with open(glyphs_path) as fp:
            text = fp.read()
        names = re.findall('\nglyphname = "(.+-.+)";\n', text)
        for old_name in names:
            new_name = old_name.replace('-', '_')
            print('Found illegal glyph name "%s", replacing all instances in '
                  'source with "%s".' % (old_name, new_name))
            text = text.replace(old_name, new_name)
        return text

    def build_masters(self, glyphs_path, is_italic=False):
        """Build master UFOs from Glyphs source."""

        master_dir = self._output_dir('ufo')
        return build_masters(glyphs_path, master_dir, is_italic)

    def build_instances(self, glyphs_path, is_italic=False):
        """Build instance UFOs from Glyphs source."""

        master_dir = self._output_dir('ufo')
        instance_dir = self._output_dir('ufo', is_instance=True)
        return build_instances(glyphs_path, master_dir, instance_dir, is_italic)

    def remove_overlaps(self, ufo):
        """Remove overlaps in a UFO's glyphs' contours."""

        for glyph in ufo:
            manager = BooleanOperationManager()
            contours = glyph.contours
            glyph.clearContours()
            manager.union(contours, glyph.getPointPen())

    def save_otf(self, ufo, is_instance=False):
        """Build OTF from UFO."""

        otf_path = self._output_path(ufo, 'otf', is_instance)
        otf = compileOTF(ufo)
        otf.save(otf_path)

    def save_ttf(self, ufo, is_instance=False):
        """Build TTF from UFO."""

        ttf_path = self._output_path(ufo, 'ttf', is_instance)
        ttf = compileTTF(ufo)
        ttf.save(ttf_path)

    def run_all(
        self, glyphs_path, interpolate=False, compatible=False,
        remove_overlaps=True, preprocess=True):
        """Run toolchain from Glyphs source to OpenType binaries."""

        is_italic = 'Italic' in glyphs_path

        if preprocess:
            print '>> Checking Glyphs source for illegal glyph names'
            glyphs_source = self.preprocess(glyphs_path)
            glyphs_path = 'tmp.glyphs'
            with open(glyphs_path, 'w') as fp:
                fp.write(glyphs_source)

        if interpolate:
            print '>> Interpolating master UFOs from Glyphs source'
            ufos = self.build_instances(glyphs_path, is_italic)
        else:
            print '>> Loading master UFOs from Glyphs source'
            ufos = self.build_masters(glyphs_path, is_italic)

        if preprocess:
            os.remove(glyphs_path)

        if remove_overlaps and not compatible:
            for ufo in ufos:
                print '>> Removing overlaps for ' + ufo.info.postscriptFullName
                self.remove_overlaps(ufo)

        for ufo in ufos:
            print '>> Saving OTF for ' + ufo.info.postscriptFullName
            self.save_otf(ufo, is_instance=interpolate)

        start_t = time()
        if compatible:
            print '>> Converting curves to quadratic'
            fonts_to_quadratic(*ufos, dump_report=True)
        else:
            for ufo in ufos:
                print '>> Converting curves for ' + ufo.info.postscriptFullName
                fonts_to_quadratic(ufo, dump_report=True)
        t = time() - start_t
        print '[took %f seconds]' % t

        for ufo in ufos:
            print '>> Saving TTF for ' + ufo.info.postscriptFullName
            self.save_ttf(ufo, is_instance=interpolate)

    def _output_dir(self, ext, is_instance=False):
        """Generate an output directory."""

        dir_prefix = 'instance_' if is_instance else 'master_'
        return os.path.join(dir_prefix + ext)

    def _output_path(self, ufo, ext, is_instance=False):
        """Generate output path for a UFO with given directory and extension."""

        family = ufo.info.familyName.replace(' ', '')
        style = ufo.info.styleName.replace(' ', '')
        out_dir = self._output_dir(ext, is_instance)
        if not os.path.exists(out_dir):
            os.makedirs(out_dir)
        return os.path.join(out_dir, '%s-%s.%s' % (family, style, ext))
<commit_msg>Update for upstream changes to cu2qu API<commit_after># Copyright 2015 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import re
from time import time

from booleanOperations import BooleanOperationManager
from cu2qu.rf import fonts_to_quadratic
from glyphs2ufo.glyphslib import build_masters, build_instances
from robofab.world import OpenFont
from ufo2ft import compileOTF, compileTTF

class FontProject:
    """Provides methods for building fonts."""

    def preprocess(self, glyphs_path):
        """Return Glyphs source with illegal glyph names changed."""

        with open(glyphs_path) as fp:
            text = fp.read()
        names = re.findall('\nglyphname = "(.+-.+)";\n', text)
        for old_name in names:
            new_name = old_name.replace('-', '_')
            print('Found illegal glyph name "%s", replacing all instances in '
                  'source with "%s".' % (old_name, new_name))
            text = text.replace(old_name, new_name)
        return text

    def build_masters(self, glyphs_path, is_italic=False):
        """Build master UFOs from Glyphs source."""

        master_dir = self._output_dir('ufo')
        return build_masters(glyphs_path, master_dir, is_italic)

    def build_instances(self, glyphs_path, is_italic=False):
        """Build instance UFOs from Glyphs source."""

        master_dir = self._output_dir('ufo')
        instance_dir = self._output_dir('ufo', is_instance=True)
        return build_instances(glyphs_path, master_dir, instance_dir, is_italic)

    def remove_overlaps(self, ufo):
        """Remove overlaps in a UFO's glyphs' contours."""

        for glyph in ufo:
            manager = BooleanOperationManager()
            contours = glyph.contours
            glyph.clearContours()
            manager.union(contours, glyph.getPointPen())

    def save_otf(self, ufo, is_instance=False):
        """Build OTF from UFO."""

        otf_path = self._output_path(ufo, 'otf', is_instance)
        otf = compileOTF(ufo)
        otf.save(otf_path)

    def save_ttf(self, ufo, is_instance=False):
        """Build TTF from UFO."""

        ttf_path = self._output_path(ufo, 'ttf', is_instance)
        ttf = compileTTF(ufo)
        ttf.save(ttf_path)

    def run_all(
        self, glyphs_path, interpolate=False, compatible=False,
        remove_overlaps=True, preprocess=True):
        """Run toolchain from Glyphs source to OpenType binaries."""

        is_italic = 'Italic' in glyphs_path

        if preprocess:
            print '>> Checking Glyphs source for illegal glyph names'
            glyphs_source = self.preprocess(glyphs_path)
            glyphs_path = 'tmp.glyphs'
            with open(glyphs_path, 'w') as fp:
                fp.write(glyphs_source)

        if interpolate:
            print '>> Interpolating master UFOs from Glyphs source'
            ufos = self.build_instances(glyphs_path, is_italic)
        else:
            print '>> Loading master UFOs from Glyphs source'
            ufos = self.build_masters(glyphs_path, is_italic)

        if preprocess:
            os.remove(glyphs_path)

        if remove_overlaps and not compatible:
            for ufo in ufos:
                print '>> Removing overlaps for ' + ufo.info.postscriptFullName
                self.remove_overlaps(ufo)

        for ufo in ufos:
            print '>> Saving OTF for ' + ufo.info.postscriptFullName
            self.save_otf(ufo, is_instance=interpolate)

        start_t = time()
        if compatible:
            print '>> Converting curves to quadratic'
            fonts_to_quadratic(ufos, dump_stats=True)
        else:
            for ufo in ufos:
                print '>> Converting curves for ' + ufo.info.postscriptFullName
                fonts_to_quadratic([ufo], dump_stats=True)
        t = time() - start_t
        print '[took %f seconds]' % t

        for ufo in ufos:
            print '>> Saving TTF for ' + ufo.info.postscriptFullName
            self.save_ttf(ufo, is_instance=interpolate)

    def _output_dir(self, ext, is_instance=False):
        """Generate an output directory."""

        dir_prefix = 'instance_' if is_instance else 'master_'
        return os.path.join(dir_prefix + ext)

    def _output_path(self, ufo, ext, is_instance=False):
        """Generate output path for a UFO with given directory and extension."""

        family = ufo.info.familyName.replace(' ', '')
        style = ufo.info.styleName.replace(' ', '')
        out_dir = self._output_dir(ext, is_instance)
        if not os.path.exists(out_dir):
            os.makedirs(out_dir)
        return os.path.join(out_dir, '%s-%s.%s' % (family, style, ext))
<|endoftext|>


<commit_before>import gzip
import os
import struct
import numpy as np

try:
    from urllib.request import urlretrieve
except ImportError:
    from urllib import urlretrieve

import cntk
from cntk.layers import Convolution, MaxPooling, Dropout, Dense

'''
-----------------------------------------------------
Functions to load or download MNIST images and unpack
into training and testing sets.
-----------------------------------------------------
'''
# loading data from local path if possible. Otherwise download from online sources
def load_or_download_mnist_files(filename, num_samples, local_data_dir):

    if (local_data_dir):
        local_path = os.path.join(local_data_dir, filename)
    else:
        local_path = os.path.join(os.getcwd(), filename)

    if os.path.exists(local_path):
        gzfname = local_path
    else:
        local_data_dir = os.path.dirname(local_path)
        if not os.path.exists(local_data_dir):
            os.makedirs(local_data_dir)
        filename = "http://yann.lecun.com/exdb/mnist/" + filename
        print ("Downloading from" + filename, end=" ")
        gzfname, h = urlretrieve(filename, local_path)
        print ("[Done]")

    return gzfname

def get_mnist_data(filename, num_samples, local_data_dir):

    gzfname = load_or_download_mnist_files(filename, num_samples, local_data_dir)

    with gzip.open(gzfname) as gz:
        n = struct.unpack('I', gz.read(4))
        # Read magic number.
        if n[0] != 0x3080000:
            raise Exception('Invalid file: unexpected magic number.')
        # Read number of entries.
        n = struct.unpack('>I', gz.read(4))[0]
        if n != num_samples:
            raise Exception('Invalid file: expected {0} entries.'.format(num_samples))
        crow = struct.unpack('>I', gz.read(4))[0]
        ccol = struct.unpack('>I', gz.read(4))[0]
        if crow != 28 or ccol != 28:
            raise Exception('Invalid file: expected 28 rows/cols per image.')
        # Read data.
        res = np.fromstring(gz.read(num_samples * crow * ccol), dtype = np.uint8)

        return res.reshape((num_samples, crow * ccol))

# loading labels from local path if possible. Otherwise download from online sources
def get_mnist_labels(filename, num_samples, local_data_dir):

    gzfname = load_or_download_mnist_files(filename, num_samples, local_data_dir)

    with gzip.open(gzfname) as gz:
        n = struct.unpack('I', gz.read(4))
        # Read magic number.
        if n[0] != 0x1080000:
            raise Exception('Invalid file: unexpected magic number.')
        # Read number of entries.
        n = struct.unpack('>I', gz.read(4))
        if n[0] != num_samples:
            raise Exception('Invalid file: expected {0} rows.'.format(num_samples))
        # Read labels.
        res = np.fromstring(gz.read(num_samples), dtype = np.uint8)

        return res.reshape((num_samples, 1))

# loading mnist data and labels
def load_mnist_data(data_filename, labels_filename, number_samples, local_data_dir=None):
    data = get_mnist_data(data_filename, number_samples, local_data_dir)
    labels = get_mnist_labels(labels_filename, number_samples, local_data_dir)
    return np.hstack((data, labels))

# Save the data files into a format compatible with CNTK text reader
def save_as_txt(filename, ndarray):
    dir = os.path.dirname(filename)

    if not os.path.exists(dir):
        os.makedirs(dir)

    if not os.path.isfile(filename):
        print("Saving to ", filename, end=" ")
        with open(filename, 'w') as f:
            labels = list(map(' '.join, np.eye(10, dtype=np.uint).astype(str)))
            for row in ndarray:
                row_str = row.astype(str)
                label_str = labels[row[-1]]
                feature_str = ' '.join(row_str[:-1])
                f.write('|labels {} |features {}\n'.format(label_str, feature_str))
    else:
        print("File already exists", filename)

'''
--------------------------------------------------
Retrieve and process the training and testing data
--------------------------------------------------
'''
# Ensure we always get the same amount of randomness
np.random.seed(0)

# Define the data dimensions
image_shape = (1, 28, 28)
input_dim = int(np.prod(image_shape, dtype=int))
output_dim = 10

num_train_samples = 60000
num_test_samples = 10000

# The local path where the training and test data might be found or will be downloaded to.
training_data_path = os.path.join(os.getcwd(), "MNIST_data", "Train-28x28_cntk_text.txt")
testing_data_path = os.path.join(os.getcwd(), "MNIST_data", "Test-28x28_cntk_text.txt")

# Download the data if they don't already exist
if not os.path.exists(training_data_path):

    url_train_image = "train-images-idx3-ubyte.gz"
    url_train_labels = "train-labels-idx1-ubyte.gz"

    print("Loading training data")
    saved_data_dir = os.path.join(os.getcwd(), "MNIST_data")
    train = load_mnist_data(url_train_image, url_train_labels, num_train_samples, local_data_dir=saved_data_dir)

    print ("Writing training data text file...")
    save_as_txt(training_data_path, train)
    print("[Done]")

if not os.path.exists(testing_data_path):

    url_test_image = "t10k-images-idx3-ubyte.gz"
    url_test_labels = "t10k-labels-idx1-ubyte.gz"

    print("Loading testing data")
    saved_data_dir = os.path.join(os.getcwd(), "MNIST_data2")
    test = load_mnist_data(url_test_image, url_test_labels, num_test_samples, saved_data_dir)

    print ("Writing testing data text file...")
    save_as_txt(testing_data_path, test)
    print("[Done]")

feature_stream_name = 'features'
labels_stream_name = 'labels'

# Convert to CNTK MinibatchSource
train_minibatch_source = cntk.text_format_minibatch_source(training_data_path, [
    cntk.StreamConfiguration(feature_stream_name, input_dim),
    cntk.StreamConfiguration(labels_stream_name, output_dim)])
training_features = train_minibatch_source[feature_stream_name]
training_labels = train_minibatch_source[labels_stream_name]

print("Training data from file %s successfully read." % training_data_path)

test_minibatch_source = cntk.text_format_minibatch_source(testing_data_path, [
    cntk.StreamConfiguration(feature_stream_name, input_dim),
    cntk.StreamConfiguration(labels_stream_name, output_dim)])
test_features = test_minibatch_source[feature_stream_name]
test_labels = test_minibatch_source[labels_stream_name]

print("Test data from file %s successfully read." % testing_data_path)

'''
---------------------------------------------
Constructing the Convolutional Neural Network
---------------------------------------------
'''
def create_convolutional_neural_network(input_vars, out_dims, dropout_prob=0.0):

    convolutional_layer_1 = Convolution((5, 5), 32, strides=1, activation=cntk.ops.relu, pad=True)(input_vars)
    pooling_layer_1 = MaxPooling((2, 2), strides=(2, 2))(convolutional_layer_1)

    convolutional_layer_2 = Convolution((5, 5), 64, strides=1, activation=cntk.ops.relu, pad=True)(pooling_layer_1)
    pooling_layer_2 = MaxPooling((2, 2), strides=(2, 2))(convolutional_layer_2)

    fully_connected_layer = Dense(1024)(pooling_layer_2)
    dropout_layer = Dropout(dropout_prob)(fully_connected_layer)
    output_layer = Dense(out_dims, activation=None)(dropout_layer)

    return output_layer

# Define the input to the neural network
input_vars = cntk.ops.input_variable(image_shape, np.float32)

# Create the convolutional neural network
output = create_convolutional_neural_network(input_vars, output_dim, dropout_prob=0.5)

'''
----------------------
Setting up the trainer
----------------------
'''
# Define the label as the other input parameter of the trainer
labels = cntk.ops.input_variable(output_dim, np.float32)

#Initialize the parameters for the trainer
train_minibatch_size = 100
learning_rate = 1e-4
momentum = 0.9 ** (1.0 / train_minibatch_size)

# Define the loss function
loss = cntk.ops.cross_entropy_with_softmax(output, labels)

# Define the function that calculates classification error
label_error = cntk.ops.classification_error(output, labels)

# Instantiate the trainer object to drive the model training
learner = cntk.adam_sgd(output.parameters, learning_rate, momentum)
trainer = cntk.Trainer(output, loss, label_error, [learner])

'''
-----------------------------------------
Training the Convolutional Neural Network
-----------------------------------------
'''
training_progress_output_freq = 10

sample_count = 0
num_minibatch = 0

# loop over minibatches in the epoch
while sample_count < num_train_samples:

    minibatch = train_minibatch_source.next_minibatch(min(train_minibatch_size, num_train_samples - sample_count))

    # Specify the mapping of input variables in the model to actual minibatch data to be trained with
    data = {input_vars: minibatch[training_features],
            labels: minibatch[training_labels]}
    trainer.train_minibatch(data)

    sample_count += data[labels].num_samples
    num_minibatch += 1

    # Print the training progress data
    if num_minibatch % training_progress_output_freq == 0:
        training_loss = cntk.get_train_loss(trainer)
        eval_error = cntk.get_train_eval_criterion(trainer)
        print("# of Samples: %6d  |  Loss: %.6f  |  Error: %.6f" % (sample_count, training_loss, eval_error))

print("Training Completed.", end="\n\n")

'''
-------------------
Classification Test
--------------------
'''
test_minibatch_size = 1000

sample_count = 0
test_results = []

while sample_count < num_test_samples:

    minibatch = test_minibatch_source.next_minibatch(min(test_minibatch_size, num_test_samples - sample_count))

    # Specify the mapping of input variables in the model to actual minibatch data to be tested with
    data = {input_vars: minibatch[test_features],
            labels: minibatch[test_labels]}
    eval_error = trainer.test_minibatch(data)
    test_results.append(eval_error)

    sample_count += data[labels].num_samples

# Printing the average of evaluation errors of all test minibatches
print("Average errors of all test minibatches: %.6f%%" % np.mean(test_results, axis=0, dtype=float))
<commit_msg>multipy the error by 100 to get the percentage.<commit_after>import gzip
import os
import struct
import numpy as np

try:
    from urllib.request import urlretrieve
except ImportError:
    from urllib import urlretrieve

import cntk
from cntk.layers import Convolution, MaxPooling, Dropout, Dense

'''
-----------------------------------------------------
Functions to load or download MNIST images and unpack
into training and testing sets.
-----------------------------------------------------
'''
# loading data from local path if possible. Otherwise download from online sources
def load_or_download_mnist_files(filename, num_samples, local_data_dir):

    if (local_data_dir):
        local_path = os.path.join(local_data_dir, filename)
    else:
        local_path = os.path.join(os.getcwd(), filename)

    if os.path.exists(local_path):
        gzfname = local_path
    else:
        local_data_dir = os.path.dirname(local_path)
        if not os.path.exists(local_data_dir):
            os.makedirs(local_data_dir)
        filename = "http://yann.lecun.com/exdb/mnist/" + filename
        print ("Downloading from" + filename, end=" ")
        gzfname, h = urlretrieve(filename, local_path)
        print ("[Done]")

    return gzfname

def get_mnist_data(filename, num_samples, local_data_dir):

    gzfname = load_or_download_mnist_files(filename, num_samples, local_data_dir)

    with gzip.open(gzfname) as gz:
        n = struct.unpack('I', gz.read(4))
        # Read magic number.
        if n[0] != 0x3080000:
            raise Exception('Invalid file: unexpected magic number.')
        # Read number of entries.
        n = struct.unpack('>I', gz.read(4))[0]
        if n != num_samples:
            raise Exception('Invalid file: expected {0} entries.'.format(num_samples))
        crow = struct.unpack('>I', gz.read(4))[0]
        ccol = struct.unpack('>I', gz.read(4))[0]
        if crow != 28 or ccol != 28:
            raise Exception('Invalid file: expected 28 rows/cols per image.')
        # Read data.
        res = np.fromstring(gz.read(num_samples * crow * ccol), dtype = np.uint8)

        return res.reshape((num_samples, crow * ccol))

# loading labels from local path if possible. Otherwise download from online sources
def get_mnist_labels(filename, num_samples, local_data_dir):

    gzfname = load_or_download_mnist_files(filename, num_samples, local_data_dir)

    with gzip.open(gzfname) as gz:
        n = struct.unpack('I', gz.read(4))
        # Read magic number.
        if n[0] != 0x1080000:
            raise Exception('Invalid file: unexpected magic number.')
        # Read number of entries.
        n = struct.unpack('>I', gz.read(4))
        if n[0] != num_samples:
            raise Exception('Invalid file: expected {0} rows.'.format(num_samples))
        # Read labels.
        res = np.fromstring(gz.read(num_samples), dtype = np.uint8)

        return res.reshape((num_samples, 1))

# loading mnist data and labels
def load_mnist_data(data_filename, labels_filename, number_samples, local_data_dir=None):
    data = get_mnist_data(data_filename, number_samples, local_data_dir)
    labels = get_mnist_labels(labels_filename, number_samples, local_data_dir)
    return np.hstack((data, labels))

# Save the data files into a format compatible with CNTK text reader
def save_as_txt(filename, ndarray):
    dir = os.path.dirname(filename)

    if not os.path.exists(dir):
        os.makedirs(dir)

    if not os.path.isfile(filename):
        print("Saving to ", filename, end=" ")
        with open(filename, 'w') as f:
            labels = list(map(' '.join, np.eye(10, dtype=np.uint).astype(str)))
            for row in ndarray:
                row_str = row.astype(str)
                label_str = labels[row[-1]]
                feature_str = ' '.join(row_str[:-1])
                f.write('|labels {} |features {}\n'.format(label_str, feature_str))
    else:
        print("File already exists", filename)

'''
--------------------------------------------------
Retrieve and process the training and testing data
--------------------------------------------------
'''
# Ensure we always get the same amount of randomness
np.random.seed(0)

# Define the data dimensions
image_shape = (1, 28, 28)
input_dim = int(np.prod(image_shape, dtype=int))
output_dim = 10

num_train_samples = 60000
num_test_samples = 10000

# The local path where the training and test data might be found or will be downloaded to.
training_data_path = os.path.join(os.getcwd(), "MNIST_data", "Train-28x28_cntk_text.txt")
testing_data_path = os.path.join(os.getcwd(), "MNIST_data", "Test-28x28_cntk_text.txt")

# Download the data if they don't already exist
if not os.path.exists(training_data_path):

    url_train_image = "train-images-idx3-ubyte.gz"
    url_train_labels = "train-labels-idx1-ubyte.gz"

    print("Loading training data")
    saved_data_dir = os.path.join(os.getcwd(), "MNIST_data")
    train = load_mnist_data(url_train_image, url_train_labels, num_train_samples, local_data_dir=saved_data_dir)

    print ("Writing training data text file...")
    save_as_txt(training_data_path, train)
    print("[Done]")

if not os.path.exists(testing_data_path):

    url_test_image = "t10k-images-idx3-ubyte.gz"
    url_test_labels = "t10k-labels-idx1-ubyte.gz"

    print("Loading testing data")
    saved_data_dir = os.path.join(os.getcwd(), "MNIST_data2")
    test = load_mnist_data(url_test_image, url_test_labels, num_test_samples, saved_data_dir)

    print ("Writing testing data text file...")
    save_as_txt(testing_data_path, test)
    print("[Done]")

feature_stream_name = 'features'
labels_stream_name = 'labels'

# Convert to CNTK MinibatchSource
train_minibatch_source = cntk.text_format_minibatch_source(training_data_path, [
    cntk.StreamConfiguration(feature_stream_name, input_dim),
    cntk.StreamConfiguration(labels_stream_name, output_dim)])
training_features = train_minibatch_source[feature_stream_name]
training_labels = train_minibatch_source[labels_stream_name]

print("Training data from file %s successfully read." % training_data_path)

test_minibatch_source = cntk.text_format_minibatch_source(testing_data_path, [
    cntk.StreamConfiguration(feature_stream_name, input_dim),
    cntk.StreamConfiguration(labels_stream_name, output_dim)])
test_features = test_minibatch_source[feature_stream_name]
test_labels = test_minibatch_source[labels_stream_name]

print("Test data from file %s successfully read." % testing_data_path)

'''
---------------------------------------------
Constructing the Convolutional Neural Network
---------------------------------------------
'''
def create_convolutional_neural_network(input_vars, out_dims, dropout_prob=0.0):

    convolutional_layer_1 = Convolution((5, 5), 32, strides=1, activation=cntk.ops.relu, pad=True)(input_vars)
    pooling_layer_1 = MaxPooling((2, 2), strides=(2, 2))(convolutional_layer_1)

    convolutional_layer_2 = Convolution((5, 5), 64, strides=1, activation=cntk.ops.relu, pad=True)(pooling_layer_1)
    pooling_layer_2 = MaxPooling((2, 2), strides=(2, 2))(convolutional_layer_2)

    fully_connected_layer = Dense(1024)(pooling_layer_2)
    dropout_layer = Dropout(dropout_prob)(fully_connected_layer)
    output_layer = Dense(out_dims, activation=None)(dropout_layer)

    return output_layer

# Define the input to the neural network
input_vars = cntk.ops.input_variable(image_shape, np.float32)

# Create the convolutional neural network
output = create_convolutional_neural_network(input_vars, output_dim, dropout_prob=0.5)

'''
----------------------
Setting up the trainer
----------------------
'''
# Define the label as the other input parameter of the trainer
labels = cntk.ops.input_variable(output_dim, np.float32)

#Initialize the parameters for the trainer
train_minibatch_size = 100
learning_rate = 1e-4
momentum = 0.9 ** (1.0 / train_minibatch_size)

# Define the loss function
loss = cntk.ops.cross_entropy_with_softmax(output, labels)

# Define the function that calculates classification error
label_error = cntk.ops.classification_error(output, labels)

# Instantiate the trainer object to drive the model training
learner = cntk.adam_sgd(output.parameters, learning_rate, momentum)
trainer = cntk.Trainer(output, loss, label_error, [learner])

'''
-----------------------------------------
Training the Convolutional Neural Network
-----------------------------------------
'''
training_progress_output_freq = 10

sample_count = 0
num_minibatch = 0

# loop over minibatches in the epoch
while sample_count < num_train_samples:

    minibatch = train_minibatch_source.next_minibatch(min(train_minibatch_size, num_train_samples - sample_count))

    # Specify the mapping of input variables in the model to actual minibatch data to be trained with
    data = {input_vars: minibatch[training_features],
            labels: minibatch[training_labels]}
    trainer.train_minibatch(data)

    sample_count += data[labels].num_samples
    num_minibatch += 1

    # Print the training progress data
    if num_minibatch % training_progress_output_freq == 0:
        training_loss = cntk.get_train_loss(trainer)
        eval_error = cntk.get_train_eval_criterion(trainer)
        print("# of Samples: %6d  |  Loss: %.6f  |  Error: %.6f" % (sample_count, training_loss, eval_error))

print("Training Completed.", end="\n\n")

'''
-------------------
Classification Test
--------------------
'''
test_minibatch_size = 1000

sample_count = 0
test_results = []

while sample_count < num_test_samples:

    minibatch = test_minibatch_source.next_minibatch(min(test_minibatch_size, num_test_samples - sample_count))

    # Specify the mapping of input variables in the model to actual minibatch data to be tested with
    data = {input_vars: minibatch[test_features],
            labels: minibatch[test_labels]}
    eval_error = trainer.test_minibatch(data)
    test_results.append(eval_error)

    sample_count += data[labels].num_samples

# Printing the average of evaluation errors of all test minibatches
print("Average errors of all test minibatches: %.6f%%" % np.mean(test_results*100, axis=0, dtype=float))
<|endoftext|>