<commit_before>"""
Handles URL downloads.

This is the low-level interface for downloading interfaces, implementations, icons, etc.

@see: L{policy.Policy.begin_iface_download}
@see: L{policy.Policy.begin_archive_download}
@see: L{policy.Policy.begin_icon_download}
"""

# Copyright (C) 2006, Thomas Leonard
# See the README file for details, or visit http://0install.net.

import tempfile, os, sys
from zeroinstall import SafeException
import traceback
from logging import warn

download_starting = "starting"	# Waiting for UI to start it
download_fetching = "fetching"	# In progress
download_checking = "checking"	# Checking GPG sig (possibly interactive)
download_complete = "complete"	# Downloaded and cached OK
download_failed = "failed"

class DownloadError(SafeException):
	pass

class Download(object):
	__slots__ = ['url', 'tempfile', 'status', 'errors', 'expected_size',
		     'expected_size', 'child_pid', 'child_stderr', 'on_success']

	def __init__(self, url):
		"Initial status is starting."
		self.url = url
		self.status = download_starting
		self.on_success = []

		self.tempfile = None		# Stream for result
		self.errors = None

		self.expected_size = None	# Final size (excluding skipped bytes)

		self.child_pid = None
		self.child_stderr = None

	def start(self):
		"""Returns stderr stream from child. Call error_stream_closed() when
		it returns EOF."""
		assert self.status == download_starting
		self.tempfile = tempfile.TemporaryFile(prefix = 'injector-dl-data-')

		error_r, error_w = os.pipe()
		self.errors = ''

		self.child_pid = os.fork()
		if self.child_pid == 0:
			# We are the child
			try:
				os.close(error_r)
				os.dup2(error_w, 2)
				os.close(error_w)
				self.download_as_child()
			finally:
				os._exit(1)

		# We are the parent
		os.close(error_w)
		self.status = download_fetching
		return os.fdopen(error_r, 'r')

	def download_as_child(self):
		from urllib2 import urlopen, HTTPError, URLError
		try:
			import shutil
			#print "Child downloading", self.url
			if self.url.startswith('/'):
				if not os.path.isfile(self.url):
					print >>sys.stderr, "File '%s' does not " \
						"exist!" % self.url
					return
				src = file(self.url)
			elif self.url.startswith('http:') or self.url.startswith('ftp:'):
				src = urlopen(self.url)
			else:
				raise Exception('Unsupported URL protocol in: ' + self.url)

			shutil.copyfileobj(src, self.tempfile)
			self.tempfile.flush()

			os._exit(0)
		except (HTTPError, URLError), ex:
			print >>sys.stderr, "Error downloading '" + self.url + "': " + str(ex)
		except:
			traceback.print_exc()

	def error_stream_data(self, data):
		"""Passed with result of os.read(error_stream, n). Can be
		called multiple times, once for each read."""
		assert data
		assert self.status is download_fetching
		self.errors += data

	def error_stream_closed(self):
		"""Ends a download. Status changes from fetching to checking.
		Calls the on_success callbacks with the rewound data stream on success,
		or throws DownloadError on failure."""
		assert self.status is download_fetching
		assert self.tempfile is not None
		assert self.child_pid is not None

		pid, status = os.waitpid(self.child_pid, 0)
		assert pid == self.child_pid
		self.child_pid = None

		errors = self.errors
		self.errors = None

		if status and not errors:
			errors = 'Download process exited with error status ' \
				 'code ' + hex(status)

		stream = self.tempfile
		self.tempfile = None

		if errors:
			error = DownloadError(errors)
		else:
			error = None

		# Check that the download has the correct size, if we know what it should be.
		if self.expected_size is not None and not error:
			size = os.fstat(stream.fileno()).st_size
			if size != self.expected_size:
				error = SafeException('Downloaded archive has incorrect size.\n'
						'URL: %s\n'
						'Expected: %d bytes\n'
						'Received: %d bytes' % (self.url, self.expected_size, size))

		if error:
			self.status = download_failed
			self.on_success = []	# Break GC cycles
			raise error
		else:
			self.status = download_checking

		for x in self.on_success:
			stream.seek(0)
			x(stream)

	def abort(self):
		if self.child_pid is not None:
			warn("Killing download process %s", self.child_pid)
			import signal
			os.kill(self.child_pid, signal.SIGTERM)
		else:
			self.status = download_failed

	def get_current_fraction(self):
		"""Returns the current fraction of this download that has been fetched (from 0 to 1),
		or None if the total size isn't known."""
		if self.status is download_starting:
			return 0
		if self.tempfile is None:
			return 1
		if self.expected_size is None:
			return None		# Unknown
		current_size = os.fstat(self.tempfile.fileno()).st_size
		return float(current_size) / self.expected_size
<commit_msg>Cancelling a download only logs at info level, not warning.<commit_after>"""
Handles URL downloads.

This is the low-level interface for downloading interfaces, implementations, icons, etc.

@see: L{policy.Policy.begin_iface_download}
@see: L{policy.Policy.begin_archive_download}
@see: L{policy.Policy.begin_icon_download}
"""

# Copyright (C) 2006, Thomas Leonard
# See the README file for details, or visit http://0install.net.

import tempfile, os, sys
from zeroinstall import SafeException
import traceback
from logging import warn

download_starting = "starting"	# Waiting for UI to start it
download_fetching = "fetching"	# In progress
download_checking = "checking"	# Checking GPG sig (possibly interactive)
download_complete = "complete"	# Downloaded and cached OK
download_failed = "failed"

class DownloadError(SafeException):
	pass

class Download(object):
	__slots__ = ['url', 'tempfile', 'status', 'errors', 'expected_size',
		     'expected_size', 'child_pid', 'child_stderr', 'on_success']

	def __init__(self, url):
		"Initial status is starting."
		self.url = url
		self.status = download_starting
		self.on_success = []

		self.tempfile = None		# Stream for result
		self.errors = None

		self.expected_size = None	# Final size (excluding skipped bytes)

		self.child_pid = None
		self.child_stderr = None

	def start(self):
		"""Returns stderr stream from child. Call error_stream_closed() when
		it returns EOF."""
		assert self.status == download_starting
		self.tempfile = tempfile.TemporaryFile(prefix = 'injector-dl-data-')

		error_r, error_w = os.pipe()
		self.errors = ''

		self.child_pid = os.fork()
		if self.child_pid == 0:
			# We are the child
			try:
				os.close(error_r)
				os.dup2(error_w, 2)
				os.close(error_w)
				self.download_as_child()
			finally:
				os._exit(1)

		# We are the parent
		os.close(error_w)
		self.status = download_fetching
		return os.fdopen(error_r, 'r')

	def download_as_child(self):
		from urllib2 import urlopen, HTTPError, URLError
		try:
			import shutil
			#print "Child downloading", self.url
			if self.url.startswith('/'):
				if not os.path.isfile(self.url):
					print >>sys.stderr, "File '%s' does not " \
						"exist!" % self.url
					return
				src = file(self.url)
			elif self.url.startswith('http:') or self.url.startswith('ftp:'):
				src = urlopen(self.url)
			else:
				raise Exception('Unsupported URL protocol in: ' + self.url)

			shutil.copyfileobj(src, self.tempfile)
			self.tempfile.flush()

			os._exit(0)
		except (HTTPError, URLError), ex:
			print >>sys.stderr, "Error downloading '" + self.url + "': " + str(ex)
		except:
			traceback.print_exc()

	def error_stream_data(self, data):
		"""Passed with result of os.read(error_stream, n). Can be
		called multiple times, once for each read."""
		assert data
		assert self.status is download_fetching
		self.errors += data

	def error_stream_closed(self):
		"""Ends a download. Status changes from fetching to checking.
		Calls the on_success callbacks with the rewound data stream on success,
		or throws DownloadError on failure."""
		assert self.status is download_fetching
		assert self.tempfile is not None
		assert self.child_pid is not None

		pid, status = os.waitpid(self.child_pid, 0)
		assert pid == self.child_pid
		self.child_pid = None

		errors = self.errors
		self.errors = None

		if status and not errors:
			errors = 'Download process exited with error status ' \
				 'code ' + hex(status)

		stream = self.tempfile
		self.tempfile = None

		if errors:
			error = DownloadError(errors)
		else:
			error = None

		# Check that the download has the correct size, if we know what it should be.
		if self.expected_size is not None and not error:
			size = os.fstat(stream.fileno()).st_size
			if size != self.expected_size:
				error = SafeException('Downloaded archive has incorrect size.\n'
						'URL: %s\n'
						'Expected: %d bytes\n'
						'Received: %d bytes' % (self.url, self.expected_size, size))

		if error:
			self.status = download_failed
			self.on_success = []	# Break GC cycles
			raise error
		else:
			self.status = download_checking

		for x in self.on_success:
			stream.seek(0)
			x(stream)

	def abort(self):
		if self.child_pid is not None:
			info("Killing download process %s", self.child_pid)
			import signal
			os.kill(self.child_pid, signal.SIGTERM)
		else:
			self.status = download_failed

	def get_current_fraction(self):
		"""Returns the current fraction of this download that has been fetched (from 0 to 1),
		or None if the total size isn't known."""
		if self.status is download_starting:
			return 0
		if self.tempfile is None:
			return 1
		if self.expected_size is None:
			return None		# Unknown
		current_size = os.fstat(self.tempfile.fileno()).st_size
		return float(current_size) / self.expected_size
<|endoftext|>


<commit_before>"""Utilities and initialization code related to the Jinja2 templates
used by the benchmarking script.
"""

from jinja2 import Environment, FileSystemLoader
import numpy as np

# Create the Template environment
env = Environment(
    loader=FileSystemLoader(['templates', 'templates/sites'])
)

#------ Below are Custom Filters used for Formatting --------

def is_blank(val):
    try:
        return not np.isfinite(val)
    except:
        return True

def filter_blank(val):
    if is_blank(val): return ''
    return val

def filter_money(val, precision = 0):
    if is_blank(val): return ''
    format_string = "$ {:,.%sf}" % precision
    return format_string.format(val)

def filter_number(val, precision = 0):
    if is_blank(val): return ''
    format_string = "{:,.%sf}" % precision
    return format_string.format(val)

def filter_percent(val, precision = 1):
    if is_blank(val): return ''
    format_string = "{:.%s%%}" % precision
    return format_string.format(val)

def filter_only_string(val):
    'Return blank for anything but strings.'
    if type(val) is str:
        return val
    else:
        return ''

# -------------- Assign the custom filters to the Environment

env.filters['blank'] = filter_blank
env.filters['money'] = filter_money
env.filters['number'] = filter_number
env.filters['percent'] = filter_percent
env.filters['only_string'] = filter_only_string

# ------------- Utility Functions -----------------

def get_template(template_filename):
    """Returns a template having the name 'template_file_name'
    """
    return env.get_template(template_filename)
<commit_msg>Eliminated the blank filter as not needed anymore.<commit_after>"""Utilities and initialization code related to the Jinja2 templates
used by the benchmarking script.
"""

from jinja2 import Environment, FileSystemLoader
import numpy as np

# Create the Template environment
env = Environment(
    loader=FileSystemLoader(['templates', 'templates/sites'])
)

#------ Below are Custom Filters used for Formatting --------

def is_blank(val):
    try:
        return not np.isfinite(val)
    except:
        return True

def filter_money(val, precision = 0):
    if is_blank(val): return ''
    format_string = "$ {:,.%sf}" % precision
    return format_string.format(val)

def filter_number(val, precision = 0):
    if is_blank(val): return ''
    format_string = "{:,.%sf}" % precision
    return format_string.format(val)

def filter_percent(val, precision = 1):
    if is_blank(val): return ''
    format_string = "{:.%s%%}" % precision
    return format_string.format(val)

def filter_only_string(val):
    'Return blank for anything but strings.'
    if type(val) is str:
        return val
    else:
        return ''

# -------------- Assign the custom filters to the Environment

env.filters['money'] = filter_money
env.filters['number'] = filter_number
env.filters['percent'] = filter_percent
env.filters['only_string'] = filter_only_string

# ------------- Utility Functions -----------------

def get_template(template_filename):
    """Returns a template having the name 'template_file_name'
    """
    return env.get_template(template_filename)
<|endoftext|>


<commit_before>"""General functional tests for the API endpoints."""

from api.v2.tests.tools import SalAPITestCase, TestGeneratorMeta

# pylint: disable=missing-docstring

class BusinessUnitTest(SalAPITestCase):
    __metaclass__ = TestGeneratorMeta
    setup_data = ['create_business_unit_data']
    tests = ['businessunit']

class ConditionTest(SalAPITestCase):
    __metaclass__ = TestGeneratorMeta
    fixtures = [
        'business_unit_fixtures.json', 'machine_group_fixtures.json',
        'machine_fixtures.json', 'conditions_fixture.json']
    tests = ['condition']

class FactTest(SalAPITestCase):
    __metaclass__ = TestGeneratorMeta
    fixtures = [
        'business_unit_fixtures.json', 'machine_group_fixtures.json',
        'machine_fixtures.json', 'fact_fixtures.json']
    tests = ['fact']

class InventoryTest(SalAPITestCase):
    __metaclass__ = TestGeneratorMeta
    fixtures = [
        'business_unit_fixtures.json', 'machine_group_fixtures.json',
        'machine_fixtures.json', 'inventory_fixtures.json']
    tests = ['inventoryitem']

class MachineGroupTest(SalAPITestCase):
    __metaclass__ = TestGeneratorMeta
    setup_data = ['create_machine_data']
    tests = ['machinegroup']

class PendingAppleUpdateTest(SalAPITestCase):
    __metaclass__ = TestGeneratorMeta
    fixtures = [
        'business_unit_fixtures.json', 'machine_group_fixtures.json',
        'machine_fixtures.json', 'pending_apple_update_fixtures.json']
    tests = ['pendingappleupdate']

class PendingUpdateTest(SalAPITestCase):
    __metaclass__ = TestGeneratorMeta
    fixtures = [
        'business_unit_fixtures.json', 'machine_group_fixtures.json',
        'machine_fixtures.json', 'pending_update_fixtures.json']
    tests = ['pendingupdate']

class PluginScriptTest(SalAPITestCase):
    __metaclass__ = TestGeneratorMeta
    fixtures = [
        'business_unit_fixtures.json', 'machine_group_fixtures.json',
        'machine_fixtures.json', 'plugin_script_fixtures.json']
    tests = ['pluginscriptrow']
<commit_msg>Fix metaclass usage for py3.<commit_after>"""General functional tests for the API endpoints."""

from api.v2.tests.tools import SalAPITestCase, TestGeneratorMeta

# pylint: disable=missing-docstring

class BusinessUnitTest(SalAPITestCase, metaclass=TestGeneratorMeta):
    setup_data = ['create_business_unit_data']
    tests = ['businessunit']

class ConditionTest(SalAPITestCase, metaclass=TestGeneratorMeta):
    fixtures = [
        'business_unit_fixtures.json', 'machine_group_fixtures.json',
        'machine_fixtures.json', 'conditions_fixture.json']
    tests = ['condition']

class FactTest(SalAPITestCase, metaclass=TestGeneratorMeta):
    fixtures = [
        'business_unit_fixtures.json', 'machine_group_fixtures.json',
        'machine_fixtures.json', 'fact_fixtures.json']
    tests = ['fact']

class InventoryTest(SalAPITestCase, metaclass=TestGeneratorMeta):
    fixtures = [
        'business_unit_fixtures.json', 'machine_group_fixtures.json',
        'machine_fixtures.json', 'inventory_fixtures.json']
    tests = ['inventoryitem']

class MachineGroupTest(SalAPITestCase, metaclass=TestGeneratorMeta):
    setup_data = ['create_machine_data']
    tests = ['machinegroup']

class PendingAppleUpdateTest(SalAPITestCase, metaclass=TestGeneratorMeta):
    fixtures = [
        'business_unit_fixtures.json', 'machine_group_fixtures.json',
        'machine_fixtures.json', 'pending_apple_update_fixtures.json']
    tests = ['pendingappleupdate']

class PendingUpdateTest(SalAPITestCase, metaclass=TestGeneratorMeta):
    fixtures = [
        'business_unit_fixtures.json', 'machine_group_fixtures.json',
        'machine_fixtures.json', 'pending_update_fixtures.json']
    tests = ['pendingupdate']

class PluginScriptTest(SalAPITestCase, metaclass=TestGeneratorMeta):
    fixtures = [
        'business_unit_fixtures.json', 'machine_group_fixtures.json',
        'machine_fixtures.json', 'plugin_script_fixtures.json']
    tests = ['pluginscriptrow']
<|endoftext|>


<commit_before>import os
import ConfigParser

from sentence_parser import SentenceParser
from lexicon import Lexicon
from spreading_activation import SpreadingActivation
from definition_parser import read as read_defs
from construction import *
from matcher import *
from sup_dic import supplementary_dictionary_reader as sdreader
from avm import AVM

class Wrapper:
    def __init__(self, cf):
        self.cfn = cf
        self.__read_config()

        self.lexicon = Lexicon()
        self.__read_files()
        self.__add_constructions()

    def __read_config(self):
        machinepath = os.path.realpath(__file__).rsplit("/", 2)[0]
        if "MACHINEPATH" in os.environ:
            machinepath = os.environ["MACHINEPATH"]
        config = ConfigParser.SafeConfigParser({"machinepath": machinepath})
        config.read(self.cfn)
        items = dict(config.items("machine"))
        self.def_files = [(s.split(":")[0].strip(), int(s.split(":")[1]))
            for s in items["definitions"].split(",")]
        self.supp_dict_fn = items["supp_dict"]
        self.stations_fn = items["stations"]

    def __read_files(self):
        self.__read_definitions()
        self.__read_supp_dict()

    def __read_definitions(self):
        for file_name, printname_index in self.def_files:
            # TODO HACK makefile needed
            if (file_name.endswith("generated") and
                not os.path.exists(file_name)):
                raise Exception("A definition file that supposed to be " +
                    "generated by pymachine/scripts/generate_translation_dict.sh" +
                    " does not exist: {0}".format(file_name))
            definitions = read_defs(file(file_name), printname_index)
            logging.debug("{0}: {1}".format(file_name, definitions.keys()))
            self.lexicon.add_static(definitions.itervalues())

    def __read_supp_dict(self):
        self.supp_dict = sdreader(file(self.supp_dict_fn))

    def __add_constructions(self):
        # TODO this should be created automatically, maybe from knowing,
        # that "megy" is actually a verb, there is a POS in the definitions
        try:
            megy_construction = VerbConstruction("megy", self.lexicon.static[
                "megy"], self.supp_dict)
            del self.lexicon.static["megy"]
            #self.lexicon.add_construction(megy_construction)
        except KeyError:
            pass
        try:
            tesz_construction = VerbConstruction("tesz", self.lexicon.static[
                "tesz"], self.supp_dict)
            del self.lexicon.static["tesz"]
            self.lexicon.add_construction(tesz_construction)
        except KeyError:
            pass
        #self.lexicon.add_construction(ElviraConstruction())

        self.lexicon.add_construction(DummyNPConstruction())
        self.lexicon.add_construction(TheConstruction())
        self.lexicon.add_construction(MaxNP_InBetweenPostP_Construction())
        self.lexicon.add_construction(PostPConstruction())

        station_matcher = FileContainsMatcher(self.stations_fn)
        ic_name_matcher = EnumMatcher("ic_name", self.lexicon)
        src_matcher = AndMatcher(self.supp_dict["@HUN_GO_SRC"],
                                 station_matcher)
        tgt_matcher = AndMatcher(self.supp_dict["@HUN_GO_TGT"],
                                 station_matcher)
        ea = elvira_avm = AVM('ElviraAVM')
        ea.add_attribute("vonat", PrintnameMatcher("vonat"), AVM.RREQ, None)
        ea.add_attribute("menetrend", PrintnameMatcher("menetrend"), AVM.RREQ, None)
        ea.add_attribute("src", src_matcher, AVM.RREQ, "Budapest")
        ea.add_attribute("tgt", tgt_matcher, AVM.RREQ, None)
        ea.add_attribute("date", PosMatcher("\[DATE\]$"), AVM.ROPT, None)
        elvira_const = AVMConstruction(ea)
        self.lexicon.add_avm_construction(elvira_const)

        pta = plain_ticket_avm = AVM('PlainTicketAvm')
        pta.add_attribute("BKSZ", PrintnameMatcher("bksz"), AVM.ROPT, None)
        pta.add_attribute("CLASS", EnumMatcher("class", self.lexicon),
                          AVM.RREQ, "2")
        pta.add_attribute("DATE", PosMatcher("\[DATE\]$"), AVM.ROPT, None)
        pta.add_attribute("DEST", tgt_matcher, AVM.RREQ, None)
        pta.add_attribute("INV", PrintnameMatcher("invoice"), AVM.ROPT, None)
        pta.add_attribute("RED", EnumMatcher("mav_reduction", self.lexicon),
                          AVM.RREQ, "full_price")
        pta.add_attribute("RET", EnumMatcher("ticket_type", self.lexicon),
                          AVM.RREQ, "one_way")
        pta.add_attribute("SRC", src_matcher, AVM.RREQ, "Budapest")
        # Elvira takes precedence
        pta.add_attribute("ELVIRA", AndMatcher(
            PrintnameMatcher('ElviraAVM'),
            SatisfiedAVMMatcher()), AVM.RNEG, None)
        # If there is an invalid seat ticket request, do not return a ticket either
        pta.add_attribute('SEAT_TICKET', AndMatcher(
            PrintnameMatcher('ICTicketAvm'),
            SatisfiedAVMMatcher(False)), AVM.RNEG, None)
        pt_const = AVMConstruction(pta)
        self.lexicon.add_construction(pt_const)

        ita = ic_ticket_avm = AVM('ICTicketAvm')
        ita.add_attribute("CLASS", EnumMatcher("class", self.lexicon), AVM.RREQ, None)
        ita.add_attribute("DATE", PosMatcher("\[DATE\]$"), AVM.ROPT, None)
        ita.add_attribute("DEST", tgt_matcher, AVM.RREQ, None)
        ita.add_attribute("INV", PrintnameMatcher("invoice"), AVM.ROPT, None)
        ita.add_attribute("PLACE", EnumMatcher("seat", self.lexicon), AVM.ROPT, None)
        ita.add_attribute("SRC", src_matcher, AVM.RREQ, "Budapest-Nyugati")
        ita.add_attribute("TIME", AndMatcher(PosMatcher("\[TIME\]$"),
                                             ic_name_matcher), AVM.ROPT, None)
        it_const = AVMConstruction(ita)
        self.lexicon.add_avm_construction(it_const)

        # TODO create shrdlu construction

    def run(self, sentence):
        """Parses a sentence, runs the spreading activation and returns the
        messages that have to be sent to the active plugins."""
        try:
            sp = SentenceParser()
            sa = SpreadingActivation(self.lexicon)
            machines = sp.parse(sentence)

            # results is a list of (url, data) tuples
            results = sa.activation_loop(machines)
            self.lexicon.clear_active()
        except Exception, e:
            import traceback
            traceback.print_exc(e)
            raise(e)

        return results

def test():
    w = Wrapper(config_filename)
    w.run([([("a", "DET"),
             ("kek", "ADJ"),
             ("kockat", "NOUN<CAS<ACC>>")
            ], "ACC")])

if __name__ == "__main__":
    test()
<commit_msg>modified:   wrapper.py ic_name matcher enum_name is renamed to intercity<commit_after>import os
import ConfigParser

from sentence_parser import SentenceParser
from lexicon import Lexicon
from spreading_activation import SpreadingActivation
from definition_parser import read as read_defs
from construction import *
from matcher import *
from sup_dic import supplementary_dictionary_reader as sdreader
from avm import AVM

class Wrapper:
    def __init__(self, cf):
        self.cfn = cf
        self.__read_config()

        self.lexicon = Lexicon()
        self.__read_files()
        self.__add_constructions()

    def __read_config(self):
        machinepath = os.path.realpath(__file__).rsplit("/", 2)[0]
        if "MACHINEPATH" in os.environ:
            machinepath = os.environ["MACHINEPATH"]
        config = ConfigParser.SafeConfigParser({"machinepath": machinepath})
        config.read(self.cfn)
        items = dict(config.items("machine"))
        self.def_files = [(s.split(":")[0].strip(), int(s.split(":")[1]))
            for s in items["definitions"].split(",")]
        self.supp_dict_fn = items["supp_dict"]
        self.stations_fn = items["stations"]

    def __read_files(self):
        self.__read_definitions()
        self.__read_supp_dict()

    def __read_definitions(self):
        for file_name, printname_index in self.def_files:
            # TODO HACK makefile needed
            if (file_name.endswith("generated") and
                not os.path.exists(file_name)):
                raise Exception("A definition file that supposed to be " +
                    "generated by pymachine/scripts/generate_translation_dict.sh" +
                    " does not exist: {0}".format(file_name))
            definitions = read_defs(file(file_name), printname_index)
            logging.debug("{0}: {1}".format(file_name, definitions.keys()))
            self.lexicon.add_static(definitions.itervalues())

    def __read_supp_dict(self):
        self.supp_dict = sdreader(file(self.supp_dict_fn))

    def __add_constructions(self):
        # TODO this should be created automatically, maybe from knowing,
        # that "megy" is actually a verb, there is a POS in the definitions
        try:
            megy_construction = VerbConstruction("megy", self.lexicon.static[
                "megy"], self.supp_dict)
            del self.lexicon.static["megy"]
            #self.lexicon.add_construction(megy_construction)
        except KeyError:
            pass
        try:
            tesz_construction = VerbConstruction("tesz", self.lexicon.static[
                "tesz"], self.supp_dict)
            del self.lexicon.static["tesz"]
            self.lexicon.add_construction(tesz_construction)
        except KeyError:
            pass
        #self.lexicon.add_construction(ElviraConstruction())

        self.lexicon.add_construction(DummyNPConstruction())
        self.lexicon.add_construction(TheConstruction())
        self.lexicon.add_construction(MaxNP_InBetweenPostP_Construction())
        self.lexicon.add_construction(PostPConstruction())

        station_matcher = FileContainsMatcher(self.stations_fn)
        ic_name_matcher = EnumMatcher("intercity", self.lexicon)
        src_matcher = AndMatcher(self.supp_dict["@HUN_GO_SRC"],
                                 station_matcher)
        tgt_matcher = AndMatcher(self.supp_dict["@HUN_GO_TGT"],
                                 station_matcher)
        ea = elvira_avm = AVM('ElviraAVM')
        ea.add_attribute("vonat", PrintnameMatcher("vonat"), AVM.RREQ, None)
        ea.add_attribute("menetrend", PrintnameMatcher("menetrend"), AVM.RREQ, None)
        ea.add_attribute("src", src_matcher, AVM.RREQ, "Budapest")
        ea.add_attribute("tgt", tgt_matcher, AVM.RREQ, None)
        ea.add_attribute("date", PosMatcher("\[DATE\]$"), AVM.ROPT, None)
        elvira_const = AVMConstruction(ea)
        self.lexicon.add_avm_construction(elvira_const)

        pta = plain_ticket_avm = AVM('PlainTicketAvm')
        pta.add_attribute("BKSZ", PrintnameMatcher("bksz"), AVM.ROPT, None)
        pta.add_attribute("CLASS", EnumMatcher("class", self.lexicon),
                          AVM.RREQ, "2")
        pta.add_attribute("DATE", PosMatcher("\[DATE\]$"), AVM.ROPT, None)
        pta.add_attribute("DEST", tgt_matcher, AVM.RREQ, None)
        pta.add_attribute("INV", PrintnameMatcher("invoice"), AVM.ROPT, None)
        pta.add_attribute("RED", EnumMatcher("mav_reduction", self.lexicon),
                          AVM.RREQ, "full_price")
        pta.add_attribute("RET", EnumMatcher("ticket_type", self.lexicon),
                          AVM.RREQ, "one_way")
        pta.add_attribute("SRC", src_matcher, AVM.RREQ, "Budapest")
        # Elvira takes precedence
        pta.add_attribute("ELVIRA", AndMatcher(
            PrintnameMatcher('ElviraAVM'),
            SatisfiedAVMMatcher()), AVM.RNEG, None)
        # If there is an invalid seat ticket request, do not return a ticket either
        pta.add_attribute('SEAT_TICKET', AndMatcher(
            PrintnameMatcher('ICTicketAvm'),
            SatisfiedAVMMatcher(False)), AVM.RNEG, None)
        pt_const = AVMConstruction(pta)
        self.lexicon.add_construction(pt_const)

        ita = ic_ticket_avm = AVM('ICTicketAvm')
        ita.add_attribute("CLASS", EnumMatcher("class", self.lexicon), AVM.RREQ, None)
        ita.add_attribute("DATE", PosMatcher("\[DATE\]$"), AVM.ROPT, None)
        ita.add_attribute("DEST", tgt_matcher, AVM.RREQ, None)
        ita.add_attribute("INV", PrintnameMatcher("invoice"), AVM.ROPT, None)
        ita.add_attribute("PLACE", EnumMatcher("seat", self.lexicon), AVM.ROPT, None)
        ita.add_attribute("SRC", src_matcher, AVM.RREQ, "Budapest-Nyugati")
        ita.add_attribute("TIME", AndMatcher(PosMatcher("\[TIME\]$"),
                                             ic_name_matcher), AVM.ROPT, None)
        it_const = AVMConstruction(ita)
        self.lexicon.add_avm_construction(it_const)

        # TODO create shrdlu construction

    def run(self, sentence):
        """Parses a sentence, runs the spreading activation and returns the
        messages that have to be sent to the active plugins."""
        try:
            sp = SentenceParser()
            sa = SpreadingActivation(self.lexicon)
            machines = sp.parse(sentence)

            # results is a list of (url, data) tuples
            results = sa.activation_loop(machines)
            self.lexicon.clear_active()
        except Exception, e:
            import traceback
            traceback.print_exc(e)
            raise(e)

        return results

def test():
    w = Wrapper(config_filename)
    w.run([([("a", "DET"),
             ("kek", "ADJ"),
             ("kockat", "NOUN<CAS<ACC>>")
            ], "ACC")])

if __name__ == "__main__":
    test()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from openerp import fields, models

class ResCompany(models.Model):
    _inherit = 'res.company'

    coop_email_contact = fields.Char(string="Contact email address for the"
                                     " cooperator")
    subscription_maximum_amount = fields.Float(string="Maximum authorised"
                                               " subscription amount")
    default_country_id = fields.Many2one('res.country',
                                         string="Default country",
                                         default=lambda self: self.country_id)
    default_lang_id = fields.Many2one('res.lang',
                                      string="Default lang")
    allow_id_card_upload = fields.Boolean(string="Allow ID Card upload")
    create_user = fields.Boolean(string="Create user for cooperator")
    board_representative = fields.Char(string="Board representative name")
    signature_scan = fields.Binary(string="Board representative signature")
    property_cooperator_account = fields.Many2one('account.account',
                                                  company_dependent=True,
                                                  string="Cooperator Account",
                                                  domain=[('internal_type', '=', 'receivable'),
                                                          ('deprecated', '=', False)],
                                                  help="This account will be"
                                                  " the default one as the"
                                                  " receivable account for the"
                                                  " cooperators",
                                                  required=True)
    unmix_share_type = fields.Boolean(string="Unmix share type",
                                      default=True,
                                      help="If checked, A cooperator will be"
                                      " authorised to have only one type"
                                      " of share")
    display_logo1 = fields.Boolean(string="Display logo 1")
    display_logo2 = fields.Boolean(string="Display logo 2")
    bottom_logo1 = fields.Binary(string="Bottom logo 1")
    bottom_logo2 = fields.Binary(string="Bottom logo 2")
<commit_msg>[IMP] make the default value at true to not break current behaviour<commit_after># -*- coding: utf-8 -*-
from openerp import fields, models

class ResCompany(models.Model):
    _inherit = 'res.company'

    coop_email_contact = fields.Char(string="Contact email address for the"
                                     " cooperator")
    subscription_maximum_amount = fields.Float(string="Maximum authorised"
                                               " subscription amount")
    default_country_id = fields.Many2one('res.country',
                                         string="Default country",
                                         default=lambda self: self.country_id)
    default_lang_id = fields.Many2one('res.lang',
                                      string="Default lang")
    allow_id_card_upload = fields.Boolean(string="Allow ID Card upload")
    create_user = fields.Boolean(string="Create user for cooperator",
                                 default=True)
    board_representative = fields.Char(string="Board representative name")
    signature_scan = fields.Binary(string="Board representative signature")
    property_cooperator_account = fields.Many2one('account.account',
                                                  company_dependent=True,
                                                  string="Cooperator Account",
                                                  domain=[('internal_type', '=', 'receivable'),
                                                          ('deprecated', '=', False)],
                                                  help="This account will be"
                                                  " the default one as the"
                                                  " receivable account for the"
                                                  " cooperators",
                                                  required=True)
    unmix_share_type = fields.Boolean(string="Unmix share type",
                                      default=True,
                                      help="If checked, A cooperator will be"
                                      " authorised to have only one type"
                                      " of share")
    display_logo1 = fields.Boolean(string="Display logo 1")
    display_logo2 = fields.Boolean(string="Display logo 2")
    bottom_logo1 = fields.Binary(string="Bottom logo 1")
    bottom_logo2 = fields.Binary(string="Bottom logo 2")
<|endoftext|>


<commit_before>from kivy.clock import Clock
import logging
import time
import utils

__author__ = 'woolly_sammoth'

from kivy.config import Config

Config.set('graphics', 'borderless', '0')
Config.set('graphics', 'width', '1000')
Config.set('graphics', 'height', '1000')
Config.set('graphics', 'resizable', '1')
Config.set('graphics', 'fullscreen', '0')
Config.set('input', 'mouse', 'mouse,disable_multitouch')

from kivy.app import App
from kivy.uix.boxlayout import BoxLayout
from kivy.uix.label import Label
from kivy.uix.button import Button
from kivy.uix.screenmanager import ScreenManager
from kivy.uix.actionbar import ActionBar
from kivy.uix.screenmanager import SlideTransition
from kivy.uix.popup import Popup
from kivy.lang import Builder

import os
import json

import screens.HomeScreen as HomeScreen

class TopActionBar(ActionBar):
    def __init__(self, **kwargs):
        super(TopActionBar, self).__init__(**kwargs)
        return

class PlungeApp(App):
    def __init__(self, **kwargs):
        super(PlungeApp, self).__init__(**kwargs)
        self.isPopup = False
        self.use_kivy_settings = False
        self.settings_cls = 'Settings'
        self.utils = utils.utils(self)
        self.exchanges = ['ccedk', 'poloniex', 'bitcoincoid', 'bter']
        self.active_exchanges = []
        self.currencies = ['btc', 'ltc', 'eur', 'usd', 'ppc']
        self.active_currencies = []

        self.logger = logging.getLogger('Plunge')
        self.logger.setLevel(logging.DEBUG)
        fh = logging.FileHandler('logs/%s_%d.log' % ('Plunge', time.time()))
        fh.setLevel(logging.DEBUG)
        ch = logging.StreamHandler()
        ch.setLevel(logging.INFO)
        formatter = logging.Formatter(fmt='%(asctime)s %(levelname)s: %(message)s', datefmt="%Y/%m/%d-%H:%M:%S")
        fh.setFormatter(formatter)
        ch.setFormatter(formatter)
        self.logger.addHandler(fh)
        self.logger.addHandler(ch)

        return

    def build(self):
        self.logger.info("Fetching language from config")
        self.language = self.config.get('standard', 'language')
        try:
            self.lang = json.load(open('res/json/languages/' + self.language.lower() + '.json', 'r'))
        except (ValueError, IOError) as e:
            self.logger.error('')
            self.logger.error('##################################################################')
            self.logger.error('')
            self.logger.error('There was an Error loading the ' + self.language + ' language file.')
            self.logger.error('')
            self.logger.error(str(e))
            self.logger.error('')
            self.logger.error('##################################################################')
            raise SystemExit

        self.root = BoxLayout(orientation='vertical')

        self.mainScreenManager = ScreenManager(transition=SlideTransition(direction='left'))
        Builder.load_file('screens/HomeScreen.kv')
        self.homeScreen = HomeScreen.HomeScreen(self)
        self.mainScreenManager.add_widget(self.homeScreen)

        self.topActionBar = TopActionBar()
        self.root.add_widget(self.topActionBar)
        self.root.add_widget(self.mainScreenManager)
        return self.root

    def get_string(self, text):
        try:
            self.logger.debug("Getting string for %s" % text)
            return_string = self.lang[text]
        except (ValueError, KeyError):
            self.logger.error("No string found for %s in %s language file" % (text, self.language))
            return_string = 'Language Error'
        return return_string

    def build_config(self, config):
        config.setdefaults('server', {'host': "104.245.36.10", 'port': 2019, 'period': 30})
        config.setdefaults('config', {'file': os.getcwd() + "/users.dat", 'override': 0})
        config.setdefaults('exchanges', {'ccedk': 0, 'poloniex': 0, 'bitcoincoid': 0, 'bter': 0})
        config.setdefaults('ccedk',
                           {'address': '', 'public': '', 'secret': '', 'nubot': 0,
                            'btc': 0, 'ltc': 0, 'ppc': 0, 'usd': 0, 'eur': 0})
        config.setdefaults('poloniex',
                           {'address': '', 'public': '', 'secret': '', 'nubot': 0, "btc": 0})
        config.setdefaults('bitcoincoid',
                           {'address': '', 'public': '', 'secret': '', 'nubot': 0, "btc": 0})
        config.setdefaults('bter',
                           {'address': '', 'public': '', 'secret': '', 'nubot': 0, "btc": 0})
        config.setdefaults('standard', {'language': 'English'})

    def build_settings(self, settings):
        settings.add_json_panel(self.get_string('Plunge_Configuration'), self.config, 'settings/plunge.json')
        if self.config.getint('exchanges', 'ccedk') == 1:
            settings.add_json_panel(self.get_string('CCEDK_Settings'), self.config, 'settings/ccedk.json')
        if self.config.getint('exchanges', 'poloniex') == 1:
            settings.add_json_panel(self.get_string('Poloniex_Settings'), self.config, 'settings/poloniex.json')
        if self.config.getint('exchanges', 'bitcoincoid') == 1:
            settings.add_json_panel(self.get_string('BitcoinCoId_Settings'), self.config, 'settings/bitcoincoid.json')
        if self.config.getint('exchanges', 'bter') == 1:
            settings.add_json_panel(self.get_string('Bter_Settings'), self.config, 'settings/bter.json')

    def on_config_change(self, config, section, key, value):
        if section == "exchanges":
            self.logger.info("%s/%s config changed to %s" % (section, key, value))
            self.close_settings()
            self.destroy_settings()
            self.open_settings()
        if section == "server" and key == "period":
            Clock.unschedule(self.homeScreen.get_stats)
            self.logger.info("Setting refresh Period to %s" % self.config.get('server', 'period'))
            Clock.schedule_interval(self.homeScreen.get_stats, self.config.getint('server', 'period'))
        self.active_exchanges = self.utils.get_active_exchanges()
        self.homeScreen.exchange_spinner.values = [self.get_string(exchange) for exchange in self.active_exchanges]
        self.homeScreen.set_exchange_spinners()

    def show_popup(self, title, text):
        content = BoxLayout(orientation='vertical')
        content.add_widget(Label(text=text, size_hint=(1, .7)))
        button = Button(text=self.get_string('OK'), size_hint=(1, .3))
        content.add_widget(button)
        self.popup = Popup(title=title, content=content, auto_dismiss=False, size_hint=(None, None), size=(500, 200))
        button.bind(on_press=self.close_popup)
        self.popup.open()
        self.isPopup = True
        return

    def close_popup(self, instance, value=False):
        self.popup.dismiss()
        self.isPopup = False
        return

if __name__ == '__main__':
    Plunge = PlungeApp()
    Plunge.run()<commit_msg>socket logger<commit_after>from kivy.clock import Clock
import logging
import time
import utils
import socketlogger

__author__ = 'woolly_sammoth'

from kivy.config import Config

Config.set('graphics', 'borderless', '0')
Config.set('graphics', 'width', '1000')
Config.set('graphics', 'height', '1000')
Config.set('graphics', 'resizable', '1')
Config.set('graphics', 'fullscreen', '0')
Config.set('input', 'mouse', 'mouse,disable_multitouch')

from kivy.app import App
from kivy.uix.boxlayout import BoxLayout
from kivy.uix.label import Label
from kivy.uix.button import Button
from kivy.uix.screenmanager import ScreenManager
from kivy.uix.actionbar import ActionBar
from kivy.uix.screenmanager import SlideTransition
from kivy.uix.popup import Popup
from kivy.lang import Builder

import os
import json

import screens.HomeScreen as HomeScreen

class TopActionBar(ActionBar):
    def __init__(self, **kwargs):
        super(TopActionBar, self).__init__(**kwargs)
        return

class PlungeApp(App):
    def __init__(self, **kwargs):
        super(PlungeApp, self).__init__(**kwargs)
        self.isPopup = False
        self.use_kivy_settings = False
        self.settings_cls = 'Settings'
        self.utils = utils.utils(self)
        self.exchanges = ['ccedk', 'poloniex', 'bitcoincoid', 'bter']
        self.active_exchanges = []
        self.currencies = ['btc', 'ltc', 'eur', 'usd', 'ppc']
        self.active_currencies = []
        self.logger = logging.getLogger('Plunge')
        self.logger.setLevel(logging.DEBUG)
        fh = logging.FileHandler('logs/%s_%d.log' % ('Plunge', time.time()))
        fh.setLevel(logging.DEBUG)
        ch = logging.StreamHandler()
        ch.setLevel(logging.INFO)
        formatter = logging.Formatter(fmt='%(asctime)s %(levelname)s: %(message)s', datefmt="%Y/%m/%d-%H:%M:%S")
        fh.setFormatter(formatter)
        ch.setFormatter(formatter)
        self.logger.addHandler(fh)
        self.logger.addHandler(ch)
        socketlogger.start_logging_receiver('Plunge')

        return

    def build(self):
        self.logger.info("Fetching language from config")
        self.language = self.config.get('standard', 'language')
        try:
            self.lang = json.load(open('res/json/languages/' + self.language.lower() + '.json', 'r'))
        except (ValueError, IOError) as e:
            self.logger.error('')
            self.logger.error('##################################################################')
            self.logger.error('')
            self.logger.error('There was an Error loading the ' + self.language + ' language file.')
            self.logger.error('')
            self.logger.error(str(e))
            self.logger.error('')
            self.logger.error('##################################################################')
            raise SystemExit

        self.root = BoxLayout(orientation='vertical')

        self.mainScreenManager = ScreenManager(transition=SlideTransition(direction='left'))
        Builder.load_file('screens/HomeScreen.kv')
        self.homeScreen = HomeScreen.HomeScreen(self)
        self.mainScreenManager.add_widget(self.homeScreen)

        self.topActionBar = TopActionBar()
        self.root.add_widget(self.topActionBar)
        self.root.add_widget(self.mainScreenManager)
        return self.root

    def get_string(self, text):
        try:
            self.logger.debug("Getting string for %s" % text)
            return_string = self.lang[text]
        except (ValueError, KeyError):
            self.logger.error("No string found for %s in %s language file" % (text, self.language))
            return_string = 'Language Error'
        return return_string

    def build_config(self, config):
        config.setdefaults('server', {'host': "104.245.36.10", 'port': 2019, 'period': 30})
        config.setdefaults('config', {'file': os.getcwd() + "/users.dat", 'override': 0})
        config.setdefaults('exchanges', {'ccedk': 0, 'poloniex': 0, 'bitcoincoid': 0, 'bter': 0})
        config.setdefaults('ccedk',
                           {'address': '', 'public': '', 'secret': '', 'nubot': 0,
                            'btc': 0, 'ltc': 0, 'ppc': 0, 'usd': 0, 'eur': 0})
        config.setdefaults('poloniex',
                           {'address': '', 'public': '', 'secret': '', 'nubot': 0, "btc": 0})
        config.setdefaults('bitcoincoid',
                           {'address': '', 'public': '', 'secret': '', 'nubot': 0, "btc": 0})
        config.setdefaults('bter',
                           {'address': '', 'public': '', 'secret': '', 'nubot': 0, "btc": 0})
        config.setdefaults('standard', {'language': 'English'})

    def build_settings(self, settings):
        settings.add_json_panel(self.get_string('Plunge_Configuration'), self.config, 'settings/plunge.json')
        if self.config.getint('exchanges', 'ccedk') == 1:
            settings.add_json_panel(self.get_string('CCEDK_Settings'), self.config, 'settings/ccedk.json')
        if self.config.getint('exchanges', 'poloniex') == 1:
            settings.add_json_panel(self.get_string('Poloniex_Settings'), self.config, 'settings/poloniex.json')
        if self.config.getint('exchanges', 'bitcoincoid') == 1:
            settings.add_json_panel(self.get_string('BitcoinCoId_Settings'), self.config, 'settings/bitcoincoid.json')
        if self.config.getint('exchanges', 'bter') == 1:
            settings.add_json_panel(self.get_string('Bter_Settings'), self.config, 'settings/bter.json')

    def on_config_change(self, config, section, key, value):
        if section == "exchanges":
            self.logger.info("%s/%s config changed to %s" % (section, key, value))
            self.close_settings()
            self.destroy_settings()
            self.open_settings()
        if section == "server" and key == "period":
            Clock.unschedule(self.homeScreen.get_stats)
            self.logger.info("Setting refresh Period to %s" % self.config.get('server', 'period'))
            Clock.schedule_interval(self.homeScreen.get_stats, self.config.getint('server', 'period'))
        self.active_exchanges = self.utils.get_active_exchanges()
        self.homeScreen.exchange_spinner.values = [self.get_string(exchange) for exchange in self.active_exchanges]
        self.homeScreen.set_exchange_spinners()

    def show_popup(self, title, text):
        content = BoxLayout(orientation='vertical')
        content.add_widget(Label(text=text, size_hint=(1, .7)))
        button = Button(text=self.get_string('OK'), size_hint=(1, .3))
        content.add_widget(button)
        self.popup = Popup(title=title, content=content, auto_dismiss=False, size_hint=(None, None), size=(500, 200))
        button.bind(on_press=self.close_popup)
        self.popup.open()
        self.isPopup = True
        return

    def close_popup(self, instance, value=False):
        self.popup.dismiss()
        self.isPopup = False
        return

if __name__ == '__main__':
    Plunge = PlungeApp()
    Plunge.run()
<|endoftext|>


<commit_before>import unittest
from hermes.GrammarFileParser import GrammarFileParser
from hermes.Grammar import Grammar
from GrammarTest import GrammarTest

class Ast2Test(GrammarTest):
  def setUp(self):
    self.loadGrammarFile('grammars/ast2.zgr', 'START')

  def test_firstSets(self):
    self.assertFirst({
      'start': {'identifier', 'ε', 'sub', 'n', 'for', 's', 'λ'},
      'statement': {'n', 'for', 'identifier', 's', 'sub', 'λ'},
      '_gen1': {'semi', 'ε'},
      '_expr': {'identifier', 'sub', 'n', 's', 'λ'},
      'for': {'for'},
      '_gen0': {'for', 'identifier', 'ε', 's', 'n', 'sub', 'λ'},
      'forsub': {'identifier', 'ε', 'sub', 'n', 's', 'λ'},
      'forbody': {'identifier', 'ε', 's', 'n', 'for', 'sub', 'λ'}
    })

  def test_followSets(self):
    self.assertFollow({
      'start': {'σ'},
      '_expr': {'semi', 'σ', 'rparen'},
      'statement': {'semi', 'σ'},
      'for': {'semi', 'σ'},
      '_gen0': {'σ'},
      'forsub': {'semi', 'rparen'},
      '_gen1': {'σ'},
      'forbody': {'rbrace'}
    })

  def test_conflicts(self):
    self.assertEqual(0, len(self.grammar.conflicts), "Expected zero conflicts")

  def test_codeGeneration(self):
    self.runWithTokens(['identifier','eq','identifier','eq','n','semi','for','lparen','identifier','eq','n','semi','semi','identifier','eq','s','add','n','rparen','lbrace','identifier','eq','n','add','identifier','mul','n','div','s','semi','rbrace','semi','identifier','add','identifier']) \
        .assertParseTree('(start: (_gen0: (statement: (_expr: identifier, eq, (_expr: identifier, eq, n))), (_gen1: semi, (statement: (for: for, lparen, (forsub: (_expr: identifier, eq, n)), semi, (forsub: ), semi, (forsub: (_expr: identifier, eq, (_expr: s, add, n))), rparen, lbrace, (forbody: (statement: (_expr: identifier, eq, (_expr: n, add, (_expr: (_expr: identifier, mul, n), div, s)))), semi, (forbody: )), rbrace)), (_gen1: semi, (statement: (_expr: identifier, add, identifier)), (_gen1: )))))') \
        .assertAst('(Program: statements=[(Assign: rhs=(Assign: rhs=n, lhs=identifier), lhs=identifier), (For: decl=(Assign: rhs=n, lhs=identifier), body=(Assign: rhs=(Add: rhs=(Divide: rhs=s, lhs=(Multiply: rhs=n, lhs=identifier)), lhs=n), lhs=identifier), cond=None, iter=(Assign: rhs=(Add: rhs=n, lhs=s), lhs=identifier)), (Add: rhs=identifier, lhs=identifier)])')

  # Same as test_codeGeneration but the last 'semi' was removed
  def test_codeGenerationWithError(self):
    self.runWithTokens(['identifier','eq','identifier','eq','n','semi','for','lparen','identifier','eq','n','semi','semi','identifier','eq','s','add','n','rparen','lbrace','identifier','eq','n','add','identifier','mul','n','div','s','semi','rbrace','identifier','add','identifier']) \
        .assertParseTree('Syntax Error: Finished parsing without consuming all tokens.')
if __name__ == '__main__':
  unittest.main()
<commit_msg>maxdiff++<commit_after>import unittest
from hermes.GrammarFileParser import GrammarFileParser
from hermes.Grammar import Grammar
from GrammarTest import GrammarTest

class Ast2Test(GrammarTest):
  def setUp(self):
    self.maxDiff = None
    self.loadGrammarFile('grammars/ast2.zgr', 'START')

  def test_firstSets(self):
    self.assertFirst({
      'start': {'identifier', 'ε', 'sub', 'n', 'for', 's', 'λ'},
      'statement': {'n', 'for', 'identifier', 's', 'sub', 'λ'},
      '_gen1': {'semi', 'ε'},
      '_expr': {'identifier', 'sub', 'n', 's', 'λ'},
      'for': {'for'},
      '_gen0': {'for', 'identifier', 'ε', 's', 'n', 'sub', 'λ'},
      'forsub': {'identifier', 'ε', 'sub', 'n', 's', 'λ'},
      'forbody': {'identifier', 'ε', 's', 'n', 'for', 'sub', 'λ'}
    })

  def test_followSets(self):
    self.assertFollow({
      'start': {'σ'},
      '_expr': {'semi', 'σ', 'rparen'},
      'statement': {'semi', 'σ'},
      'for': {'semi', 'σ'},
      '_gen0': {'σ'},
      'forsub': {'semi', 'rparen'},
      '_gen1': {'σ'},
      'forbody': {'rbrace'}
    })

  def test_conflicts(self):
    self.assertEqual(0, len(self.grammar.conflicts), "Expected zero conflicts")

  def test_codeGeneration(self):
    self.runWithTokens(['identifier','eq','identifier','eq','n','semi','for','lparen','identifier','eq','n','semi','semi','identifier','eq','s','add','n','rparen','lbrace','identifier','eq','n','add','identifier','mul','n','div','s','semi','rbrace','semi','identifier','add','identifier']) \
        .assertParseTree('(start: (_gen0: (statement: (_expr: identifier, eq, (_expr: identifier, eq, n))), (_gen1: semi, (statement: (for: for, lparen, (forsub: (_expr: identifier, eq, n)), semi, (forsub: ), semi, (forsub: (_expr: identifier, eq, (_expr: s, add, n))), rparen, lbrace, (forbody: (statement: (_expr: identifier, eq, (_expr: n, add, (_expr: (_expr: identifier, mul, n), div, s)))), semi, (forbody: )), rbrace)), (_gen1: semi, (statement: (_expr: identifier, add, identifier)), (_gen1: )))))') \
        .assertAst('(Program: statements=[(Assign: rhs=(Assign: rhs=n, lhs=identifier), lhs=identifier), (For: decl=(Assign: rhs=n, lhs=identifier), body=(Assign: rhs=(Add: rhs=(Divide: rhs=s, lhs=(Multiply: rhs=n, lhs=identifier)), lhs=n), lhs=identifier), cond=None, iter=(Assign: rhs=(Add: rhs=n, lhs=s), lhs=identifier)), (Add: rhs=identifier, lhs=identifier)])')

  # Same as test_codeGeneration but the last 'semi' was removed
  def test_codeGenerationWithError(self):
    self.runWithTokens(['identifier','eq','identifier','eq','n','semi','for','lparen','identifier','eq','n','semi','semi','identifier','eq','s','add','n','rparen','lbrace','identifier','eq','n','add','identifier','mul','n','div','s','semi','rbrace','identifier','add','identifier']) \
        .assertParseTree('Syntax Error: Finished parsing without consuming all tokens.')
if __name__ == '__main__':
  unittest.main()
<|endoftext|>


<commit_before><commit_msg>deal with a different respond<commit_after><|endoftext|>


<commit_before>"""
fast simulated annealing
"""

from __future__ import division,print_function
import  sys,random,math,collections
sys.dont_write_bytecode = True

rseed = random.seed
any   = random.choice

class o:
  id=0
  def __init__(i, **d):
    i.id = o.id = o.id + 1
    i.has().update(**d)
  def has(i): return i.__dict__
  def update(i,**d) : i.has().update(d); return i
  def __repr__(i)   :
    show=[':%s %s' % (k,i.has()[k])
          for k in sorted(i.has().keys() )
          if k[0] is not "_"]
    txt = ' '.join(show)
    if len(txt) > 60:
      show=map(lambda x: '\t'+x+'\n',show)
    return '{'+' '.join(show)+'}'

class S:
  def __init__(i):
    i.n = i.most = 0
    i.mode = None
    i.w    = 1
    i.counts = collections.defaultdict(lambda : 0)
  def diff(i,x,y):
    return 0 if x == y else 1
  def near(i,x,y,z,f=0.3):
    w = y if rand() < f else z
    return x if w == x else w
  def __iadd__(i,x):
    assert not isinstance(x,(float,int))
    i.n += 1
    c = i.counts[x] = i.counts[x] + 1
    if c > i.most:
      i.mode, i.most = x, i.n
    return i

def _s():
  rseed(1)
  l = list('abcdefhhijklmnopqrstuvwxyz')
  s = S()
  for _ in xrange(1000): s += any(l)
  print(s.counts,s.mode,s.most)

class N:
  def __init__(i):
    i.lo, i.hi = 10**32, -1*10**32
    i.w = 0
  def near(i,x,y,z,f=0.3):
    return x + f*(y-z)
  def diff(i,x,y):
    x = (x - i.lo)/(i.hi - i.lo + 0.00001)
    y = (y - i.lo)/(i.hi - i.lo + 0.00001)
    return (x-y)**2
  def __iadd__(i,x):
    assert isinstance(x,(float,int))
    i.lo = min(i.lo,x)
    i.hi = max(i.hi,x)
    return i

def _n():
  rseed(1)
  l = range(0,23)
  n = N()
  for _ in xrange(10): n += any(l)
  print(n.lo,n.hi)

class G:
  def __init__(i,width=12):
    i.width = width
    i.tiles = i.grid0()
    i.east  = i.west = None
    i.c = 0
    i.v    = []
    i.cols = []
  def grid0(i):
    return [[[] for _ in range(i.width)]
            for j in range(i.width)]
  def newPoles(west,east):
    old = [item for sublist in i.tiles
                for item in sublist]
    i.west, i.east = west, east
    i.c = i.west - i.east
    i.tiles = i.grid0()
    for row in old:
      row.xy0()
  def tell(i,row):
    x,y = row.xy()
    x = int(round(x * i.width))
    y = int(round(y * i.width))
    i.tiles[x][y] += [row]
  def __iadd__(i, cells):
    def ns(x):
      return N if isinstance(x,(float,int)) else S
    i.cols = i.cols or [ns(x)() for x in cells]
    for cell,col in zip(cells,i.cols):
      col += cell
    row = Row(i,cells)
    if not i.east :
      i.east = row
    elif not i.west :
      i.west = row
      i.c = i.east - i.west
    else:
      a = row - i.west
      b = row - i.east
      if a > c:
        i.newPoles(row,i.east)
      if b > c:
        i.newPoles(i.west,row)
      i.tell(row)
    return i

print(G().tiles)

class Row:
  id = 0
  def __init__(i,g,cells=[]):
    i.id = Row.id = Row.id + 1
    i.g, i.cells, i.lastUpdate = g, cells, -1
    i.x = i.y = None
  def xy(i):
    if not i.x or not i.y:
      i.xy0()
    return i.x, i.y
  def xy0(i):
    a  = i - i.west
    b  = i - i.east
    c  = i.g.c
    i.x= (a*a + c*c - b*b)/(2*c)
    i.y= max(0, a**2 - i.x**2)**0.5
  def __sub__(i,j):
    ds = ws = 0
    for x,y,col in zip(i.cells, j.cells,  g.cols):
      ds += col.diff(x,y)
      ws += col.w
    return ds**0.5 / ws**0.5

"""
@include "lib.awk"
@include "genicLib.awk"

function _genic(      _Genic,o) {
  if (args("-d,data/weather.csv,--dump,0"\
           ",-k,10,-m,20,-n,100,-s,1",o)) {
    k = o["-k"]
    m = o["-m"]
    n = o["-n"]
    print "!!",FS,OFS
    srand(o["-s"])
    genic("cat " o["-d"], _Genic) }
}
function genic(com, _Genic,    rows) {
  a(centroid) a(age)
  while((com | getline)  > 0) {
    gsub(/([ \t\r])|#.*)/,"")
    if($0)  {
      if (length(centroid) < k)
	more(_Genic)
      else {
	 genic1(rows++,_Genic)
	 if (rows % n == 0) less(_Genic)
}}}}
function genic1(rows,_Genic,   i,this,row) {
  say("=")
  for(i=1;i<=NF;i++) {
    row[i] = $i
    rows==0 ? head(i,$i,_Genic) : lohi(i,$i,_Genic)
  }
  this = nearest(row,_Genic)
  move(++weight[this],centroid[this],row)
}
function more(_Genic,   i,new) {
  say("+")
  new = gensym("center")
  weight[new] = age[new] = 0
  for(i=1;i<=NF;i++)
    centroid[new][i]=$i
}
function less(_Genic,   sum,c,die) {
  for(c in centroid) {
    sum    += weight[c]
    age[c] += centroid[c]
  }
  for(c in centroid) if (weight[c]/sum < rand()) die[c]
  for(c in die) {
    delete centroid[c]
    delete age[c]
  }
  delete weight
}
function move(w,olds,news,_Genic,   old,new,pos) {
  for(pos in name) {
    if (pos in goalp) continue
    new = news[pos]
    old = olds[pos]
    if (new == avoid) { continue }
    if (old == avoid) { olds[pos] = new; continue }
    if (pos in nump)
      olds[pos] = (old*w + new)/(w + 1)
    else
      olds[pos] = (rand() < 1/w ? old : new)
}}
"""
<commit_msg>trying to work out how to fast update incremental updates<commit_after>"""
fast simulated annealing
"""

from __future__ import division,print_function
import  sys,random,math,collections
sys.dont_write_bytecode = True

rseed = random.seed
any   = random.choice

class o:
  id=0
  def __init__(i, **d):
    i.id = o.id = o.id + 1
    i.has().update(**d)
  def has(i): return i.__dict__
  def update(i,**d) : i.has().update(d); return i
  def __repr__(i)   :
    show=[':%s %s' % (k,i.has()[k])
          for k in sorted(i.has().keys() )
          if k[0] is not "_"]
    txt = ' '.join(show)
    if len(txt) > 60:
      show=map(lambda x: '\t'+x+'\n',show)
    return '{'+' '.join(show)+'}'

def items(x):
  if isinstance(x,list):
    for y in x:
      for z in items(y):
        yield z
  elif isinstance(x,dict):
    for y in x.values():
      for z in items(y):
        yield z
  else:
    yield x

class S:
  def __init__(i,pos=0):
    i.pos,i.n,i.most,i.mode,i.w = pos, 0, 0, None, 1
    i.counts = collections.defaultdict(lambda : 0)
  def diff(i,x,y):
    return 0 if x == y else 1
  def near(i,x,y,z,f=0.3):
    w = y if rand() < f else z
    return x if w == x else w
  def __iadd__(i,x):
    assert not isinstance(x,(float,int))
    i.n += 1
    c = i.counts[x] = i.counts[x] + 1
    if c > i.most:
      i.mode, i.most = x, i.n
    return i

def _s():
  rseed(1)
  l = list('abcdefhhijklmnopqrstuvwxyz')
  s = S()
  for _ in xrange(1000): s += any(l)
  print(s.counts,s.mode,s.most)

class N:
  def __init__(i,pos=0):
    i.pos,i.lo,i.hi,i.w = pos, 10**32, -1*10**32, 1
  def near(i,x,y,z,f=0.3):
    return x + f*(y-z)
  def diff(i,x,y):
    x = (x - i.lo)/(i.hi - i.lo + 0.00001)
    y = (y - i.lo)/(i.hi - i.lo + 0.00001)
    return (x-y)**2
  def __iadd__(i,x):
    assert isinstance(x,(float,int))
    i.lo = min(i.lo,x)
    i.hi = max(i.hi,x)
    return i

def _n():
  rseed(1)
  l = range(0,23)
  n = N()
  for _ in xrange(10): n += any(l)
  print(n.lo,n.hi)

class G:
  def __init__(i,cells1,cells2,width=12):
    i.width, i.cols,i._tiles,i.xys = width, [], None,None
    row1, row2 = Row(i,cells1), Row(i,cells2)
    i.tell(row1);  i.tell(row2)
    i.poles(row1,row2)
    i.place(row1); i.place(row2)
  def tiles(i):
    i._tiles = i._tiles or [
               [{} for _ in range(i.width)]
               for j in range(i.width)]
    return i._tiles
  def poles(west,east):
    i.west,i.east,i._tiles,i.xys = west,east,None,{}
  def tell(i,row):
    def ns(x,n):
      return N if isinstance(x,(float,int)) else S
    def cols0():
      return [ns(x)(n) for
              n,x in enumerate(row.cells)]
    i.cols = i.cols or cols0()
    for cell,col in zip(row.cells,i.cols):
      col += cell
  def place(i,row):
    x,y = row.xy()
    x = int(round(x * i.width))
    y = int(round(y * i.width))
    i.tiles()[x][y][row.id] = row
  def __iadd__(i, cells):
    row = Row(i,cells)
    i.tell(row)
    a = row - i.west
    b = row - i.east
    if a > c: i.poles(row,i.east)
    if b > c: i.poles(i.west,row)
    return i

class Row:
  id = 0
  def __init__(i,g,cells=[]):
    i.id = Row.id = Row.id + 1
    i.g, i.cells, i.x, i.y = g, cells, None, None
  def xy(i):
    cache = i.xys()
    key   = i.id
    if not key in cache:
      cache[key] = i.xy0()
    return cache[key]
  def xy0(i):
    a = i - i.west
    b = i - i.east
    c = i.g.c
    x = (a*a + c*c - b*b)/(2*c + 0.0001)
    y = max(0, a**2 - x**2)**0.5
    return x,y
  def __sub__(i,j):
    ds = ws = 0
    for x,y,col in zip(i.cells, j.cells,  g.cols):
      ds += col.diff(x,y)
      ws += col.w
    return ds**0.5 / ws**0.5

"""
@include "lib.awk"
@include "genicLib.awk"

function _genic(      _Genic,o) {
  if (args("-d,data/weather.csv,--dump,0"\
           ",-k,10,-m,20,-n,100,-s,1",o)) {
    k = o["-k"]
    m = o["-m"]
    n = o["-n"]
    print "!!",FS,OFS
    srand(o["-s"])
    genic("cat " o["-d"], _Genic) }
}
function genic(com, _Genic,    rows) {
  a(centroid) a(age)
  while((com | getline)  > 0) {
    gsub(/([ \t\r])|#.*)/,"")
    if($0)  {
      if (length(centroid) < k)
	more(_Genic)
      else {
	 genic1(rows++,_Genic)
	 if (rows % n == 0) less(_Genic)
}}}}
function genic1(rows,_Genic,   i,this,row) {
  say("=")
  for(i=1;i<=NF;i++) {
    row[i] = $i
    rows==0 ? head(i,$i,_Genic) : lohi(i,$i,_Genic)
  }
  this = nearest(row,_Genic)
  move(++weight[this],centroid[this],row)
}
function more(_Genic,   i,new) {
  say("+")
  new = gensym("center")
  weight[new] = age[new] = 0
  for(i=1;i<=NF;i++)
    centroid[new][i]=$i
}
function less(_Genic,   sum,c,die) {
  for(c in centroid) {
    sum    += weight[c]
    age[c] += centroid[c]
  }
  for(c in centroid) if (weight[c]/sum < rand()) die[c]
  for(c in die) {
    delete centroid[c]
    delete age[c]
  }
  delete weight
}
function move(w,olds,news,_Genic,   old,new,pos) {
  for(pos in name) {
    if (pos in goalp) continue
    new = news[pos]
    old = olds[pos]
    if (new == avoid) { continue }
    if (old == avoid) { olds[pos] = new; continue }
    if (pos in nump)
      olds[pos] = (old*w + new)/(w + 1)
    else
      olds[pos] = (rand() < 1/w ? old : new)
}}
"""
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

'''
Written by Robert Fairburn
robert.fairburn@c2fo.com

http://www.c2fo.com
http://devop.ninja
https://github.com/rfairburn/ec2_snapshots

Generate a list of hosts in ec2, Take snapshost of them.  Clean up
old snapshots.

It expects the file .boto to exist in your home directory with contents
as follows:

[Credentials]
aws_access_key_id = <AWS_ACCESS_KEY_ID>
aws_secret_access_key = <AWS_SECRET_ACCESS_KEY>
'''

import boto.ec2
import os
import sys
import yaml
import time
import argparse
import threading
import calendar

from boto.exception import BotoClientError, BotoServerError, EC2ResponseError
from datetime import datetime, timedelta
from threading import Thread, Lock
from Queue import Queue, Empty
from getch import _Getch

class StoppableThread(Thread):
    '''
    Thread class with a stop() method. The thread itself has to check
    regularly for the stopped() condition.
    http://stackoverflow.com/questions/323972/is-there-any-way-to-kill-a-thread-in-python
    '''
    def __init__(self, *args, **kwargs):
        '''
        Pass init back to Thread and register stop event
        '''
        super(StoppableThread, self).__init__(*args, **kwargs)
        self._stop = threading.Event()

    def stop(self):
        '''
        Set stop on stop event
        '''
        self._stop.set()

    def stopped(self):
        '''
        Check if we are stopped
        '''
        return self._stop.isSet()

def kill_all_threads():
    '''
    Loop the global threads and tell each to stop.
    Join to block until they quit.
    '''
    print 'Bleeding out the queue'
    while not q.empty():
        try:
            q.get(False)
        except Empty:
            continue
        q.task_done()
    print 'Killing worker threads'
    # send the kill event without blocking
    for thread in threads:
        thread.stop()
    # block them all until they die
    # this may be better handled another way
    # but will work for now
    for thread in threads:
        thread.join()

def progress_bar(percent):
    '''
    Take percent and make a progress bar
    twenty characters wide
    '''
    complete = int(percent // 5)
    todo = int(20 - complete)
    bar_text = '[{0}{1}] {2}%'.format('#' * complete, '-' * todo, percent)
    return bar_text

def parse_args():
    '''
    Get arguments from the command like with argparse
    '''
    parser = argparse.ArgumentParser(
        description='Generate a list of hosts and generate/cleanup snapshots'
        )
    parser.add_argument(
        '--interactive', '-i',
        required=False,
        help='Interactive',
        action='store_true')
    parser.add_argument(
        '--days', '-d',
        required=False, type=int,
        help='Snapshot Keep Days',
        default='3')
    parser.add_argument(
        '--threads', '-t',
        required=False, type=int,
        help='Worker Threads',
        default='15')
    parser.add_argument(
        '--region', '-r',
        required=False, help='EC2 Region',
        default='us-west-2')
    args = parser.parse_args()
    return args

def generate_host_dict(instance, args):
    '''
    Generate the dictionary of hosts from Amazon EC2 that we do the rest of
    the work on.  Format should look like this:
    {
        hostname:
            {
                id: ec2 instance.id
                block_devices:
                    {
                        block_device_name: amazon_volume_id
                    }
                snapshot_keep_days: ec2 instance.tags['snapshot_keep_days']
            }

    }

    Also generate a dict used for progress bars that looks like this
    (very redundant):
    {
        hostname:
            {
                block_device_name (block_device_id): completed_percent
            }
    }
    '''
    instance_id = str(instance.id)
    if 'snapshot_keep_days' in instance.tags:
        snapshot_keep_days = int(
            instance.tags['snapshot_keep_days'])
    else:
        # Temporarily zero to test
        snapshot_keep_days = int(args.days)
    if 'Name' in instance.tags:
        hostname = str(instance.tags['Name'])
    else:
        hostname = instance_id
    block_device_mapping = instance.block_device_mapping
    block_devices = {}
    block_device_status = {}
    for block_device_name, block_device_type in block_device_mapping.items():
        # This seems terribly redundant, but we need a dict to use
        # and one for the current progress, so there we are.
        # Generate Block Devices for Queue
        block_devices.update(
            {str(block_device_name): str(block_device_type.volume_id)}
        )
        # Block devices for the status dict
        block_device_text = '{0} ({1})'.format(
            str(block_device_name),
            str(block_device_type.volume_id))
        block_device_status.update({block_device_text: int(0)})
    hosts.update({hostname: block_device_status})

    host = (
        {
            hostname:
                {
                    'id': instance_id,
                    'snapshot_keep_days': snapshot_keep_days,
                    'block_devices': block_devices
                }
        }
    )
    return host

def populate_queue(conn, args):
    '''
    Use generate_host_dict to make dictionary items to work on then populate
    the queue with them.

    Afterward get all snapshots who's expire dates are older than now.
    '''
    # Add all hosts in region to be backed up to queue
    print 'Populating queue with instances to snapshot...'
    reservations = conn.get_all_reservations()
    for reservation in reservations:
        instances = reservation.instances
        for instance in instances:
            if instance.state == 'running':
                host = generate_host_dict(instance, args)
                q.put(['create', host])
    # Add all expired snapshots to queue
    print 'Populating queue with snapshots to delete...'
    snapshots = conn.get_all_snapshots()
    for snapshot in snapshots:
        if 'expire_time_unix' in snapshot.tags.keys():
            # get now in unixtime
            now = calendar.timegm(datetime.utcnow().utctimetuple())
            if now > int(snapshot.tags['expire_time_unix']):
                print '{0} scheduled for deletion'.format(snapshot.description)
                q.put(['delete', snapshot])

def acquire_lock(fun, *args, **kwargs):
    '''
    Run lock and run the function so that we maintain thread
    safety on shared data.
    '''
    lock.acquire()
    try:
        return_info = fun(*args, **kwargs)
    finally:
        lock.release()
    return return_info

def make_gui_hosts(make_all=False):
    '''
    Take the hosts dict made in generate_hosts_dict and change the percent to
    pretty bars.  Optionally only show the in-progress hosts (make_all=False)

    Dict should look like this:
    {
        hostname:
            {
                bd_name (bd_id): [#####---------------] 25%
            }
    }

    '''
    gui_hosts = {}
    for host, block_devices in hosts.items():
        update_host = False
        gui_block_devices = {}
        for block_device, percent in block_devices.items():
            if (percent > 0 and percent < 100) or make_all:
                gui_block_devices.update({block_device: progress_bar(percent)})
                update_host = True
        if update_host:
            gui_hosts.update({host: gui_block_devices})
    return gui_hosts

def show_completed_count(action, length):
    '''
    Show which items returned successfully and are complete
    '''
    use_s = 's' if length is not 1 else ''
    if action == 'create':
        print 'Completed {0} host{1}'.format(length, use_s)
    elif action == 'delete':
        print 'Deleted {0} snapshot{1}'.format(length, use_s)

def draw_gui_hosts(make_all=False):
    '''
    Draw the output.  Use sys.stdout.write for formatting while threaded
    '''
    gui_hosts = make_gui_hosts(make_all)
    if gui_hosts:
        output = yaml.dump(dict(gui_hosts), default_flow_style=False)
    else:
        output = ''
    print output
    length = len(completed_list)
    show_completed_count('create', length)
    print 'Hosts: {0}'.format(', '.join(completed_list))
    print 'Failed Hosts: {0}'.format(', '.join(failed_list))
    length = len(deleted_list)
    show_completed_count('delete', length)
    print 'Deleted Snapshots: {0}'.format(','.join(deleted_list))
    print 'Failed Snapshot Deletions: {0}'.format(','.join(failed_delete_list))
    return gui_hosts

def interactive_watcher():
    '''
    Watcher for when we want to actively display the progress
    Auto draw gui hosts and whatnot.
    '''
    text = _Getch(timeout=1)
    character = None
    full = False
    # avoid race condition where threads have not spawned yet
    can_exit = False
    # Just us and the main thread means it is time to bail out
    while True:
        if threading.active_count() > 2:
            # Threads have spawned safe to exit
            can_exit = True
        if threading.active_count() <= 2 and can_exit:
            # exit out because they have now closed
            break
        character = text()
        os.system('cls' if sys.platform == 'win32' else 'clear')
        acquire_lock(draw_gui_hosts, make_all=full)
        print 'Press ESC to (q)uit or obtain (f)ull details.'
        if character == 'f':
            full = not full
        if character == 'q' or character == '\x1b':
            kill_all_threads()
            break
    print 'Exiting.'

def passive_watcher():
    '''
    Don't show progress bars.  Just output 'completed' when host complets
    '''
    print 'Non-interactive mode enabled.  You may still press ESC to (q)uit.'
    length = acquire_lock(len, completed_list)
    show_completed_count('create', length)
    delete_length = acquire_lock(len, deleted_list)
    show_completed_count('delete', delete_length)
    # avoid a race condition where threads have not spawned yet
    text = _Getch(timeout=1)
    character = None
    can_exit = False
    while True:
        character = text()
        if threading.active_count() > 2:
            can_exit = True
        old_length = length
        length = acquire_lock(len, completed_list)
        if old_length < length:
            show_completed_count('create', length)
        old_delete_length = delete_length
        delete_length = acquire_lock(len, deleted_list)
        if old_delete_length < delete_length:
            show_completed_count('delete', delete_length)
        # Stop if just the main thread and us.
        # queue.empty check misses some so do this.
        if threading.active_count() <= 2 and can_exit:
            break
        if character == 'q' or character == '\x1b':
            kill_all_threads()
            break

def create_snapshot(host, conn):
    '''
    This function does the heavy lifting.
    '''
    # Assume true unless issue
    success = True
    snapshots = []
    canupdate = False
    hostname, host_data = host.items()[0]
    # print hostname
    for block_device, volume_id in host_data['block_devices'].items():
        name = '{0} ({1}): {2}'.format(hostname, host_data['id'], volume_id)
        description = '{0} ({1})'.format(block_device, volume_id)
        try:
            snapshot = conn.create_snapshot(
                volume_id,
                description=description,
            )
            print 'Creating snapshot {0} named {1}'.format(
                str(snapshot.id), name)
            now = datetime.utcnow()
            expire_time = now + timedelta(
                days=host_data['snapshot_keep_days']
            )
            expire_time_unix = str(
                calendar.timegm(expire_time.utctimetuple())
            )
            expire_time_human = time.strftime(
                "%a, %d %b %Y %H:%M:%S +0000",
                expire_time.utctimetuple()
            )
            conn.create_tags(
                [snapshot.id],
                {
                    'Name': name,
                    'expire_time_unix': expire_time_unix,
                    'expire_time_human': expire_time_human,
                }
            )
            snapshots.extend([snapshot])
        except (BotoClientError, BotoServerError, EC2ResponseError):
            acquire_lock(failed_list.extend, [hostname])
            return
    while True:
        if threading.current_thread().stopped():
            break
        snapshots_complete = True
        block_device_status = {}
        for snapshot in snapshots:
            # snapshot = conn.get_all_snapshots([snapshot.id])[0]
            if canupdate:
                snapshot.update()
            else:
                canupdate = True
            percent_string = str(snapshot.progress)
            try:
                percent = int(percent_string[:-1])
            except ValueError:
                # show at least 1% to populate progress bars
                percent = 1
            if snapshot.status == 'pending':
                snapshots_complete = False
            elif snapshot.status == 'error':
                success = False
            bd_text = str(snapshot.description)
            block_device_status.update({bd_text: percent})
        status = {hostname: block_device_status}
        acquire_lock(hosts.update, status)
        if snapshots_complete:
            if success:
                acquire_lock(completed_list.extend, [hostname])
                break
            else:
                acquire_lock(failed_list.extend, [hostname])
                break
        time.sleep(30)

def delete_snapshot(snapshot):
    '''
    Delete a snapshot
    '''
    # This is a pretty short-running function, but if the event says we should
    # be stopped, then don't run
    if threading.current_thread().stopped():
        return False
    # Obtain information for logging
    if 'Name' in snapshot.tags.keys():
        name = str(snapshot.tags['Name'])
    else:
        name = str(snapshot.description)
    # Delete the snapshot
    deleted = snapshot.delete()
    if deleted:
        acquire_lock(deleted_list.extend, [name])
    else:
        acquire_lock(failed_delete_list.extend, [name])
    # Throttle to keep from hammering api
    time.sleep(30)
    return deleted

def worker(conn):
    '''
    Worker thread. Does 3 things:
        1. Grab an item from the queue.
        2. Send it through the correct processor.
        3. Update queue status.
    '''
    while not (q.empty() or threading.current_thread().stopped()):
        item = q.get()
        if item[0] == 'create':
            create_snapshot(item[1], conn)
        elif item[0] == 'delete':
            delete_snapshot(item[1])
        q.task_done()

def main():
    '''
    Main thread.  Adds shared memory globals and spawns the other threads.
    '''
    global q
    q = Queue()
    global hosts
    hosts = dict()
    global lock
    lock = Lock()
    global completed_list
    completed_list = []
    global failed_list
    failed_list = []
    global threads
    threads = []
    global deleted_list
    deleted_list = []
    global failed_delete_list
    failed_delete_list = []
    args = parse_args()
    conn = boto.ec2.connect_to_region(args.region)
    populate_queue(conn, args)
    print 'Initial Queue Size {0}'.format(int(q.qsize()))
    if args.interactive:
        watcher = Thread(target=interactive_watcher)
    else:
        watcher = Thread(target=passive_watcher)
    watcher.daemon = False
    watcher.start()
    num_worker_threads = args.threads
    for _ in range(num_worker_threads):
        thread = StoppableThread(target=worker, args=(conn,))
        thread.daemon = False
        thread.start()
        threads.extend([thread])
    q.join()
    watcher.join()
    acquire_lock(draw_gui_hosts, make_all=True)

if __name__ == '__main__':
    sys.exit(main())
<commit_msg>documentation fix<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

'''
Written by Robert Fairburn
robert.fairburn@c2fo.com

http://www.c2fo.com
http://devop.ninja
https://github.com/rfairburn/ec2_snapshots

Generate a list of hosts in ec2, Take snapshost of them.  Clean up
old snapshots.

It expects the file .boto to exist in your home directory with contents
as follows:

[Credentials]
aws_access_key_id = <AWS_ACCESS_KEY_ID>
aws_secret_access_key = <AWS_SECRET_ACCESS_KEY>
'''

import boto.ec2
import os
import sys
import yaml
import time
import argparse
import threading
import calendar

from boto.exception import BotoClientError, BotoServerError, EC2ResponseError
from datetime import datetime, timedelta
from threading import Thread, Lock
from Queue import Queue, Empty
from getch import _Getch

class StoppableThread(Thread):
    '''
    Thread class with a stop() method. The thread itself has to check
    regularly for the stopped() condition.
    http://stackoverflow.com/questions/323972/is-there-any-way-to-kill-a-thread-in-python
    '''
    def __init__(self, *args, **kwargs):
        '''
        Pass init back to Thread and register stop event
        '''
        super(StoppableThread, self).__init__(*args, **kwargs)
        self._stop = threading.Event()

    def stop(self):
        '''
        Set stop on stop event
        '''
        self._stop.set()

    def stopped(self):
        '''
        Check if we are stopped
        '''
        return self._stop.isSet()

def kill_all_threads():
    '''
    Loop the global threads and tell each to stop.
    Join to block until they quit.
    '''
    print 'Bleeding out the queue'
    while not q.empty():
        try:
            q.get(False)
        except Empty:
            continue
        q.task_done()
    print 'Killing worker threads'
    # send the kill event without blocking
    for thread in threads:
        thread.stop()
    # block them all until they die
    # this may be better handled another way
    # but will work for now
    for thread in threads:
        thread.join()

def progress_bar(percent):
    '''
    Take percent and make a progress bar
    twenty characters wide
    '''
    complete = int(percent // 5)
    todo = int(20 - complete)
    bar_text = '[{0}{1}] {2}%'.format('#' * complete, '-' * todo, percent)
    return bar_text

def parse_args():
    '''
    Get arguments from the command like with argparse
    '''
    parser = argparse.ArgumentParser(
        description='Generate a list of hosts and generate/cleanup snapshots'
        )
    parser.add_argument(
        '--interactive', '-i',
        required=False,
        help='Interactive',
        action='store_true')
    parser.add_argument(
        '--days', '-d',
        required=False, type=int,
        help='Snapshot Keep Days',
        default='3')
    parser.add_argument(
        '--threads', '-t',
        required=False, type=int,
        help='Worker Threads',
        default='15')
    parser.add_argument(
        '--region', '-r',
        required=False, help='EC2 Region',
        default='us-west-2')
    args = parser.parse_args()
    return args

def generate_host_dict(instance, args):
    '''
    Generate the dictionary of hosts from Amazon EC2 that we do the rest of
    the work on.  Format should look like this:
    {
        hostname:
            {
                id: ec2 instance.id
                block_devices:
                    {
                        block_device_name: amazon_volume_id
                    }
                snapshot_keep_days: ec2 instance.tags['snapshot_keep_days']
            }

    }

    Also generate a dict used for progress bars that looks like this
    (very redundant):
    {
        hostname:
            {
                block_device_name (block_device_id): completed_percent
            }
    }
    '''
    instance_id = str(instance.id)
    if 'snapshot_keep_days' in instance.tags:
        snapshot_keep_days = int(
            instance.tags['snapshot_keep_days'])
    else:
        # Temporarily zero to test
        snapshot_keep_days = int(args.days)
    if 'Name' in instance.tags:
        hostname = str(instance.tags['Name'])
    else:
        hostname = instance_id
    block_device_mapping = instance.block_device_mapping
    block_devices = {}
    block_device_status = {}
    for block_device_name, block_device_type in block_device_mapping.items():
        # This seems terribly redundant, but we need a dict to use
        # and one for the current progress, so there we are.
        # Generate Block Devices for Queue
        block_devices.update(
            {str(block_device_name): str(block_device_type.volume_id)}
        )
        # Block devices for the status dict
        block_device_text = '{0} ({1})'.format(
            str(block_device_name),
            str(block_device_type.volume_id))
        block_device_status.update({block_device_text: int(0)})
    hosts.update({hostname: block_device_status})

    host = (
        {
            hostname:
                {
                    'id': instance_id,
                    'snapshot_keep_days': snapshot_keep_days,
                    'block_devices': block_devices
                }
        }
    )
    return host

def populate_queue(conn, args):
    '''
    Use generate_host_dict to make dictionary items to work on then populate
    the queue with them.

    Afterward get all snapshots who's expire dates are older than now.
    '''
    # Add all hosts in region to be backed up to queue
    print 'Populating queue with instances to snapshot...'
    reservations = conn.get_all_reservations()
    for reservation in reservations:
        instances = reservation.instances
        for instance in instances:
            if instance.state == 'running':
                host = generate_host_dict(instance, args)
                q.put(['create', host])
    # Add all expired snapshots to queue
    print 'Populating queue with snapshots to delete...'
    snapshots = conn.get_all_snapshots()
    for snapshot in snapshots:
        if 'expire_time_unix' in snapshot.tags.keys():
            # get now in unixtime
            now = calendar.timegm(datetime.utcnow().utctimetuple())
            if now > int(snapshot.tags['expire_time_unix']):
                print '{0} scheduled for deletion'.format(snapshot.description)
                q.put(['delete', snapshot])

def acquire_lock(fun, *args, **kwargs):
    '''
    Run lock and run the function so that we maintain thread
    safety on shared data.
    '''
    lock.acquire()
    try:
        return_info = fun(*args, **kwargs)
    finally:
        lock.release()
    return return_info

def make_gui_hosts(make_all=False):
    '''
    Take the hosts dict made in generate_hosts_dict and change the percent to
    pretty bars.  Optionally only show the in-progress hosts (make_all=False)

    Dict should look like this:
    {
        hostname:
            {
                bd_name (bd_id): [#####---------------] 25%
            }
    }

    '''
    gui_hosts = {}
    for host, block_devices in hosts.items():
        update_host = False
        gui_block_devices = {}
        for block_device, percent in block_devices.items():
            if (percent > 0 and percent < 100) or make_all:
                gui_block_devices.update({block_device: progress_bar(percent)})
                update_host = True
        if update_host:
            gui_hosts.update({host: gui_block_devices})
    return gui_hosts

def show_completed_count(action, length):
    '''
    Show which items returned successfully and are complete
    '''
    use_s = 's' if length is not 1 else ''
    if action == 'create':
        print 'Completed {0} host{1}'.format(length, use_s)
    elif action == 'delete':
        print 'Deleted {0} snapshot{1}'.format(length, use_s)

def draw_gui_hosts(make_all=False):
    '''
    Draw the output.  This will use yaml dump to write out the
    dict all nice and pretty.  It will also showed the completions
    and failures
    '''
    gui_hosts = make_gui_hosts(make_all)
    if gui_hosts:
        output = yaml.dump(dict(gui_hosts), default_flow_style=False)
    else:
        output = ''
    print output
    length = len(completed_list)
    show_completed_count('create', length)
    print 'Hosts: {0}'.format(', '.join(completed_list))
    print 'Failed Hosts: {0}'.format(', '.join(failed_list))
    length = len(deleted_list)
    show_completed_count('delete', length)
    print 'Deleted Snapshots: {0}'.format(','.join(deleted_list))
    print 'Failed Snapshot Deletions: {0}'.format(','.join(failed_delete_list))
    return gui_hosts

def interactive_watcher():
    '''
    Watcher for when we want to actively display the progress
    Auto draw gui hosts and whatnot.
    '''
    text = _Getch(timeout=1)
    character = None
    full = False
    # avoid race condition where threads have not spawned yet
    can_exit = False
    # Just us and the main thread means it is time to bail out
    while True:
        if threading.active_count() > 2:
            # Threads have spawned safe to exit
            can_exit = True
        if threading.active_count() <= 2 and can_exit:
            # exit out because they have now closed
            break
        character = text()
        os.system('cls' if sys.platform == 'win32' else 'clear')
        acquire_lock(draw_gui_hosts, make_all=full)
        print 'Press ESC to (q)uit or obtain (f)ull details.'
        if character == 'f':
            full = not full
        if character == 'q' or character == '\x1b':
            kill_all_threads()
            break
    print 'Exiting.'

def passive_watcher():
    '''
    Don't show progress bars.  Just output 'completed' when host complets
    '''
    print 'Non-interactive mode enabled.  You may still press ESC to (q)uit.'
    length = acquire_lock(len, completed_list)
    show_completed_count('create', length)
    delete_length = acquire_lock(len, deleted_list)
    show_completed_count('delete', delete_length)
    # avoid a race condition where threads have not spawned yet
    text = _Getch(timeout=1)
    character = None
    can_exit = False
    while True:
        character = text()
        if threading.active_count() > 2:
            can_exit = True
        old_length = length
        length = acquire_lock(len, completed_list)
        if old_length < length:
            show_completed_count('create', length)
        old_delete_length = delete_length
        delete_length = acquire_lock(len, deleted_list)
        if old_delete_length < delete_length:
            show_completed_count('delete', delete_length)
        # Stop if just the main thread and us.
        # queue.empty check misses some so do this.
        if threading.active_count() <= 2 and can_exit:
            break
        if character == 'q' or character == '\x1b':
            kill_all_threads()
            break

def create_snapshot(host, conn):
    '''
    This function does the heavy lifting.
    '''
    # Assume true unless issue
    success = True
    snapshots = []
    canupdate = False
    hostname, host_data = host.items()[0]
    # print hostname
    for block_device, volume_id in host_data['block_devices'].items():
        name = '{0} ({1}): {2}'.format(hostname, host_data['id'], volume_id)
        description = '{0} ({1})'.format(block_device, volume_id)
        try:
            snapshot = conn.create_snapshot(
                volume_id,
                description=description,
            )
            print 'Creating snapshot {0} named {1}'.format(
                str(snapshot.id), name)
            now = datetime.utcnow()
            expire_time = now + timedelta(
                days=host_data['snapshot_keep_days']
            )
            expire_time_unix = str(
                calendar.timegm(expire_time.utctimetuple())
            )
            expire_time_human = time.strftime(
                "%a, %d %b %Y %H:%M:%S +0000",
                expire_time.utctimetuple()
            )
            conn.create_tags(
                [snapshot.id],
                {
                    'Name': name,
                    'expire_time_unix': expire_time_unix,
                    'expire_time_human': expire_time_human,
                }
            )
            snapshots.extend([snapshot])
        except (BotoClientError, BotoServerError, EC2ResponseError):
            acquire_lock(failed_list.extend, [hostname])
            return
    while True:
        if threading.current_thread().stopped():
            break
        snapshots_complete = True
        block_device_status = {}
        for snapshot in snapshots:
            # snapshot = conn.get_all_snapshots([snapshot.id])[0]
            if canupdate:
                snapshot.update()
            else:
                canupdate = True
            percent_string = str(snapshot.progress)
            try:
                percent = int(percent_string[:-1])
            except ValueError:
                # show at least 1% to populate progress bars
                percent = 1
            if snapshot.status == 'pending':
                snapshots_complete = False
            elif snapshot.status == 'error':
                success = False
            bd_text = str(snapshot.description)
            block_device_status.update({bd_text: percent})
        status = {hostname: block_device_status}
        acquire_lock(hosts.update, status)
        if snapshots_complete:
            if success:
                acquire_lock(completed_list.extend, [hostname])
                break
            else:
                acquire_lock(failed_list.extend, [hostname])
                break
        time.sleep(30)

def delete_snapshot(snapshot):
    '''
    Delete a snapshot
    '''
    # This is a pretty short-running function, but if the event says we should
    # be stopped, then don't run
    if threading.current_thread().stopped():
        return False
    # Obtain information for logging
    if 'Name' in snapshot.tags.keys():
        name = str(snapshot.tags['Name'])
    else:
        name = str(snapshot.description)
    # Delete the snapshot
    deleted = snapshot.delete()
    if deleted:
        acquire_lock(deleted_list.extend, [name])
    else:
        acquire_lock(failed_delete_list.extend, [name])
    # Throttle to keep from hammering api
    time.sleep(30)
    return deleted

def worker(conn):
    '''
    Worker thread. Does 3 things:
        1. Grab an item from the queue.
        2. Send it through the correct processor.
        3. Update queue status.
    '''
    while not (q.empty() or threading.current_thread().stopped()):
        item = q.get()
        if item[0] == 'create':
            create_snapshot(item[1], conn)
        elif item[0] == 'delete':
            delete_snapshot(item[1])
        q.task_done()

def main():
    '''
    Main thread.  Adds shared memory globals and spawns the other threads.
    '''
    global q
    q = Queue()
    global hosts
    hosts = dict()
    global lock
    lock = Lock()
    global completed_list
    completed_list = []
    global failed_list
    failed_list = []
    global threads
    threads = []
    global deleted_list
    deleted_list = []
    global failed_delete_list
    failed_delete_list = []
    args = parse_args()
    conn = boto.ec2.connect_to_region(args.region)
    populate_queue(conn, args)
    print 'Initial Queue Size {0}'.format(int(q.qsize()))
    if args.interactive:
        watcher = Thread(target=interactive_watcher)
    else:
        watcher = Thread(target=passive_watcher)
    watcher.daemon = False
    watcher.start()
    num_worker_threads = args.threads
    for _ in range(num_worker_threads):
        thread = StoppableThread(target=worker, args=(conn,))
        thread.daemon = False
        thread.start()
        threads.extend([thread])
    q.join()
    watcher.join()
    acquire_lock(draw_gui_hosts, make_all=True)

if __name__ == '__main__':
    sys.exit(main())
<|endoftext|>