<commit_before>import sublime
import sublime_plugin
import re
import os

try:
	# ST2, Python 2
	from HTMLParser import HTMLParser
except:
	# ST3, Python 3
	from html.parser import HTMLParser

class EcsstractorCommand(sublime_plugin.WindowCommand):
	def run(self, bem_nesting = "default"):

		view = self.window.active_view()

		plugin_settings = sublime.load_settings('eCSStractor.sublime-settings')
		self.brackets = plugin_settings.get('brackets')

		if bem_nesting is "default":
			bem_nesting = plugin_settings.get('bem_nesting')

		syntax = 'Packages/CSS/CSS.tmLanguage'

		# if view have any selection then work with selection, else with whole view
		selection = view.sel()[0]

		if len(selection) > 0:

			for sel in view.sel():
				region = sublime.Region(
					view.line(min(sel.a, sel.b)).a,  # line start of first line
					view.line(max(sel.a, sel.b)).b   # line end of last line
				)

		else:
			region = sublime.Region(0, view.size())

		self.source = view.substr(region)

		if bem_nesting:
			output = self.generateBEM()

			# set sass syntax if proper package is installed
			scss_syntax = os.path.join(sublime.packages_path(), 'Syntax Highlighting for Sass', 'Syntaxes', 'SCSS.tmLanguage')

			if os.path.exists(scss_syntax):

				syntax = 'Packages/Syntax Highlighting for Sass/Syntaxes/SCSS.tmLanguage'

		else:
			output = self.generateOutput()

		# if output not empty
		if output:
			# create new view
			new_file = self.window.new_file()
			new_file.set_syntax_file(syntax)
			new_file.run_command("ecsstractor_insert", {"text": output})
		else:
			sublime.status_message("eCSStractor can't find any classes")

	def generateOutput(self):

		css_template = "selector {\n}"
		output = ""

		# parsing
		parsed = parser()
		parsed.feed(self.source)

		# format output
		for i in range(len(parsed.classes)):
			output += css_template.replace("selector", "." + parsed.classes[i]) + "\n"

		return output

	def generateBEM(self):

		plugin_settings = sublime.load_settings('eCSStractor.sublime-settings')
		indentation = plugin_settings.get('indentation', '\t')
		element_separator = plugin_settings.get('bem.element_separator', '__')
		modifier_separator = plugin_settings.get('bem.modifier_separator', '--')
		parent_symbol = plugin_settings.get('preprocessor.parent_symbol', '&')

		output = ""
		selectors = []

		# parsing
		parsed = parser()
		parsed.feed(self.source)

		# build tree
		for i in range(len(parsed.classes)):
			selector = parsed.classes[i]

			block = {}
			element = {}

			# if block has element
			if element_separator in selector:

				parts = selector.split(element_separator)

				# check if block with this name exist already
				hasBlock = self.hasChild(selectors, "name", parts[0])

				# if block is exist link to list
				if hasBlock is not False:
					block = selectors[hasBlock]

				# if block is not exist give it name
				if hasBlock is False:
					block["name"] = parts[0]

				# if elements list exist in block
				if "elements" not in block:
					block["elements"] = []

				# get element and his modifier
				elementParts = parts[1].split(modifier_separator)

				# check if element with this name exist in block already
				hasElement = self.hasChild(block["elements"], "name", elementParts[0])

				# if element is exist link to list
				if hasElement is not False:
					element = block["elements"][hasElement]

				# if element is not exist give it name
				if hasElement is False:
					element["name"] = elementParts[0]

				# if element has modifier
				if len(elementParts) > 1:

					# if modifiers list exist in element
					if "modifiers" not in element:
						element["modifiers"] = []

					# add modifier
					element["modifiers"].append(elementParts[1])

				# if it is new element add it to block
				if hasElement is False:
					block["elements"].append(element)

				# if it is new block add it to list
				if hasBlock is False:
					selectors.append(block)

			# if block has modifier
			elif modifier_separator in selector:

				parts = selector.split(modifier_separator)

				hasBlock = self.hasChild(selectors, "name", parts[0])

				if hasBlock is not False:
					block = selectors[hasBlock]

				if hasBlock is False:
					block["name"] = parts[0]

				if "modifiers" not in block:
					block["modifiers"] = []

				block["modifiers"].append(parts[1])

				if hasBlock is False:
					selectors.append(block)

			else:

				hasBlock = self.hasChild(selectors, "name", selector)

				if hasBlock is False:
					block["name"] = selector
					selectors.append(block)

		# format output
		for block in selectors:

			if self.brackets:
				output += "." + block["name"] + " {\n"
			else:
				output += "." + block["name"] + "\n"

			indent = indentation
			indent1 = indent * 1
			indent2 = indent * 2

			if "modifiers" in block:

				for modifier in block["modifiers"]:
					if self.brackets:
						output += indent1 + parent_symbol + modifier_separator + modifier + " {\n"
						output += indent1 + "}\n"
					else:
						output += indent1 + parent_symbol + modifier_separator + modifier + "\n"
						output += "\n"

			if "elements" in block:

				for element in block["elements"]:
					if self.brackets:
						output += indent1 + parent_symbol + element_separator + element["name"] + " {\n"
					else:
						output += indent1 + parent_symbol + element_separator + element["name"] + "\n"

					if "modifiers" in element:

						for modifier in element["modifiers"]:
							if self.brackets:
								output += indent2 + parent_symbol + modifier_separator + modifier + " {\n"
								output += indent2 + "}\n"
							else:
								output += indent2 + parent_symbol + modifier_separator + modifier + "\n"
								output += "\n"

					if self.brackets:
						output += indent1 + "}\n"
					else:
						output += "\n"

			if self.brackets:
				output += "}\n"
			else:
				output += "\n"

		if not self.brackets:
			output = output.replace("\n\n\n\n", "\n\n")
			output = output.replace("\n\n\n", "\n\n")

		return output

	# check existance of key_name:key in listo
	def hasChild(self, listo, key_name, key):

		for y in range(len(listo)):
			if listo[y][key_name] == key:
				return y

		return False

class EcsstractorInsertCommand(sublime_plugin.TextCommand):
	def run(self, edit, text):
		self.view.insert(edit, 0, text)

class parser(HTMLParser):
	def __init__(self):
		HTMLParser.__init__(self)
		self.classes = []

	def handle_starttag(self, tag, attrs):

		ignore = sublime.load_settings('eCSStractor.sublime-settings').get('ignore', "")
		ignore_regex = sublime.load_settings('eCSStractor.sublime-settings').get('ignore_regex', "")

		for name, value in attrs:

			if name == "class":

				# remove whitespaces before and after string
				value = value.strip();

				# if class="" not empty
				if len(value) > 0:

					# split class string by whitespaces, in case multiple classes presented
					elementClasses = re.split("\s+", value)

					for i in range(len(elementClasses)):

						currentClass = elementClasses[i]

						# possible add class to list if it's already not there and not in ignore list
						if currentClass not in (self.classes and ignore):

							# check if it's pass regex ignore list check
							itspass = True

							for y in range(len(ignore_regex)):

								if re.compile(ignore_regex[y]).match(currentClass):
									itspass = False
									print(currentClass + " — " + ignore_regex[y])
									break

							# add class to list if it's not already in the list, and pass both ignore list checks
							if itspass:
								self.classes.append(currentClass)
<commit_msg>Fix broken uniqueness in classes list<commit_after>import sublime
import sublime_plugin
import re
import os

try:
	# ST2, Python 2
	from HTMLParser import HTMLParser
except:
	# ST3, Python 3
	from html.parser import HTMLParser

class EcsstractorCommand(sublime_plugin.WindowCommand):
	def run(self, bem_nesting = "default"):

		view = self.window.active_view()

		plugin_settings = sublime.load_settings('eCSStractor.sublime-settings')
		self.brackets = plugin_settings.get('brackets')

		if bem_nesting is "default":
			bem_nesting = plugin_settings.get('bem_nesting')

		syntax = 'Packages/CSS/CSS.tmLanguage'

		# if view have any selection then work with selection, else with whole view
		selection = view.sel()[0]

		if len(selection) > 0:

			for sel in view.sel():
				region = sublime.Region(
					view.line(min(sel.a, sel.b)).a,  # line start of first line
					view.line(max(sel.a, sel.b)).b   # line end of last line
				)

		else:
			region = sublime.Region(0, view.size())

		self.source = view.substr(region)

		if bem_nesting:
			output = self.generateBEM()

			# set sass syntax if proper package is installed
			scss_syntax = os.path.join(sublime.packages_path(), 'Syntax Highlighting for Sass', 'Syntaxes', 'SCSS.tmLanguage')

			if os.path.exists(scss_syntax):

				syntax = 'Packages/Syntax Highlighting for Sass/Syntaxes/SCSS.tmLanguage'

		else:
			output = self.generateOutput()

		# if output not empty
		if output:
			# create new view
			new_file = self.window.new_file()
			new_file.set_syntax_file(syntax)
			new_file.run_command("ecsstractor_insert", {"text": output})
		else:
			sublime.status_message("eCSStractor can't find any classes")

	def generateOutput(self):

		css_template = "selector {\n}"
		output = ""

		# parsing
		parsed = parser()
		parsed.feed(self.source)

		# format output
		for i in range(len(parsed.classes)):
			output += css_template.replace("selector", "." + parsed.classes[i]) + "\n"

		return output

	def generateBEM(self):

		plugin_settings = sublime.load_settings('eCSStractor.sublime-settings')
		indentation = plugin_settings.get('indentation', '\t')
		element_separator = plugin_settings.get('bem.element_separator', '__')
		modifier_separator = plugin_settings.get('bem.modifier_separator', '--')
		parent_symbol = plugin_settings.get('preprocessor.parent_symbol', '&')

		output = ""
		selectors = []

		# parsing
		parsed = parser()
		parsed.feed(self.source)

		# build tree
		for i in range(len(parsed.classes)):
			selector = parsed.classes[i]

			block = {}
			element = {}

			# if block has element
			if element_separator in selector:

				parts = selector.split(element_separator)

				# check if block with this name exist already
				hasBlock = self.hasChild(selectors, "name", parts[0])

				# if block is exist link to list
				if hasBlock is not False:
					block = selectors[hasBlock]

				# if block is not exist give it name
				if hasBlock is False:
					block["name"] = parts[0]

				# if elements list exist in block
				if "elements" not in block:
					block["elements"] = []

				# get element and his modifier
				elementParts = parts[1].split(modifier_separator)

				# check if element with this name exist in block already
				hasElement = self.hasChild(block["elements"], "name", elementParts[0])

				# if element is exist link to list
				if hasElement is not False:
					element = block["elements"][hasElement]

				# if element is not exist give it name
				if hasElement is False:
					element["name"] = elementParts[0]

				# if element has modifier
				if len(elementParts) > 1:

					# if modifiers list exist in element
					if "modifiers" not in element:
						element["modifiers"] = []

					# add modifier
					element["modifiers"].append(elementParts[1])

				# if it is new element add it to block
				if hasElement is False:
					block["elements"].append(element)

				# if it is new block add it to list
				if hasBlock is False:
					selectors.append(block)

			# if block has modifier
			elif modifier_separator in selector:

				parts = selector.split(modifier_separator)

				hasBlock = self.hasChild(selectors, "name", parts[0])

				if hasBlock is not False:
					block = selectors[hasBlock]

				if hasBlock is False:
					block["name"] = parts[0]

				if "modifiers" not in block:
					block["modifiers"] = []

				block["modifiers"].append(parts[1])

				if hasBlock is False:
					selectors.append(block)

			else:

				hasBlock = self.hasChild(selectors, "name", selector)

				if hasBlock is False:
					block["name"] = selector
					selectors.append(block)

		# format output
		for block in selectors:

			if self.brackets:
				output += "." + block["name"] + " {\n"
			else:
				output += "." + block["name"] + "\n"

			indent = indentation
			indent1 = indent * 1
			indent2 = indent * 2

			if "modifiers" in block:

				for modifier in block["modifiers"]:
					if self.brackets:
						output += indent1 + parent_symbol + modifier_separator + modifier + " {\n"
						output += indent1 + "}\n"
					else:
						output += indent1 + parent_symbol + modifier_separator + modifier + "\n"
						output += "\n"

			if "elements" in block:

				for element in block["elements"]:
					if self.brackets:
						output += indent1 + parent_symbol + element_separator + element["name"] + " {\n"
					else:
						output += indent1 + parent_symbol + element_separator + element["name"] + "\n"

					if "modifiers" in element:

						for modifier in element["modifiers"]:
							if self.brackets:
								output += indent2 + parent_symbol + modifier_separator + modifier + " {\n"
								output += indent2 + "}\n"
							else:
								output += indent2 + parent_symbol + modifier_separator + modifier + "\n"
								output += "\n"

					if self.brackets:
						output += indent1 + "}\n"
					else:
						output += "\n"

			if self.brackets:
				output += "}\n"
			else:
				output += "\n"

		if not self.brackets:
			output = output.replace("\n\n\n\n", "\n\n")
			output = output.replace("\n\n\n", "\n\n")

		return output

	# check existance of key_name:key in listo
	def hasChild(self, listo, key_name, key):

		for y in range(len(listo)):
			if listo[y][key_name] == key:
				return y

		return False

class EcsstractorInsertCommand(sublime_plugin.TextCommand):
	def run(self, edit, text):
		self.view.insert(edit, 0, text)

class parser(HTMLParser):
	def __init__(self):
		HTMLParser.__init__(self)
		self.classes = []

	def handle_starttag(self, tag, attrs):

		ignore = sublime.load_settings('eCSStractor.sublime-settings').get('ignore', "")
		ignore_regex = sublime.load_settings('eCSStractor.sublime-settings').get('ignore_regex', "")

		for name, value in attrs:

			if name == "class":

				# remove whitespaces before and after string
				value = value.strip();

				# if class="" not empty
				if len(value) > 0:

					# split class string by whitespaces, in case multiple classes presented
					elementClasses = re.split("\s+", value)

					for i in range(len(elementClasses)):

						currentClass = elementClasses[i]

						# possible add class to list if it's already not there and not in ignore list
						if currentClass not in self.classes and currentClass not in ignore:

							# check if it's pass regex ignore list check
							itspass = True

							for y in range(len(ignore_regex)):

								if re.compile(ignore_regex[y]).match(currentClass):
									itspass = False
									break

							# add class to list if it's not already in the list, and pass both ignore list checks
							if itspass:
								self.classes.append(currentClass)
<|endoftext|>


<commit_before># vim:ts=4:sts=4:sw=4:expandtab
import logging
from blist import sortedlist
from satori.core.checking.utils import RestTable
from satori.core.checking.aggregators import parse_params
from satori.core.models import TestMapping

class ReporterBase(object):
    def __init__(self, test_suite_result):
        super(ReporterBase, self).__init__()
        self.test_suite_result = test_suite_result
        params_map = self.test_suite_result.test_suite.params_get_map()
        new_params_map = {}
        for param_name in params_map:
            if param_name.startswith(self.__class__.__name__ + '.'):
                new_params_map[param_name[len(self.__class__.__name__) + 1:]] = params_map[param_name]
        self.params = parse_params(self.__doc__, 'reporter', 'general', new_params_map)

    def init(self):
        pass

    def accumulate(self, test_result):
        pass

    def status(self):
        return True

    def deinit(self):
        pass

class AssignmentReporter(ReporterBase):
    def __init__(self, test_suite_result):
        super(AssignmentReporter, self).__init__(test_suite_result)

    def init(self):
        self.test_suite_result.oa_set_str('status', 'ACC')
        self.test_suite_result.status = 'ACC'
        self.test_suite_result.report = ''
        self.test_suite_result.save()

    def accumulate(self, test_result):
        pass

    def status(self):
        return True

    def deinit(self):
        status = 'ACC'
        report = ''
        oa_map = self.test_suite_result.submit.overrides_get_map()
        ostatus = oa_map.get('status', None)
        if ostatus is not None and not ostatus.is_blob:
            status = ostatus.value
        oreport = oa_map.get('report', None)
        if oreport is not None and not oreport.is_blob:
            report = oreport.value
        self.test_suite_result.status = status
        self.test_suite_result.report = report
        self.test_suite_result.save()
        self.test_suite_result.oa_set_map(oa_map)

class StatusReporter(ReporterBase):
    """
#@<reporter name="Status reporter">
#@      <general>
#@      </general>
#@</reporter>
    """
    def __init__(self, test_suite_result):
        super(StatusReporter, self).__init__(test_suite_result)

    def init(self):
        self._status = 'OK'
        self.test_suite_result.oa_set_str('status', 'QUE')
        self.test_suite_result.status = 'QUE'
        self.test_suite_result.report = ''
        self.test_suite_result.save()

    def accumulate(self, test_result):
        status = test_result.oa_get_str('status')
        logging.debug('Status Reporter %s: %s += %s', self.test_suite_result.id, self._status, status)
        if status is None:
            status = 'INT'
        if self._status == 'OK' and status != 'OK':
            self._status = status

    def status(self):
        return self._status == 'OK'

    def deinit(self):
        logging.debug('Status Reporter %s: %s', self.test_suite_result.id, self._status)
        status = self._status
        os = self.test_suite_result.submit.overrides_get('status')
        if os is not None and not os.is_blob:
            status = os.value
        self.test_suite_result.oa_set_str('status', status)
        self.test_suite_result.status = status
        self.test_suite_result.report = 'Finished checking: {0}'.format(self._status)
        self.test_suite_result.save()

class MultipleStatusReporter(ReporterBase):
    def __init__(self, test_suite_result):
        super(MultipleStatusReporter, self).__init__(test_suite_result)

    def init(self):
        self._statuses = {}
        self._status = 'OK'
        self.test_suite_result.oa_set_str('status', 'QUE')
        self.test_suite_result.status = 'QUE'
        self.test_suite_result.report = ''
        self.test_suite_result.save()

    def accumulate(self, test_result):
        test = test_result.test
        code = TestMapping.objects().filter(test=test, suite= self.test_suite_result.suite)[0].order
        status = test_result.oa_get_str('status')
        self._statuses[code] = status
        logging.debug('Status Reporter %s: %s += %s', self.test_suite_result.id, self._status, status)
        if status is None:
            status = 'INT'
        if self._status == 'OK' and status != 'OK':
            self._status = status

    def status(self):
        return True

    def deinit(self):
        logging.debug('Status Reporter %s: %s', self.test_suite_result.id, self._status)
        status = self._status
        os = self.test_suite_result.submit.overrides_get('status')
        if os is not None and not os.is_blob:
            status = os.value
        self.test_suite_result.oa_set_str('status', status)
        self.test_suite_result.status = status
        report = u'Finished checking: {0}'.format(self._status)
        report += ' (' + u', '.join([unicode(code) + ' ' + unicode(status) for (code, status) in self._statuses.objects().sorted()]) + ')'
        self.test_suite_result.report = report
        self.test_suite_result.save()

class ACMReporter(ReporterBase):
    """
#@<reporter name="ACM style reporter">
#@      <general>
#@              <param type="bool"     name="show_tests"     description="Show individual test results" default="true"/>
#@      </general>
#@</reporter>
    """

    columns = [('status', 'Status'), ('execute_time_cpu', 'CPU time'), ('execute_time_gpu', 'GPU time'), ('message', 'Message')]

    def __init__(self, test_suite_result):
        super(ACMReporter, self).__init__(test_suite_result)

    def init(self):
        self._names = {}
        self._codes = []
        self._column_values = {}
        self._column_nonempty = {}
        self._status = 'OK'
        for (column_id, column_name) in self.columns:
        	self._column_values[column_id] = {}
        	self._column_nonempty[column_id] = False
        self.test_suite_result.oa_set_str('status', 'QUE')
        self.test_suite_result.status = 'QUE'
        self.test_suite_result.report = ''
        self.test_suite_result.save()

    def accumulate(self, test_result):
        test = test_result.test
        code = TestMapping.objects.get(test=test, suite=self.test_suite_result.test_suite).order
        status = test_result.oa_get_str('status')
        for (column_id, column_name) in self.columns:
            value = test_result.oa_get_str(column_id)
            if value is not None:
            	self._column_values[column_id][code] = value
            	self._column_nonempty[column_id] = True
            else:
            	self._column_values[column_id][code] = ''
        self._codes.append(code)
        self._names[code] = test_result.test.name
        logging.debug('ACM Reporter %s: %s += %s', self.test_suite_result.id, self._status, status)
        if status is None:
            status = 'INT'
        if self._status == 'OK' and status != 'OK':
            self._status = status

    def status(self):
        return self._status == 'OK'

    def deinit(self):
        logging.debug('ACM Reporter %s: %s', self.test_suite_result.id, self._status)
        status = self._status
        os = self.test_suite_result.submit.overrides_get('status')
        if os is not None and not os.is_blob:
            status = os.value
        self.test_suite_result.oa_set_str('status', status)
        self.test_suite_result.status = status
        if self.params.show_tests:
        	columns = [(20, 'Test')]
            for (column_id, column_name) in self.columns:
                if self._column_nonempty[column_id]:
                	columns.append((15, column_name))
            table = RestTable(*columns)
            report = table.row_separator + table.header_row + table.header_separator
            for code in sorted(self._codes):
            	values = [self._names[code]]
                for (column_id, column_name) in self.columns:
                    if self._column_nonempty[column_id]:
                        values.append(self._column_values[column_id][code])
                report += table.generate_row(*values) + table.row_separator
        else:
            report = ''
        self.test_suite_result.report = report
        self.test_suite_result.save()

class PointsReporter(ReporterBase):
    """
#@<reporter name="Points reporter">
#@      <general>
#@              <param type="bool"     name="show_tests"     description="Show individual test results" default="true"/>
#@      </general>
#@</reporter>
    """
    def __init__(self, test_suite_result):
        super(PointsReporter, self).__init__(test_suite_result)

    def init(self):
        self.checked = 0
        self.passed = 0
        self._status = ''
        self.reportlines = sortedlist(key=lambda row: row[0])
        self.test_suite_result.status = 'QUE'
        self.test_suite_result.report = ''
        self.test_suite_result.save()

    def accumulate(self, test_result):
        test = test_result.test
        code = TestMapping.objects.get(test=test, suite=self.test_suite_result.test_suite).order
        result = test_result.oa_get_str('status')
        self.checked = self.checked+1
        if result is None:
            result = 'INT'
        if result == 'OK':
            self.passed = self.passed+1
        self._status = str(self.passed) + ' / '+ str(self.checked)
        self.reportlines.add([test_result.test.name,result,test.result_oa_get_str('execute_time_cpu')+ ' / ' +test.data_get_str('time')])
        self.test_suite_result.save()

    def status(self):
        return True

    def deinit(self):
        if self.params.show_tests:
        	columns = [(20, 'Test'), (10, 'Status'), (20,'Time')]
            table = RestTable(*columns)
            report = table.row_separator + table.header_row + table.header_separator
            for line in self.reportlines:
                values = line
                report += table.generate_row(*values) + table.row_separator
        else:
            report = ''
        self.test_suite_result.report = report
        self.test_suite_result.status = self._status
        self.test_suite_result.oa_set_str('status', self._status)
        self.test_suite_result.oa_set_str('checked', self.checked)
        self.test_suite_result.oa_set_str('passed', self.passed)
        self.test_suite_result.save()

reporters = {}
for item in globals().values():
    if isinstance(item, type) and issubclass(item, ReporterBase) and (item != ReporterBase):
        reporters[item.__name__] = item
<commit_msg>typo.<commit_after># vim:ts=4:sts=4:sw=4:expandtab
import logging
from blist import sortedlist
from satori.core.checking.utils import RestTable
from satori.core.checking.aggregators import parse_params
from satori.core.models import TestMapping

class ReporterBase(object):
    def __init__(self, test_suite_result):
        super(ReporterBase, self).__init__()
        self.test_suite_result = test_suite_result
        params_map = self.test_suite_result.test_suite.params_get_map()
        new_params_map = {}
        for param_name in params_map:
            if param_name.startswith(self.__class__.__name__ + '.'):
                new_params_map[param_name[len(self.__class__.__name__) + 1:]] = params_map[param_name]
        self.params = parse_params(self.__doc__, 'reporter', 'general', new_params_map)

    def init(self):
        pass

    def accumulate(self, test_result):
        pass

    def status(self):
        return True

    def deinit(self):
        pass

class AssignmentReporter(ReporterBase):
    def __init__(self, test_suite_result):
        super(AssignmentReporter, self).__init__(test_suite_result)

    def init(self):
        self.test_suite_result.oa_set_str('status', 'ACC')
        self.test_suite_result.status = 'ACC'
        self.test_suite_result.report = ''
        self.test_suite_result.save()

    def accumulate(self, test_result):
        pass

    def status(self):
        return True

    def deinit(self):
        status = 'ACC'
        report = ''
        oa_map = self.test_suite_result.submit.overrides_get_map()
        ostatus = oa_map.get('status', None)
        if ostatus is not None and not ostatus.is_blob:
            status = ostatus.value
        oreport = oa_map.get('report', None)
        if oreport is not None and not oreport.is_blob:
            report = oreport.value
        self.test_suite_result.status = status
        self.test_suite_result.report = report
        self.test_suite_result.save()
        self.test_suite_result.oa_set_map(oa_map)

class StatusReporter(ReporterBase):
    """
#@<reporter name="Status reporter">
#@      <general>
#@      </general>
#@</reporter>
    """
    def __init__(self, test_suite_result):
        super(StatusReporter, self).__init__(test_suite_result)

    def init(self):
        self._status = 'OK'
        self.test_suite_result.oa_set_str('status', 'QUE')
        self.test_suite_result.status = 'QUE'
        self.test_suite_result.report = ''
        self.test_suite_result.save()

    def accumulate(self, test_result):
        status = test_result.oa_get_str('status')
        logging.debug('Status Reporter %s: %s += %s', self.test_suite_result.id, self._status, status)
        if status is None:
            status = 'INT'
        if self._status == 'OK' and status != 'OK':
            self._status = status

    def status(self):
        return self._status == 'OK'

    def deinit(self):
        logging.debug('Status Reporter %s: %s', self.test_suite_result.id, self._status)
        status = self._status
        os = self.test_suite_result.submit.overrides_get('status')
        if os is not None and not os.is_blob:
            status = os.value
        self.test_suite_result.oa_set_str('status', status)
        self.test_suite_result.status = status
        self.test_suite_result.report = 'Finished checking: {0}'.format(self._status)
        self.test_suite_result.save()

class MultipleStatusReporter(ReporterBase):
    def __init__(self, test_suite_result):
        super(MultipleStatusReporter, self).__init__(test_suite_result)

    def init(self):
        self._statuses = {}
        self._status = 'OK'
        self.test_suite_result.oa_set_str('status', 'QUE')
        self.test_suite_result.status = 'QUE'
        self.test_suite_result.report = ''
        self.test_suite_result.save()

    def accumulate(self, test_result):
        test = test_result.test
        code = TestMapping.objects().filter(test=test, suite= self.test_suite_result.suite)[0].order
        status = test_result.oa_get_str('status')
        self._statuses[code] = status
        logging.debug('Status Reporter %s: %s += %s', self.test_suite_result.id, self._status, status)
        if status is None:
            status = 'INT'
        if self._status == 'OK' and status != 'OK':
            self._status = status

    def status(self):
        return True

    def deinit(self):
        logging.debug('Status Reporter %s: %s', self.test_suite_result.id, self._status)
        status = self._status
        os = self.test_suite_result.submit.overrides_get('status')
        if os is not None and not os.is_blob:
            status = os.value
        self.test_suite_result.oa_set_str('status', status)
        self.test_suite_result.status = status
        report = u'Finished checking: {0}'.format(self._status)
        report += ' (' + u', '.join([unicode(code) + ' ' + unicode(status) for (code, status) in self._statuses.objects().sorted()]) + ')'
        self.test_suite_result.report = report
        self.test_suite_result.save()

class ACMReporter(ReporterBase):
    """
#@<reporter name="ACM style reporter">
#@      <general>
#@              <param type="bool"     name="show_tests"     description="Show individual test results" default="true"/>
#@      </general>
#@</reporter>
    """

    columns = [('status', 'Status'), ('execute_time_cpu', 'CPU time'), ('execute_time_gpu', 'GPU time'), ('message', 'Message')]

    def __init__(self, test_suite_result):
        super(ACMReporter, self).__init__(test_suite_result)

    def init(self):
        self._names = {}
        self._codes = []
        self._column_values = {}
        self._column_nonempty = {}
        self._status = 'OK'
        for (column_id, column_name) in self.columns:
        	self._column_values[column_id] = {}
        	self._column_nonempty[column_id] = False
        self.test_suite_result.oa_set_str('status', 'QUE')
        self.test_suite_result.status = 'QUE'
        self.test_suite_result.report = ''
        self.test_suite_result.save()

    def accumulate(self, test_result):
        test = test_result.test
        code = TestMapping.objects.get(test=test, suite=self.test_suite_result.test_suite).order
        status = test_result.oa_get_str('status')
        for (column_id, column_name) in self.columns:
            value = test_result.oa_get_str(column_id)
            if value is not None:
            	self._column_values[column_id][code] = value
            	self._column_nonempty[column_id] = True
            else:
            	self._column_values[column_id][code] = ''
        self._codes.append(code)
        self._names[code] = test_result.test.name
        logging.debug('ACM Reporter %s: %s += %s', self.test_suite_result.id, self._status, status)
        if status is None:
            status = 'INT'
        if self._status == 'OK' and status != 'OK':
            self._status = status

    def status(self):
        return self._status == 'OK'

    def deinit(self):
        logging.debug('ACM Reporter %s: %s', self.test_suite_result.id, self._status)
        status = self._status
        os = self.test_suite_result.submit.overrides_get('status')
        if os is not None and not os.is_blob:
            status = os.value
        self.test_suite_result.oa_set_str('status', status)
        self.test_suite_result.status = status
        if self.params.show_tests:
        	columns = [(20, 'Test')]
            for (column_id, column_name) in self.columns:
                if self._column_nonempty[column_id]:
                	columns.append((15, column_name))
            table = RestTable(*columns)
            report = table.row_separator + table.header_row + table.header_separator
            for code in sorted(self._codes):
            	values = [self._names[code]]
                for (column_id, column_name) in self.columns:
                    if self._column_nonempty[column_id]:
                        values.append(self._column_values[column_id][code])
                report += table.generate_row(*values) + table.row_separator
        else:
            report = ''
        self.test_suite_result.report = report
        self.test_suite_result.save()

class PointsReporter(ReporterBase):
    """
#@<reporter name="Points reporter">
#@      <general>
#@              <param type="bool"     name="show_tests"     description="Show individual test results" default="true"/>
#@      </general>
#@</reporter>
    """
    def __init__(self, test_suite_result):
        super(PointsReporter, self).__init__(test_suite_result)

    def init(self):
        self.checked = 0
        self.passed = 0
        self._status = ''
        self.reportlines = sortedlist(key=lambda row: row[0])
        self.test_suite_result.status = 'QUE'
        self.test_suite_result.report = ''
        self.test_suite_result.save()

    def accumulate(self, test_result):
        test = test_result.test
        code = TestMapping.objects.get(test=test, suite=self.test_suite_result.test_suite).order
        result = test_result.oa_get_str('status')
        self.checked = self.checked+1
        if result is None:
            result = 'INT'
        if result == 'OK':
            self.passed = self.passed+1
        self._status = str(self.passed) + ' / '+ str(self.checked)
        self.reportlines.add([test_result.test.name,result,test_result.oa_get_str('execute_time_cpu')+ ' / ' +test.data_get_str('time')])
        self.test_suite_result.save()

    def status(self):
        return True

    def deinit(self):
        if self.params.show_tests:
        	columns = [(20, 'Test'), (10, 'Status'), (20,'Time')]
            table = RestTable(*columns)
            report = table.row_separator + table.header_row + table.header_separator
            for line in self.reportlines:
                values = line
                report += table.generate_row(*values) + table.row_separator
        else:
            report = ''
        self.test_suite_result.report = report
        self.test_suite_result.status = self._status
        self.test_suite_result.oa_set_str('status', self._status)
        self.test_suite_result.oa_set_str('checked', self.checked)
        self.test_suite_result.oa_set_str('passed', self.passed)
        self.test_suite_result.save()

reporters = {}
for item in globals().values():
    if isinstance(item, type) and issubclass(item, ReporterBase) and (item != ReporterBase):
        reporters[item.__name__] = item
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from distutils.version import LooseVersion
from django import get_version

import os

django_version = LooseVersion(get_version())

HELPER_SETTINGS = {
    'TIME_ZONE': 'Europe/Zurich',
    'LANGUAGES': (
        ('en', 'English'),
        ('de', 'German'),
        ('fr', 'French'),
    ),
    'INSTALLED_APPS': [
        'aldryn_apphook_reload',
        'aldryn_apphooks_config',
        'aldryn_boilerplates',
        'aldryn_categories',
        'aldryn_newsblog',
        'aldryn_people',
        'aldryn_reversion',
        'djangocms_text_ckeditor',
        'easy_thumbnails',
        'filer',
        'mptt',
        'parler',
        'reversion',
        'sortedm2m',
        'taggit',
    ],
    'STATICFILES_FINDERS': [
        'django.contrib.staticfiles.finders.FileSystemFinder',
        # important! place right before:
        #     django.contrib.staticfiles.finders.AppDirectoriesFinder
        'aldryn_boilerplates.staticfile_finders.AppDirectoriesFinder',
        'django.contrib.staticfiles.finders.AppDirectoriesFinder',
    ],
    'ALDRYN_NEWSBLOG_TEMPLATE_PREFIXES': [('dummy', 'dummy'), ],
    'ALDRYN_BOILERPLATE_NAME': 'bootstrap3',
    # app-specific
    'PARLER_LANGUAGES': {
        1: (
            {'code': 'de', },
            {'code': 'fr', },
            {'code': 'en', },
        ),
        'default': {
            'hide_untranslated': True,  # PLEASE DO NOT CHANGE THIS
        }
    },
    'SITE_ID': 1,
    'CMS_LANGUAGES': {
        1: [
            {
                'code': 'de',
                'name': 'Deutsche',
                'fallbacks': ['en', ]  # FOR TESTING DO NOT ADD 'fr' HERE
            },
            {
                'code': 'fr',
                'name': u'Française',
                'fallbacks': ['en', ]  # FOR TESTING DO NOT ADD 'de' HERE
            },
            {
                'code': 'en',
                'name': 'English',
                'fallbacks': ['de', 'fr', ]
            },
            {
                'code': 'it',
                'name': 'Italiano',
                'fallbacks': ['fr', ]  # FOR TESTING, LEAVE AS ONLY 'fr'
            },
        ],
        'default': {
            'redirect_on_fallback': True,  # PLEASE DO NOT CHANGE THIS
        }
    },
    #
    # NOTE: The following setting `PARLER_ENABLE_CACHING = False` is required
    # for tests to pass.
    #
    # There appears to be a bug in Parler which leaves translations in Parler's
    # cache even after the parent object has been deleted. In production
    # environments, this is unlikely to affect anything, because newly created
    # objects will have new IDs. In testing, new objects are created with IDs
    # that were previously used, which reveals this issue.
    #
    'PARLER_ENABLE_CACHING': False,
    'ALDRYN_SEARCH_DEFAULT_LANGUAGE': 'en',
    'HAYSTACK_CONNECTIONS': {
        'default': {
            'ENGINE': 'haystack.backends.simple_backend.SimpleEngine',
        },
        'de': {
            'ENGINE': 'haystack.backends.simple_backend.SimpleEngine',
        },
    },
    'THUMBNAIL_HIGH_RESOLUTION': True,
    'THUMBNAIL_PROCESSORS': (
        'easy_thumbnails.processors.colorspace',
        'easy_thumbnails.processors.autocrop',
        # 'easy_thumbnails.processors.scale_and_crop',
        'filer.thumbnail_processors.scale_and_crop_with_subject_location',
        'easy_thumbnails.processors.filters',
    ),
    # 'DATABASES': {
    #     'default': {
    #         'ENGINE': 'django.db.backends.sqlite3',
    #         'NAME': 'mydatabase',
    #     },
    #     'mysql': {
    #         'ENGINE': 'django.db.backends.mysql',
    #         'NAME': 'newsblog_test',
    #         'USER': 'root',
    #         'PASSWORD': '',
    #         'HOST': '',
    #         'PORT': '3306',
    #     },
    #     'postgres': {
    #         'ENGINE': 'django.db.backends.postgresql_psycopg2',
    #         'NAME': 'newsblog_test',
    #         'USER': 'test',
    #         'PASSWORD': '',
    #         'HOST': '127.0.0.1',
    #         'PORT': '5432',
    #     }
    # }
}

if django_version < LooseVersion('1.8.0'):
    HELPER_SETTINGS.update({
        'CONTEXT_PROCESSORS': [
            'aldryn_boilerplates.context_processors.boilerplate',
        ],
        'TEMPLATE_DIRS': (
            os.path.join(
                os.path.dirname(__file__),
                'aldryn_newsblog', 'tests', 'templates'), ),
        'TEMPLATE_LOADERS': [
            'django.template.loaders.filesystem.Loader',
            # important! place right before:
            #     django.template.loaders.app_directories.Loader
            'aldryn_boilerplates.template_loaders.AppDirectoriesLoader',
            'django.template.loaders.app_directories.Loader',
            'django.template.loaders.eggs.Loader',
        ],
    })
elif django_version >= LooseVersion('1.8.0'):
    HELPER_SETTINGS.update({
        'TEMPLATES': [
            {
                'BACKEND': 'django.template.backends.django.DjangoTemplates',
                'DIRS': [
                    os.path.join(
                        os.path.dirname(__file__),
                        'aldryn_newsblog', 'tests', 'templates'),
                ],
                'OPTIONS': {
                    'context_processors': [
                        'django.contrib.auth.context_processors.auth',
                        'django.contrib.messages.context_processors.messages',
                        'django.core.context_processors.i18n',
                        'django.core.context_processors.debug',
                        'django.core.context_processors.request',
                        'django.core.context_processors.media',
                        'django.core.context_processors.csrf',
                        'django.core.context_processors.tz',
                        'sekizai.context_processors.sekizai',
                        'django.core.context_processors.static',
                        'cms.context_processors.cms_settings',
                        'aldryn_boilerplates.context_processors.boilerplate',
                    ],
                    'loaders': [
                        'django.template.loaders.filesystem.Loader',
                        'aldryn_boilerplates.template_loaders.AppDirectoriesLoader',
                        'django.template.loaders.app_directories.Loader',
                        'django.template.loaders.eggs.Loader',
                    ],
                },
            },
        ]
    })

# This set of MW classes should work for Django 1.6 and 1.7.
MIDDLEWARE_CLASSES_17 = [
    'aldryn_apphook_reload.middleware.ApphookReloadMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.locale.LocaleMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
    'cms.middleware.user.CurrentUserMiddleware',
    'cms.middleware.page.CurrentPageMiddleware',
    'cms.middleware.toolbar.ToolbarMiddleware',
    'cms.middleware.language.LanguageCookieMiddleware'
]

HELPER_SETTINGS['MIDDLEWARE_CLASSES'] = MIDDLEWARE_CLASSES_17

def run():
    from djangocms_helper import runner
    runner.cms('aldryn_newsblog')

if __name__ == "__main__":
    run()
<commit_msg>Fix flake issue<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from distutils.version import LooseVersion
from django import get_version

import os

django_version = LooseVersion(get_version())

HELPER_SETTINGS = {
    'TIME_ZONE': 'Europe/Zurich',
    'LANGUAGES': (
        ('en', 'English'),
        ('de', 'German'),
        ('fr', 'French'),
    ),
    'INSTALLED_APPS': [
        'aldryn_apphook_reload',
        'aldryn_apphooks_config',
        'aldryn_boilerplates',
        'aldryn_categories',
        'aldryn_newsblog',
        'aldryn_people',
        'aldryn_reversion',
        'djangocms_text_ckeditor',
        'easy_thumbnails',
        'filer',
        'mptt',
        'parler',
        'reversion',
        'sortedm2m',
        'taggit',
    ],
    'STATICFILES_FINDERS': [
        'django.contrib.staticfiles.finders.FileSystemFinder',
        # important! place right before:
        #     django.contrib.staticfiles.finders.AppDirectoriesFinder
        'aldryn_boilerplates.staticfile_finders.AppDirectoriesFinder',
        'django.contrib.staticfiles.finders.AppDirectoriesFinder',
    ],
    'ALDRYN_NEWSBLOG_TEMPLATE_PREFIXES': [('dummy', 'dummy'), ],
    'ALDRYN_BOILERPLATE_NAME': 'bootstrap3',
    # app-specific
    'PARLER_LANGUAGES': {
        1: (
            {'code': 'de', },
            {'code': 'fr', },
            {'code': 'en', },
        ),
        'default': {
            'hide_untranslated': True,  # PLEASE DO NOT CHANGE THIS
        }
    },
    'SITE_ID': 1,
    'CMS_LANGUAGES': {
        1: [
            {
                'code': 'de',
                'name': 'Deutsche',
                'fallbacks': ['en', ]  # FOR TESTING DO NOT ADD 'fr' HERE
            },
            {
                'code': 'fr',
                'name': u'Française',
                'fallbacks': ['en', ]  # FOR TESTING DO NOT ADD 'de' HERE
            },
            {
                'code': 'en',
                'name': 'English',
                'fallbacks': ['de', 'fr', ]
            },
            {
                'code': 'it',
                'name': 'Italiano',
                'fallbacks': ['fr', ]  # FOR TESTING, LEAVE AS ONLY 'fr'
            },
        ],
        'default': {
            'redirect_on_fallback': True,  # PLEASE DO NOT CHANGE THIS
        }
    },
    #
    # NOTE: The following setting `PARLER_ENABLE_CACHING = False` is required
    # for tests to pass.
    #
    # There appears to be a bug in Parler which leaves translations in Parler's
    # cache even after the parent object has been deleted. In production
    # environments, this is unlikely to affect anything, because newly created
    # objects will have new IDs. In testing, new objects are created with IDs
    # that were previously used, which reveals this issue.
    #
    'PARLER_ENABLE_CACHING': False,
    'ALDRYN_SEARCH_DEFAULT_LANGUAGE': 'en',
    'HAYSTACK_CONNECTIONS': {
        'default': {
            'ENGINE': 'haystack.backends.simple_backend.SimpleEngine',
        },
        'de': {
            'ENGINE': 'haystack.backends.simple_backend.SimpleEngine',
        },
    },
    'THUMBNAIL_HIGH_RESOLUTION': True,
    'THUMBNAIL_PROCESSORS': (
        'easy_thumbnails.processors.colorspace',
        'easy_thumbnails.processors.autocrop',
        # 'easy_thumbnails.processors.scale_and_crop',
        'filer.thumbnail_processors.scale_and_crop_with_subject_location',
        'easy_thumbnails.processors.filters',
    ),
    # 'DATABASES': {
    #     'default': {
    #         'ENGINE': 'django.db.backends.sqlite3',
    #         'NAME': 'mydatabase',
    #     },
    #     'mysql': {
    #         'ENGINE': 'django.db.backends.mysql',
    #         'NAME': 'newsblog_test',
    #         'USER': 'root',
    #         'PASSWORD': '',
    #         'HOST': '',
    #         'PORT': '3306',
    #     },
    #     'postgres': {
    #         'ENGINE': 'django.db.backends.postgresql_psycopg2',
    #         'NAME': 'newsblog_test',
    #         'USER': 'test',
    #         'PASSWORD': '',
    #         'HOST': '127.0.0.1',
    #         'PORT': '5432',
    #     }
    # }
}

if django_version < LooseVersion('1.8.0'):
    HELPER_SETTINGS.update({
        'CONTEXT_PROCESSORS': [
            'aldryn_boilerplates.context_processors.boilerplate',
        ],
        'TEMPLATE_DIRS': (
            os.path.join(
                os.path.dirname(__file__),
                'aldryn_newsblog', 'tests', 'templates'), ),
        'TEMPLATE_LOADERS': [
            'django.template.loaders.filesystem.Loader',
            # important! place right before:
            #     django.template.loaders.app_directories.Loader
            'aldryn_boilerplates.template_loaders.AppDirectoriesLoader',
            'django.template.loaders.app_directories.Loader',
            'django.template.loaders.eggs.Loader',
        ],
    })
elif django_version >= LooseVersion('1.8.0'):
    HELPER_SETTINGS.update({
        'TEMPLATES': [
            {
                'BACKEND': 'django.template.backends.django.DjangoTemplates',
                'DIRS': [
                    os.path.join(
                        os.path.dirname(__file__),
                        'aldryn_newsblog', 'tests', 'templates'),
                ],
                'OPTIONS': {
                    'context_processors': [
                        'django.contrib.auth.context_processors.auth',
                        'django.contrib.messages.context_processors.messages',
                        'django.core.context_processors.i18n',
                        'django.core.context_processors.debug',
                        'django.core.context_processors.request',
                        'django.core.context_processors.media',
                        'django.core.context_processors.csrf',
                        'django.core.context_processors.tz',
                        'sekizai.context_processors.sekizai',
                        'django.core.context_processors.static',
                        'cms.context_processors.cms_settings',
                        'aldryn_boilerplates.context_processors.boilerplate',
                    ],
                    'loaders': [
                        'django.template.loaders.filesystem.Loader',
                        'aldryn_boilerplates.template_loaders.AppDirectoriesLoader',  # flake8: noqa
                        'django.template.loaders.app_directories.Loader',
                        'django.template.loaders.eggs.Loader',
                    ],
                },
            },
        ]
    })

# This set of MW classes should work for Django 1.6 and 1.7.
MIDDLEWARE_CLASSES_17 = [
    'aldryn_apphook_reload.middleware.ApphookReloadMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.locale.LocaleMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
    'cms.middleware.user.CurrentUserMiddleware',
    'cms.middleware.page.CurrentPageMiddleware',
    'cms.middleware.toolbar.ToolbarMiddleware',
    'cms.middleware.language.LanguageCookieMiddleware'
]

HELPER_SETTINGS['MIDDLEWARE_CLASSES'] = MIDDLEWARE_CLASSES_17

def run():
    from djangocms_helper import runner
    runner.cms('aldryn_newsblog')

if __name__ == "__main__":
    run()
<|endoftext|>


<commit_before>import abc
from nalaf.structures.data import *
from nltk.stem.lancaster import LancasterStemmer
from nltk.corpus import stopwords
from progress.bar import Bar
from spacy.en import English

class Parser:
    """
    Abstract class for generating parse tree for each sentence.
    Subclasses that inherit this class should:
    * Be named [Name]ParseTree
    * Implement the abstract method parse
    * Append new items to the list field "edges" of each Part in the dataset
    """

    @abc.abstractmethod
    def parse(self, dataset):
        """
        :type dataset: nalaf.structures.data.Dataset
        """
        return

class SpacyParser(Parser):
    """
    Implementation of the SpaCy English for parsing the each sentence in each
    part separately, finding dependencies, parts of speech tags, lemmas and
    head words for each entity.

    :param nlp: an instance of spacy.en.English
    :type nlp: spacy.en.English
    :param constituency_parser: the constituency parser to use to generate
        syntactic (constituency) parse trees. Currently, supports only 'bllip'.
    :type constituency_parser: str
    """

    def __init__(self, nlp, constituency_parser=False):
        self.nlp = nlp
        """an instance of spacy.en.English"""
        self.constituency_parser = constituency_parser
        """the type of constituency parser to use, current supports only bllip"""
        # NOTE: SpaCy may soon have its own constituency parser: https://github.com/explosion/spaCy/issues/59

        if (not isinstance(self.nlp, English)):
            raise TypeError('Not an instance of spacy.en.English')

        if self.constituency_parser is True:
            self.parser = BllipParser(only_parse=True)

        # Use the default tokenization done by a call to
        # nalaf.preprocessing.Tokenizer before.
        # TODO check this, this is not using the final Tokenizer ???
        old_tokenizer = self.nlp.tokenizer
        self.nlp.tokenizer = lambda string: old_tokenizer.tokens_from_list(self._tokenize(string))

    def parse(self, dataset):
        """
        :type dataset: nalaf.structures.data.Dataset
        """
        outer_bar = Bar('Processing [SpaCy]', max=len(list(dataset.parts())))
        for part in dataset.parts():
            sentences = part.get_sentence_string_array()
            for index, sentence in enumerate(sentences):
                doc = self.nlp(sentence)
                for token in doc:
                    tok = part.sentences[index][token.i]
                    tok.features = {
                                    'id': token.i,
                                    'pos': token.tag_,
                                    'dep': token.dep_,
                                    'lemma': token.lemma_,
                                    'prob': token.prob,
                                    'is_punct': token.is_punct,
                                    'is_stop': token.is_stop,
                                    'cluster': token.cluster,
                                    'dependency_from': None,
                                    'dependency_to': [],
                                    'is_root': False,
                                   }
                    part.tokens.append(tok)

                for tok in doc:
                    self._dependency_path(tok, index, part)

            part.percolate_tokens_to_entities()
            part.calculate_token_scores()
            part.set_head_tokens()
            outer_bar.next()
        outer_bar.finish()

        if self.constituency_parser is True:
            self.parser.parse(dataset)

    def _tokenize(self, text):
        return text.split(' ')

    def _dependency_path(self, tok, index, part):
        token = part.sentences[index][tok.i]
        token.features['dependency_from'] = (part.sentences[index][tok.head.i], tok.dep_)
        token_from = part.sentences[index][tok.head.i]
        if (tok.i != tok.head.i):
            token_from.features['dependency_to'].append((token, tok.dep_))
        else:
            token.features['is_root'] = True

class BllipParser(Parser):
    """
    Implementation of the bllipparser for parsing the each sentence in each
    part separately, finding dependencies, parts of speech tags, lemmas and
    head words for each entity.

    Uses preprocessed text

    :param nbest: the number of parse trees to obtain
    :type nbest: int
    :param overparsing: overparsing determines how much more time the parser
        will spend on a sentence relative to the time it took to find the
        first possible complete parse
    :type overparsing: int
    """
    def __init__(self, nbest=10, overparsing=10, only_parse=False, stop_words=None):
        try:
            from bllipparser import RerankingParser
            # WARNING if only_parse=False, BllipParser depends on PyStanfordDependencies: pip install PyStanfordDependencies
        except ImportError:
            raise ImportError('BllipParser not installed, perhaps it is not supported on OS X yet')

        self.parser = RerankingParser.fetch_and_load('GENIA+PubMed', verbose=True)
        # WARNING this can take a long while. Install manually: `python -mbllipparser.ModelFetcher -i GENIA+PubMed`

        """create a Reranking Parser from BllipParser"""
        self.parser.set_parser_options(nbest=nbest, overparsing=overparsing)
        """set parser options"""
        self.only_parse=only_parse
        """whether features should be used from the BllipParser"""
        self.stemmer = LancasterStemmer()
        """an instance of LancasterStemmer from NLTK"""
        self.stop_words = stop_words
        if self.stop_words is None:
            self.stop_words = stopwords.words('english')

    def parse(self, dataset):
        outer_bar = Bar('Processing [Bllip]', max=len(list(dataset.parts())))
        for part in dataset.parts():
            outer_bar.next()
            if len(part.sentence_parse_trees)>0:
                continue
            for index, sentence in enumerate(part.sentences):
                sentence = [token.word for token in part.sentences[index]]
                parse = self.parser.parse(sentence)
                parsed = parse[0]
                part.sentence_parse_trees.append(str(parsed.ptb_parse))
                if not self.only_parse:
                    tokens = parsed.ptb_parse.sd_tokens()
                    for token in tokens:
                        tok = part.sentences[index][token.index-1]
                        is_stop = False
                        if tok.word.lower() in self.stop_words:
                            is_stop = True
                        tok.features = {
                                        'id': token.index-1,
                                        'pos': token.pos,
                                        'lemma': self.stemmer.stem(tok.word),
                                        'is_punct': self._is_punct(tok.word),
                                        'dep': token.deprel,
                                        'is_stop': is_stop,
                                        'dependency_from': None,
                                        'dependency_to': [],
                                        'is_root': False,
                                        }
                        part.tokens.append(tok)

                    for token in tokens:
                        tok = part.sentences[index][token.index-1]
                        self._dependency_path(token, tok, part, index)

            part.percolate_tokens_to_entities()
            part.calculate_token_scores()
            part.set_head_tokens()

        outer_bar.finish()

    def _dependency_path(self, bllip_token, token, part, index):
        if bllip_token.head-1>=0:
            token.features['dependency_from'] = (part.sentences[index][bllip_token.head-1], bllip_token.deprel)
        else:
            token.features['dependency_from'] = (part.sentences[index][token.features['id']], bllip_token.deprel)
        token_from = part.sentences[index][bllip_token.head-1]
        if (bllip_token.index != bllip_token.head):
            token_from.features['dependency_to'].append((token, bllip_token.deprel))
        else:
            token.features['is_root'] = True

    def _is_punct(self, text):
        if text in ['.', ',', '-']:
            return True
        return False
<commit_msg>do not manipulate nlp, which is a singleton from nalaf<commit_after>import abc
from nalaf.structures.data import *
from nltk.stem.lancaster import LancasterStemmer
from nltk.corpus import stopwords
from progress.bar import Bar
from spacy.en import English

class Parser:
    """
    Abstract class for generating parse tree for each sentence.
    Subclasses that inherit this class should:
    * Be named [Name]ParseTree
    * Implement the abstract method parse
    * Append new items to the list field "edges" of each Part in the dataset
    """

    @abc.abstractmethod
    def parse(self, dataset):
        """
        :type dataset: nalaf.structures.data.Dataset
        """
        return

class SpacyParser(Parser):
    """
    Implementation of the SpaCy English for parsing the each sentence in each
    part separately, finding dependencies, parts of speech tags, lemmas and
    head words for each entity.

    :param nlp: an instance of spacy.en.English
    :type nlp: spacy.en.English
    :param constituency_parser: the constituency parser to use to generate
        syntactic (constituency) parse trees. Currently, supports only 'bllip'.
    :type constituency_parser: str
    """

    def __init__(self, nlp, constituency_parser=False):
        self.nlp = nlp
        """an instance of spacy.en.English"""
        self.constituency_parser = constituency_parser
        """the type of constituency parser to use, current supports only bllip"""
        # NOTE: SpaCy may soon have its own constituency parser: https://github.com/explosion/spaCy/issues/59

        if (not isinstance(self.nlp, English)):
            raise TypeError('Not an instance of spacy.en.English')

        if self.constituency_parser is True:
            self.parser = BllipParser(only_parse=True)

    def parse(self, dataset):
        """
        :type dataset: nalaf.structures.data.Dataset
        """
        outer_bar = Bar('Processing [SpaCy]', max=len(list(dataset.parts())))
        for part in dataset.parts():
            sentences = part.get_sentence_string_array()

            for index, sentence in enumerate(sentences):
                doc = self.nlp.tokenizer.tokens_from_list(self._tokenize(sentence))

                for token in doc:
                    tok = part.sentences[index][token.i]
                    tok.features = {
                                    'id': token.i,
                                    'pos': token.tag_,
                                    'dep': token.dep_,
                                    'lemma': token.lemma_,
                                    'prob': token.prob,
                                    'is_punct': token.is_punct,
                                    'is_stop': token.is_stop,
                                    'cluster': token.cluster,
                                    'dependency_from': None,
                                    'dependency_to': [],
                                    'is_root': False,
                                   }
                    part.tokens.append(tok)

                for tok in doc:
                    self._dependency_path(tok, index, part)

            part.percolate_tokens_to_entities()
            part.calculate_token_scores()
            part.set_head_tokens()
            outer_bar.next()
        outer_bar.finish()

        if self.constituency_parser is True:
            self.parser.parse(dataset)

    # Use the default tokenization done by a call to
    # nalaf.preprocessing.Tokenizer before.
    # TODO check this, this is not using the final Tokenizer ???
    def _tokenize(self, text):
        return text.split(' ')

    def _dependency_path(self, tok, index, part):
        token = part.sentences[index][tok.i]
        token.features['dependency_from'] = (part.sentences[index][tok.head.i], tok.dep_)
        token_from = part.sentences[index][tok.head.i]
        if (tok.i != tok.head.i):
            token_from.features['dependency_to'].append((token, tok.dep_))
        else:
            token.features['is_root'] = True

class BllipParser(Parser):
    """
    Implementation of the bllipparser for parsing the each sentence in each
    part separately, finding dependencies, parts of speech tags, lemmas and
    head words for each entity.

    Uses preprocessed text

    :param nbest: the number of parse trees to obtain
    :type nbest: int
    :param overparsing: overparsing determines how much more time the parser
        will spend on a sentence relative to the time it took to find the
        first possible complete parse
    :type overparsing: int
    """
    def __init__(self, nbest=10, overparsing=10, only_parse=False, stop_words=None):
        try:
            from bllipparser import RerankingParser
            # WARNING if only_parse=False, BllipParser depends on PyStanfordDependencies: pip install PyStanfordDependencies
        except ImportError:
            raise ImportError('BllipParser not installed, perhaps it is not supported on OS X yet')

        self.parser = RerankingParser.fetch_and_load('GENIA+PubMed', verbose=True)
        # WARNING this can take a long while. Install manually: `python -mbllipparser.ModelFetcher -i GENIA+PubMed`

        """create a Reranking Parser from BllipParser"""
        self.parser.set_parser_options(nbest=nbest, overparsing=overparsing)
        """set parser options"""
        self.only_parse=only_parse
        """whether features should be used from the BllipParser"""
        self.stemmer = LancasterStemmer()
        """an instance of LancasterStemmer from NLTK"""
        self.stop_words = stop_words
        if self.stop_words is None:
            self.stop_words = stopwords.words('english')

    def parse(self, dataset):
        outer_bar = Bar('Processing [Bllip]', max=len(list(dataset.parts())))
        for part in dataset.parts():
            outer_bar.next()
            if len(part.sentence_parse_trees)>0:
                continue
            for index, sentence in enumerate(part.sentences):
                sentence = [token.word for token in part.sentences[index]]
                parse = self.parser.parse(sentence)
                parsed = parse[0]
                part.sentence_parse_trees.append(str(parsed.ptb_parse))
                if not self.only_parse:
                    tokens = parsed.ptb_parse.sd_tokens()
                    for token in tokens:
                        tok = part.sentences[index][token.index-1]
                        is_stop = False
                        if tok.word.lower() in self.stop_words:
                            is_stop = True
                        tok.features = {
                                        'id': token.index-1,
                                        'pos': token.pos,
                                        'lemma': self.stemmer.stem(tok.word),
                                        'is_punct': self._is_punct(tok.word),
                                        'dep': token.deprel,
                                        'is_stop': is_stop,
                                        'dependency_from': None,
                                        'dependency_to': [],
                                        'is_root': False,
                                        }
                        part.tokens.append(tok)

                    for token in tokens:
                        tok = part.sentences[index][token.index-1]
                        self._dependency_path(token, tok, part, index)

            part.percolate_tokens_to_entities()
            part.calculate_token_scores()
            part.set_head_tokens()

        outer_bar.finish()

    def _dependency_path(self, bllip_token, token, part, index):
        if bllip_token.head-1>=0:
            token.features['dependency_from'] = (part.sentences[index][bllip_token.head-1], bllip_token.deprel)
        else:
            token.features['dependency_from'] = (part.sentences[index][token.features['id']], bllip_token.deprel)
        token_from = part.sentences[index][bllip_token.head-1]
        if (bllip_token.index != bllip_token.head):
            token_from.features['dependency_to'].append((token, bllip_token.deprel))
        else:
            token.features['is_root'] = True

    def _is_punct(self, text):
        if text in ['.', ',', '-']:
            return True
        return False
<|endoftext|>


<commit_before>from django.db import models
from django.contrib.sessions.models import Session
from django.utils.translation import ugettext as _
from django.db.models.manager import EmptyManager
import lasso

from authentic2.saml.models import *

AUTHSAML2_UNAUTH_PERSISTENT = (
    ('AUTHSAML2_UNAUTH_PERSISTENT_ACCOUNT_LINKING_BY_AUTH',
        _('Account linking by authentication')),
    ('AUTHSAML2_UNAUTH_PERSISTENT_CREATE_USER_PSEUDONYMOUS',
        _('Create new account')),
)

AUTHSAML2_UNAUTH_TRANSIENT = (
    ('AUTHSAML2_UNAUTH_TRANSIENT_ASK_AUTH', _('Ask authentication')),
    ('AUTHSAML2_UNAUTH_TRANSIENT_OPEN_SESSION', _('Open a session')),
)

# TODO: remove options after handling
class MyServiceProvider(models.Model):
    handle_persistent = models.CharField(
            max_length=80,
            verbose_name = 'Account Policy with persistent nameId',
            choices=AUTHSAML2_UNAUTH_PERSISTENT)
    handle_transient = models.CharField(
            max_length=80,
            verbose_name = 'Access Policy with transient nameId',
            choices=AUTHSAML2_UNAUTH_TRANSIENT)
    back_url = models.CharField(
            max_length = 80,
            verbose_name = 'Return URL after a successful authentication')

    class Meta:
        verbose_name = _('Service provider core configuration')

    def __unicode__(self):
        return "Service provider core configuration"

class SAML2TransientUser(object):
    '''Class compatible with django.contrib.auth.models.User
       which represent an user authenticated using a Transient
       federation'''
    id = None
    is_staff = False
    is_active = False
    is_superuser = False
    _groups = EmptyManager()
    _user_permissions = EmptyManager()

    def __init__(self, id):
        self.id = id

    def __unicode__(self):
        return 'AnonymousUser'

    def __str__(self):
        return unicode(self).encode('utf-8')

    def __eq__(self, other):
        return isinstance(other, self.__class__)

    def __ne__(self, other):
        return not self.__eq__(other)

    def __hash__(self):
        return 1 # instances always return the same hash value

    def save(self):
        pass

    def delete(self):
        raise NotImplementedError

    def set_password(self, raw_password):
        raise NotImplementedError

    def check_password(self, raw_password):
        raise NotImplementedError

    def _get_groups(self):
        return self._groups
    groups = property(_get_groups)

    def _get_user_permissions(self):
        return self._user_permissions
    user_permissions = property(_get_user_permissions)

    def get_group_permissions(self, obj=None):
        return set()

    def get_all_permissions(self, obj=None):
        return _user_get_all_permissions(self, obj=obj)

    def has_perm(self, perm, obj=None):
        return _user_has_perm(self, perm, obj=obj)

    def has_perms(self, perm_list, obj=None):
        for perm in perm_list:
            if not self.has_perm(perm, obj):
                return False
        return True

    def has_module_perms(self, module):
        return _user_has_module_perms(self, module)

    def get_and_delete_messages(self):
        return []

    def is_anonymous(self):
        return False

    def is_authenticated(self):
        return True

    def get_username(self):
        return _('Anonymous')
    username = property(get_username)
<commit_msg>[authsaml2] SAML2TransientUser object more conform<commit_after>from django.db import models
from django.contrib.sessions.models import Session
from django.utils.translation import ugettext as _
from django.db.models.manager import EmptyManager
import lasso

from authentic2.saml.models import *

AUTHSAML2_UNAUTH_PERSISTENT = (
    ('AUTHSAML2_UNAUTH_PERSISTENT_ACCOUNT_LINKING_BY_AUTH',
        _('Account linking by authentication')),
    ('AUTHSAML2_UNAUTH_PERSISTENT_CREATE_USER_PSEUDONYMOUS',
        _('Create new account')),
)

AUTHSAML2_UNAUTH_TRANSIENT = (
    ('AUTHSAML2_UNAUTH_TRANSIENT_ASK_AUTH', _('Ask authentication')),
    ('AUTHSAML2_UNAUTH_TRANSIENT_OPEN_SESSION', _('Open a session')),
)

# TODO: remove options after handling
class MyServiceProvider(models.Model):
    handle_persistent = models.CharField(
            max_length=80,
            verbose_name = 'Account Policy with persistent nameId',
            choices=AUTHSAML2_UNAUTH_PERSISTENT)
    handle_transient = models.CharField(
            max_length=80,
            verbose_name = 'Access Policy with transient nameId',
            choices=AUTHSAML2_UNAUTH_TRANSIENT)
    back_url = models.CharField(
            max_length = 80,
            verbose_name = 'Return URL after a successful authentication')

    class Meta:
        verbose_name = _('Service provider core configuration')

    def __unicode__(self):
        return "Service provider core configuration"

class FakePk:
    name = '_pk'

class FakeMeta:
    pk = FakePk()

class SAML2TransientUser(object):
    '''Class compatible with django.contrib.auth.models.User
       which represent an user authenticated using a Transient
       federation'''
    id = None
    is_staff = False
    is_active = False
    is_superuser = False
    _groups = EmptyManager()
    _user_permissions = EmptyManager()
    _pk = -1
    _meta = FakeMeta()

    def __init__(self, id):
        self.id = id

    def __unicode__(self):
        return 'AnonymousUser'

    def __str__(self):
        return unicode(self).encode('utf-8')

    def __eq__(self, other):
        return isinstance(other, self.__class__)

    def __ne__(self, other):
        return not self.__eq__(other)

    def __hash__(self):
        return 1 # instances always return the same hash value

    def save(self):
        pass

    def delete(self):
        raise NotImplementedError

    def set_password(self, raw_password):
        raise NotImplementedError

    def check_password(self, raw_password):
        raise NotImplementedError

    def _get_groups(self):
        return self._groups
    groups = property(_get_groups)

    def _get_user_permissions(self):
        return self._user_permissions
    user_permissions = property(_get_user_permissions)

    def get_group_permissions(self, obj=None):
        return set()

    def get_all_permissions(self, obj=None):
        return _user_get_all_permissions(self, obj=obj)

    def has_perm(self, perm, obj=None):
        return _user_has_perm(self, perm, obj=obj)

    def has_perms(self, perm_list, obj=None):
        for perm in perm_list:
            if not self.has_perm(perm, obj):
                return False
        return True

    def has_module_perms(self, module):
        return _user_has_module_perms(self, module)

    def get_and_delete_messages(self):
        return []

    def is_anonymous(self):
        #XXX: Should return True
        return False

    def is_authenticated(self):
        return True

    def get_username(self):
        return _('Anonymous')
    username = property(get_username)
<|endoftext|>


<commit_before>import pandas
import numpy as np

def simple_heuristic(file_path):
    """
    In this exercise, we will perform some rudimentary practices similar to those of
    an actual data scientist.

    Part of a data scientist's job is to use her or his intuition and insight to
    write algorithms and heuristics. A data scientist also creates mathematical models
    to make predictions based on some attributes from the data that they are examining.

    We would like for you to take your knowledge and intuition about the Titanic
    and its passengers' attributes to predict whether or not the passengers survived
    or perished. You can read more about the Titanic and specifics about this dataset at:
    http://en.wikipedia.org/wiki/RMS_Titanic
    http://www.kaggle.com/c/titanic-gettingStarted

    In this exercise and the following ones, you are given a list of Titanic passengers
    and their associated information. More information about the data can be seen at the
    link below:
    http://www.kaggle.com/c/titanic-gettingStarted/data.

    For this exercise, you need to write a simple heuristic that will use
    the passengers' gender to predict if that person survived the Titanic disaster.

    You prediction should be 78% accurate or higher.

    Here's a simple heuristic to start off:
       1) If the passenger is female, your heuristic should assume that the
       passenger survived.
       2) If the passenger is male, you heuristic should
       assume that the passenger did not survive.

    You can access the gender of a passenger via passenger['Sex'].
    If the passenger is male, passenger['Sex'] will return a string "male".
    If the passenger is female, passenger['Sex'] will return a string "female".

    Write your prediction back into the "predictions" dictionary. The
    key of the dictionary should be the passenger's id (which can be accessed
    via passenger["PassengerId"]) and the associated value should be 1 if the
    passenger survived or 0 otherwise.

    For example, if a passenger is predicted to have survived:
    passenger_id = passenger['PassengerId']
    predictions[passenger_id] = 1

    And if a passenger is predicted to have perished in the disaster:
    passenger_id = passenger['PassengerId']
    predictions[passenger_id] = 0

    You can also look at the Titanic data that you will be working with
    at the link below:
    https://s3.amazonaws.com/content.udacity-data.com/courses/ud359/titanic_data.csv
    """

    predictions = {}
    df = pandas.read_csv(file_path)
    for passenger_index, passenger in df.iterrows():
        passenger_id = passenger['PassengerId']

        # Your code here:
        # For example, let's assume that if the passenger
        # is a male, then the passenger survived.
        #     if passenger['Sex'] == 'male':
        #         predictions[passenger_id] = 1

        if passenger['Sex'] == 'female':
            predictions[passenger_id] = 1
        elif passenger['Sex'] == 'male':
            predictions[passenger_id] = 0
        else:
            print('warning: wrong gender found: Id ',passenger['PassengerId'],' gender ',passenger['Sex'])

    return predictions

file_path = 'titanic_data.csv'

prediction = simple_heuristic(file_path)

df = pandas.read_csv(file_path)
df['prediction'] = prediction.values()
accuracy = np.mean(list(df['prediction']==df['Survived']))

print("Your heuristic is {:.2f}% accurate".format(accuracy*100))<commit_msg>feat: simple heuristic<commit_after>import numpy as np
import pandas

def simple_heuristic(file_path):
    """
    In this exercise, we will perform some rudimentary practices similar to those of
    an actual data scientist.

    Part of a data scientist's job is to use her or his intuition and insight to
    write algorithms and heuristics. A data scientist also creates mathematical models
    to make predictions based on some attributes from the data that they are examining.

    We would like for you to take your knowledge and intuition about the Titanic
    and its passengers' attributes to predict whether or not the passengers survived
    or perished. You can read more about the Titanic and specifics about this dataset at:
    http://en.wikipedia.org/wiki/RMS_Titanic
    http://www.kaggle.com/c/titanic-gettingStarted

    In this exercise and the following ones, you are given a list of Titanic passengers
    and their associated information. More information about the data can be seen at the
    link below:
    http://www.kaggle.com/c/titanic-gettingStarted/data.

    For this exercise, you need to write a simple heuristic that will use
    the passengers' gender to predict if that person survived the Titanic disaster.

    You prediction should be 78% accurate or higher.

    Here's a simple heuristic to start off:
       1) If the passenger is female, your heuristic should assume that the
       passenger survived.
       2) If the passenger is male, you heuristic should
       assume that the passenger did not survive.

    You can access the gender of a passenger via passenger['Sex'].
    If the passenger is male, passenger['Sex'] will return a string "male".
    If the passenger is female, passenger['Sex'] will return a string "female".

    Write your prediction back into the "predictions" dictionary. The
    key of the dictionary should be the passenger's id (which can be accessed
    via passenger["PassengerId"]) and the associated value should be 1 if the
    passenger survived or 0 otherwise.

    For example, if a passenger is predicted to have survived:
    passenger_id = passenger['PassengerId']
    predictions[passenger_id] = 1

    And if a passenger is predicted to have perished in the disaster:
    passenger_id = passenger['PassengerId']
    predictions[passenger_id] = 0

    You can also look at the Titanic data that you will be working with
    at the link below:
    https://s3.amazonaws.com/content.udacity-data.com/courses/ud359/titanic_data.csv
    """

    predictions = {}
    df = pandas.read_csv(file_path)
    for passenger_index, passenger in df.iterrows():
        passenger_id = passenger['PassengerId']

        # Your code here:
        # For example, let's assume that if the passenger
        # is a male, then the passenger survived.
        #     if passenger['Sex'] == 'male':
        #         predictions[passenger_id] = 1

        if passenger['Sex'] == 'female':
            predictions[passenger_id] = 1
        elif passenger['Sex'] == 'male':
            predictions[passenger_id] = 0
        else:
            print('warning: wrong gender found: Id ', passenger['PassengerId'], ' gender ', passenger['Sex'])

    return predictions

f_path = 'titanic_data.csv'

prediction = simple_heuristic(f_path)

dframe = pandas.read_csv(f_path)
dframe['prediction'] = prediction.values()
accuracy = np.mean(list(dframe['prediction'] == dframe['Survived']))

print("Your heuristic is {:.2f}% accurate".format(accuracy * 100))
<|endoftext|>


<commit_before>#     Copyright 2018, Kay Hayen, mailto:kay.hayen@gmail.com
#
#     Part of "Nuitka", an optimizing Python compiler that is compatible and
#     integrates with CPython, but also works on its own.
#
#     Licensed under the Apache License, Version 2.0 (the "License");
#     you may not use this file except in compliance with the License.
#     You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#     Unless required by applicable law or agreed to in writing, software
#     distributed under the License is distributed on an "AS IS" BASIS,
#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#     See the License for the specific language governing permissions and
#     limitations under the License.
#
""" Scons interface.

Interaction with scons. Find the binary, and run it with a set of given
options.

"""

import contextlib
import os
import subprocess
import sys

from nuitka import Options, Tracing
from nuitka.__past__ import unicode  # pylint: disable=I0021,redefined-builtin
from nuitka.PythonVersions import getTargetPythonDLLPath, python_version
from nuitka.utils import Execution, Utils

def getSconsDataPath():
    """ Return path to where data for scons lives, e.g. static C source files.

    """

    return os.path.dirname(__file__)

def _getSconsInlinePath():
    """ Return path to inline copy of scons. """

    return os.path.join(
        getSconsDataPath(),
        "inline_copy"
    )

def _getSconsBinaryCall():
    """ Return a way to execute Scons.

    Using potentially in-line copy if no system Scons is available
    or if we are on Windows, there it is mandatory.
    """
    if Utils.getOS() != "Windows":
        scons_path = Execution.getExecutablePath("scons")

        if scons_path is not None:
            return [
                scons_path
            ]

    return [
        _getPythonForSconsExePath(),
        os.path.join(
            _getSconsInlinePath(),
            "bin",
            "scons.py"
        )
    ]

def _getPythonSconsExePathWindows():
    """ Find Python2 on Windows.

    First try a few guesses, the look into registry for user or system wide
    installations of Python2. Both Python 2.6 and 2.7, and 3.5 or higher
    will do.
    """

    # Ordered in the list of preference.
    scons_supported = ("2.7", "2.6", "3.5", "3.6", "3.7")

    # Shortcuts for the default installation directories, to avoid going to
    # registry at all unless necessary. Any Python2 will do for Scons, so it
    # might be avoided entirely.
    for search in scons_supported:
        candidate = r"c:\Python%s\python.exe" % search.replace('.', "")

        if os.path.isfile(candidate):
            return candidate

    # Windows only code, pylint: disable=I0021,import-error,undefined-variable
    if python_version < 300:
        import _winreg as winreg # @UnresolvedImport pylint: disable=I0021,import-error,no-name-in-module
    else:
        import winreg  # @Reimport @UnresolvedImport pylint: disable=I0021,import-error,no-name-in-module

    for search in scons_supported:
        for hkey_branch in (winreg.HKEY_LOCAL_MACHINE, winreg.HKEY_CURRENT_USER):
            for arch_key in 0, winreg.KEY_WOW64_32KEY, winreg.KEY_WOW64_64KEY:
                try:
                    key = winreg.OpenKey(
                        hkey_branch,
                        r"SOFTWARE\Python\PythonCore\%s\InstallPath" % search,
                        0,
                        winreg.KEY_READ | arch_key
                    )

                    return os.path.join(
                        winreg.QueryValue(key, ""),
                        "python.exe"
                    )
                except WindowsError:  # @UndefinedVariable
                    pass

def _getPythonForSconsExePath():
    """ Find a way to call any Python2.

    Scons needs it as it doesn't support Python3.
    """
    python_exe = Options.getPythonPathForScons()

    if python_exe is not None:
        return python_exe

    if python_version < 300 or python_version >= 350:
        return sys.executable
    elif Utils.getOS() == "Windows":
        python_exe = _getPythonSconsExePathWindows()

        if python_exe is not None:
            return python_exe
        else:
            sys.exit("""\
Error, while Nuitka works with Python 3.2 to 3.4, scons does not, and Nuitka
needs to find a Python executable 2.6/2.7 or 3.5 or higher. Simply under the
C:\\PythonXY, e.g. C:\\Python27 to execute the scons utility which is used
to build the C files to binary.

You may provide it using option "--python-for-scons=path_to_python.exe"
in case it is not visible in registry, e.g. due to using uninstalled
AnaConda Python.
""")

    for version_candidate in ("2.7", "2.6", "3.5", "3.6"):
        candidate = Execution.getExecutablePath("python" + version_candidate)

        if candidate is not None:
            return candidate

    # Lets be optimistic, this is most often going to be new enough or a
    # Python2 variant.
    return "python"

@contextlib.contextmanager
def _setupSconsEnvironment():
    """ Setup the scons execution environment.

    For the scons inline copy on Windows needs to find the library, using
    the "SCONS_LIB_DIR" environment variable "NUITKA_SCONS". And for the
    target Python we provide "NUITKA_PYTHON_DLL_PATH" to see where the
    Python DLL lives, in case it needs to be copied, and then the
    "NUITKA_PYTHON_EXE_PATH" to find the Python binary itself.
    """

    if Utils.getOS() == "Windows":
        # On Windows this Scons variable must be set by us.
        os.environ["SCONS_LIB_DIR"] = os.path.join(
            _getSconsInlinePath(),
            "lib",
            "scons-2.3.2"
        )

        # On Windows, we use the Python.DLL path for some things. We pass it
        # via environment variable
        os.environ["NUITKA_PYTHON_DLL_PATH"] = getTargetPythonDLLPath()

    os.environ["NUITKA_PYTHON_EXE_PATH"] = sys.executable

    # Remove environment variables that can only harm if we have to switch
    # major Python versions, these cannot help Python2 to execute scons, this
    # is a bit of noise, but helpful.
    if python_version >= 300:
        if "PYTHONPATH" in os.environ:
            old_pythonpath = os.environ["PYTHONPATH"]
            del os.environ["PYTHONPATH"]
        else:
            old_pythonpath = None

        if "PYTHONHOME" in os.environ:
            old_pythonhome = os.environ["PYTHONHOME"]
            del os.environ["PYTHONHOME"]
        else:
            old_pythonhome = None

    yield

    if python_version >= 300:
        if old_pythonpath is not None:
            os.environ["PYTHONPATH"] = old_pythonpath

        if old_pythonhome is not None:
            os.environ["PYTHONHOME"] = old_pythonhome

    if Utils.getOS() == "Windows":
        del os.environ["NUITKA_PYTHON_DLL_PATH"]
        del os.environ["NUITKA_PYTHON_EXE_PATH"]

def _buildSconsCommand(quiet, options):
    """ Build the scons command to run.

    The options are a dictionary to be passed to scons as a command line,
    and other scons stuff is set.
    """

    scons_command = _getSconsBinaryCall()

    if quiet:
        scons_command.append("--quiet")

    scons_command += [
        # The scons file
        "-f",
        os.path.join(
            getSconsDataPath(),
            "SingleExe.scons"
        ),

        # Parallel compilation.
        "--jobs",
        str(Options.getJobLimit()),

        # Do not warn about deprecation from Scons
        "--warn=no-deprecated",

        # Don't load "site_scons" at all.
        "--no-site-dir",
    ]

    if Options.isShowScons():
        scons_command.append("--debug=explain")

    # Python2, encoding unicode values
    def encode(value):
        if str is bytes and type(value) is unicode:
            return value.encode("utf8")
        else:
            return value

    # Option values to provide to scons. Find these in the caller.
    for key, value in options.items():
        scons_command.append(
            key + '=' + encode(value)
        )

    # Python2, make argument encoding recognizable.
    if str is bytes:
        scons_command.append(
            "arg_encoding=utf8"
        )

    return scons_command

def runScons(options, quiet):
    with _setupSconsEnvironment():
        scons_command = _buildSconsCommand(quiet, options)

        if Options.isShowScons():
            Tracing.printLine("Scons command:", ' '.join(scons_command))

        Tracing.flushStdout()
        return subprocess.call(scons_command, shell = False) == 0
<commit_msg>Scons: Disable Python warnings when running scons.<commit_after>#     Copyright 2018, Kay Hayen, mailto:kay.hayen@gmail.com
#
#     Part of "Nuitka", an optimizing Python compiler that is compatible and
#     integrates with CPython, but also works on its own.
#
#     Licensed under the Apache License, Version 2.0 (the "License");
#     you may not use this file except in compliance with the License.
#     You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#     Unless required by applicable law or agreed to in writing, software
#     distributed under the License is distributed on an "AS IS" BASIS,
#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#     See the License for the specific language governing permissions and
#     limitations under the License.
#
""" Scons interface.

Interaction with scons. Find the binary, and run it with a set of given
options.

"""

import contextlib
import os
import subprocess
import sys

from nuitka import Options, Tracing
from nuitka.__past__ import unicode  # pylint: disable=I0021,redefined-builtin
from nuitka.PythonVersions import getTargetPythonDLLPath, python_version
from nuitka.utils import Execution, Utils

def getSconsDataPath():
    """ Return path to where data for scons lives, e.g. static C source files.

    """

    return os.path.dirname(__file__)

def _getSconsInlinePath():
    """ Return path to inline copy of scons. """

    return os.path.join(
        getSconsDataPath(),
        "inline_copy"
    )

def _getSconsBinaryCall():
    """ Return a way to execute Scons.

    Using potentially in-line copy if no system Scons is available
    or if we are on Windows, there it is mandatory.
    """
    if Utils.getOS() != "Windows":
        scons_path = Execution.getExecutablePath("scons")

        if scons_path is not None:
            return [
                scons_path
            ]

    return [
        _getPythonForSconsExePath(),
        "-W", "ignore", # Disable Python warnings in case of debug Python.
        os.path.join(
            _getSconsInlinePath(),
            "bin",
            "scons.py"
        )
    ]

def _getPythonSconsExePathWindows():
    """ Find Python2 on Windows.

    First try a few guesses, the look into registry for user or system wide
    installations of Python2. Both Python 2.6 and 2.7, and 3.5 or higher
    will do.
    """

    # Ordered in the list of preference.
    scons_supported = ("2.7", "2.6", "3.5", "3.6", "3.7")

    # Shortcuts for the default installation directories, to avoid going to
    # registry at all unless necessary. Any Python2 will do for Scons, so it
    # might be avoided entirely.
    for search in scons_supported:
        candidate = r"c:\Python%s\python.exe" % search.replace('.', "")

        if os.path.isfile(candidate):
            return candidate

    # Windows only code, pylint: disable=I0021,import-error,undefined-variable
    if python_version < 300:
        import _winreg as winreg # @UnresolvedImport pylint: disable=I0021,import-error,no-name-in-module
    else:
        import winreg  # @Reimport @UnresolvedImport pylint: disable=I0021,import-error,no-name-in-module

    for search in scons_supported:
        for hkey_branch in (winreg.HKEY_LOCAL_MACHINE, winreg.HKEY_CURRENT_USER):
            for arch_key in 0, winreg.KEY_WOW64_32KEY, winreg.KEY_WOW64_64KEY:
                try:
                    key = winreg.OpenKey(
                        hkey_branch,
                        r"SOFTWARE\Python\PythonCore\%s\InstallPath" % search,
                        0,
                        winreg.KEY_READ | arch_key
                    )

                    return os.path.join(
                        winreg.QueryValue(key, ""),
                        "python.exe"
                    )
                except WindowsError:  # @UndefinedVariable
                    pass

def _getPythonForSconsExePath():
    """ Find a way to call any Python2.

    Scons needs it as it doesn't support Python3.
    """
    python_exe = Options.getPythonPathForScons()

    if python_exe is not None:
        return python_exe

    if python_version < 300 or python_version >= 350:
        return sys.executable
    elif Utils.getOS() == "Windows":
        python_exe = _getPythonSconsExePathWindows()

        if python_exe is not None:
            return python_exe
        else:
            sys.exit("""\
Error, while Nuitka works with Python 3.2 to 3.4, scons does not, and Nuitka
needs to find a Python executable 2.6/2.7 or 3.5 or higher. Simply under the
C:\\PythonXY, e.g. C:\\Python27 to execute the scons utility which is used
to build the C files to binary.

You may provide it using option "--python-for-scons=path_to_python.exe"
in case it is not visible in registry, e.g. due to using uninstalled
AnaConda Python.
""")

    for version_candidate in ("2.7", "2.6", "3.5", "3.6"):
        candidate = Execution.getExecutablePath("python" + version_candidate)

        if candidate is not None:
            return candidate

    # Lets be optimistic, this is most often going to be new enough or a
    # Python2 variant.
    return "python"

@contextlib.contextmanager
def _setupSconsEnvironment():
    """ Setup the scons execution environment.

    For the scons inline copy on Windows needs to find the library, using
    the "SCONS_LIB_DIR" environment variable "NUITKA_SCONS". And for the
    target Python we provide "NUITKA_PYTHON_DLL_PATH" to see where the
    Python DLL lives, in case it needs to be copied, and then the
    "NUITKA_PYTHON_EXE_PATH" to find the Python binary itself.
    """

    if Utils.getOS() == "Windows":
        # On Windows this Scons variable must be set by us.
        os.environ["SCONS_LIB_DIR"] = os.path.join(
            _getSconsInlinePath(),
            "lib",
            "scons-2.3.2"
        )

        # On Windows, we use the Python.DLL path for some things. We pass it
        # via environment variable
        os.environ["NUITKA_PYTHON_DLL_PATH"] = getTargetPythonDLLPath()

    os.environ["NUITKA_PYTHON_EXE_PATH"] = sys.executable

    # Remove environment variables that can only harm if we have to switch
    # major Python versions, these cannot help Python2 to execute scons, this
    # is a bit of noise, but helpful.
    if python_version >= 300:
        if "PYTHONPATH" in os.environ:
            old_pythonpath = os.environ["PYTHONPATH"]
            del os.environ["PYTHONPATH"]
        else:
            old_pythonpath = None

        if "PYTHONHOME" in os.environ:
            old_pythonhome = os.environ["PYTHONHOME"]
            del os.environ["PYTHONHOME"]
        else:
            old_pythonhome = None

    yield

    if python_version >= 300:
        if old_pythonpath is not None:
            os.environ["PYTHONPATH"] = old_pythonpath

        if old_pythonhome is not None:
            os.environ["PYTHONHOME"] = old_pythonhome

    if Utils.getOS() == "Windows":
        del os.environ["NUITKA_PYTHON_DLL_PATH"]
        del os.environ["NUITKA_PYTHON_EXE_PATH"]

def _buildSconsCommand(quiet, options):
    """ Build the scons command to run.

    The options are a dictionary to be passed to scons as a command line,
    and other scons stuff is set.
    """

    scons_command = _getSconsBinaryCall()

    if quiet:
        scons_command.append("--quiet")

    scons_command += [
        # The scons file
        "-f",
        os.path.join(
            getSconsDataPath(),
            "SingleExe.scons"
        ),

        # Parallel compilation.
        "--jobs",
        str(Options.getJobLimit()),

        # Do not warn about deprecation from Scons
        "--warn=no-deprecated",

        # Don't load "site_scons" at all.
        "--no-site-dir",
    ]

    if Options.isShowScons():
        scons_command.append("--debug=explain")

    # Python2, encoding unicode values
    def encode(value):
        if str is bytes and type(value) is unicode:
            return value.encode("utf8")
        else:
            return value

    # Option values to provide to scons. Find these in the caller.
    for key, value in options.items():
        scons_command.append(
            key + '=' + encode(value)
        )

    # Python2, make argument encoding recognizable.
    if str is bytes:
        scons_command.append(
            "arg_encoding=utf8"
        )

    return scons_command

def runScons(options, quiet):
    with _setupSconsEnvironment():
        scons_command = _buildSconsCommand(quiet, options)

        if Options.isShowScons():
            Tracing.printLine("Scons command:", ' '.join(scons_command))

        Tracing.flushStdout()
        return subprocess.call(scons_command, shell = False) == 0
<|endoftext|>