<commit_before>#!/usr/bin/python3

## runtest.py
# The runtest.py script runs regression tests on the CuraEngine.
# It parses the json file for settings to know which settings can be passed towards the engine.
# It runs the following test:
# * Defaults
# * Bounds/possible values
# * Single random value
# * All settings random

import ast #For safe function evaluation.
import math #For evaluating setting inheritance functions.
import sys
import subprocess
import os
import time
import stat
import argparse
import random
import json
import threading
from xml.etree import ElementTree

## The TestSuite class stores the test results of a single set of tests.
#  TestSuite objects are created by the TestResults class.
class TestSuite:
    def __init__(self, name):
        self._name = name
        self._successes = []
        self._failures = []

    ## Add a successful test result to the test suite.
    def success(self, class_name, test_name):
        self._successes.append((class_name, test_name))

    ## Add a failed test result to the test suite.
    def failure(self, class_name, test_name, error_message):
        self._failures.append((class_name, test_name, error_message))

    ## Return the number of tests in this test suite
    def getTestCount(self):
        return self.getSuccessCount() + self.getFailureCount()

    ## Return the number of successful tests in this test suite
    def getSuccessCount(self):
        return len(self._successes)

    ## Return the number of failed tests in this test suite
    def getFailureCount(self):
        return len(self._failures)

## The TestResults class stores a group of TestSuite objects, each TestSuite object contains failed and successful test.
#  This class can output the result of the tests in a JUnit xml format for parsing in Jenkins.
class TestResults:
    def __init__(self):
        self._testsuites = []

    ## Create a new test suite with the name.
    def addTestSuite(self, name):
        suite = TestSuite(name)
        self._testsuites.append(suite)
        return suite

    def getFailureCount(self):
        result = 0
        for testsuite in self._testsuites:
            result += testsuite.getFailureCount()
        return result

    ## Save the test results to the file given in the filename.
    def saveXML(self, filename):
        xml = ElementTree.Element("testsuites")
        xml.text = "\n"
        for testsuite in self._testsuites:
            testsuite_xml = ElementTree.SubElement(xml, "testsuite", {"name": testsuite._name, "errors": "0", "tests": str(testsuite.getTestCount()), "failures": str(testsuite.getFailureCount()), "time": "0", "timestamp": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime())})
            testsuite_xml.text = "\n"
            testsuite_xml.tail = "\n"
            for class_name, test_name in testsuite._successes:
                testcase_xml = ElementTree.SubElement(testsuite_xml, "testcase", {"classname": class_name, "name": test_name})
                testcase_xml.text = "\n"
                testcase_xml.tail = "\n"
            for class_name, test_name, error_message in testsuite._failures:
                testcase_xml = ElementTree.SubElement(testsuite_xml, "testcase", {"classname": class_name, "name": test_name})
                testcase_xml.text = "\n"
                testcase_xml.tail = "\n"
                failure = ElementTree.SubElement(testcase_xml, "failure", {"message": "test failure"})
                failure.text = error_message
                failure.tail = "\n"
        return ElementTree.ElementTree(xml).write(filename, "utf-8", True)

class Setting:
    ##  Creates a new setting from a JSON node.
    #
    #   Some parts of the setting may have to be evaluated as functions. For
    #   these, the default values of all settings are added as local variables.
    #
    #   \param key The name of the setting.
    #   \param data The JSON node of the setting, containing the default value,
    #   the setting type, the minimum value, maximum value, the minimum warning
    #   value, the maximum warning value and (for enum types) the options.
    #   \param locals The local variables for eventual function evaluation.
    def __init__(self, key, data, locals):
        self._key = key
        self._default = self._evaluateFunction(data.get("value", str(data.get("default_value", 0))), locals) #Evaluate "value" if we can, otherwise evaluate a stringified "default_value".
        self._type = data["type"]
        self._min_value = self._evaluateFunction(data.get("minimum_value", None), locals)
        self._max_value = self._evaluateFunction(data.get("maximum_value", None), locals)
        self._min_value_warning = self._evaluateFunction(data.get("minimum_value_warning", None), locals)
        self._max_value_warning = self._evaluateFunction(data.get("maximum_value_warning", None), locals)
        self._options = data.get("options", None)
        if self._options is not None:
            self._options = list(self._options.keys())

    ##  Gets the default value of this setting, according to the source JSON.
    #
    #   \return The default value.
    def getDefault(self):
        return self._default

    ## Return a list of possible values for this setting. This list depends on the setting type.
    #  For number values it contains the minimal and maximal values.
    #  For enums and booleans it will contain the exact possible values.
    #  For string settings only the default value is returned.
    def getSettingValues(self):
        if self._type == "bool":
            return ["True", "False"]
        if self._type == "float" or self._type == "int":
            ret = [self._default]
            if self._min_value is not None:
                ret.append(self._min_value)
            else:
                ret.append(-2)
            if self._min_value_warning is not None:
                ret.append(self._min_value_warning)
            if self._max_value is not None:
                ret.append(self._max_value)
            else:
                if self._max_value_warning is None:
                    ret.append(10000)
                elif self._type == "float":
                    ret.append(float(self._max_value_warning) + 100)
                    ret.append(float(self._max_value_warning) * 10)
                elif self._type == "int":
                    ret.append(int(self._max_value_warning) + 100)
                    ret.append(int(self._max_value_warning) * 10)
                # If the type is boolean, string or enum, the warning values make no sense anyway, so don't test them.
            if self._max_value_warning is not None:
                ret.append(self._max_value_warning)

            if self._type == "int":
                for n in range(0, len(ret)):
                    ret[n] = int(ret[n])
            return ret
        if self._type == "enum":
            return self._options
        if self._type == "str":
            return self._default
        if self._type == "extruder":
            return self._default # TODO: also allow for other values below machine_extruder_count
        print("Unknown setting type:", self._type)

    ## Return a random value for this setting. The returned value will be a valid value according to the settings json file.
    def getRandomValue(self):
        if self._type == "float" or self._type == "int":
            min = -2
            if self._min_value_warning is not None:
                min = self._min_value_warning
            if self._min_value is not None:
                min = self._min_value
            max = 10000
            if self._max_value_warning is not None:
                max = self._max_value_warning
            if self._max_value is not None:
                max = self._max_value
            if self._type == "int":
                return random.randint(int(min), int(max))
            return random.uniform(float(min), float(max))
        return random.choice(self.getSettingValues())

    ##  Evaluates a setting value that is described as a function.
    #
    #   Note that this function should behave EXACTLY the same as it does in
    #   UM/Settings/Setting.py:_createFunction. The only differences should be
    #   that this evaluation always uses the default values instead of the
    #   current profile values, and that this function directly evaluates the
    #   setting instead of returning a function with which to evaluate the
    #   setting. Also, this function doesn't need to compile the list of
    #   settings that this setting depends on.
    #
    #   \param code The string to evaluate as a function of default values of
    #   other settings.
    #   \param locals The default values of other settings, as dictionary keyed
    #   by the setting names.
    #   \return The evaluated value of the setting, or None if \p code was None.
    def _evaluateFunction(self, code, locals):
        if not code: #The input was None. This setting value doesn't exist in the JSON.
            return None
        try:
            tree = ast.parse(code, "eval")
            compiled = compile(code, self._key, "eval")
        except (SyntaxError, TypeError) as e:
            print("Parse error in function (" + code + ") for setting", self._key + ":", str(e))
        except IllegalMethodError as e:
            print("Use of illegal method", str(e), "in function (" + code + ") for setting", self._key)
        except Exception as e:
            print("Exception in function (" + code + ") for setting", self._key + ":", str(e))

        return eval(compiled, globals(), locals)

class EngineTest:
    def __init__(self, json_filename, engine_filename, models):
        self._json_filename = json_filename
        self._json = json.load(open(json_filename, "r"))
        self._locals = {}
        self._addAllLocals() #Fills the _locals dictionary.
        self._engine = engine_filename
        self._models = models
        self._settings = {}
        self._test_results = TestResults()

        self._flattenAllSettings()

    def _flattenAllSettings(self):
        for key, data in self._json["settings"].items(): # top level settings are categories
            self._flattenSettings(data["children"]) # actual settings are children of top level category-settings

    def _flattenSettings(self, settings):
            for key, setting in settings.items():
                if not ("type" in setting and setting["type"] == "category"):
                    self._settings[key] = Setting(key, setting, self._locals)
                if "children" in setting:
                    self._flattenSettings(setting["children"])

    def testDefaults(self):
        suite = self._test_results.addTestSuite("Defaults")
        self._runTest(suite, "defaults", {})
        return suite.getFailureCount()

    def testSingleChanges(self):
        suite = self._test_results.addTestSuite("SingleSetting")
        for key, setting in self._settings.items():
            for value in setting.getSettingValues():
                self._runTest(suite, key, {key: value})
        return suite.getFailureCount()

    def testSingleRandom(self):
        suite = self._test_results.addTestSuite("SingleRandom")
        for key, setting in self._settings.items():
            self._runTest(suite, key, {key: setting.getRandomValue()})
        return suite.getFailureCount()

    def testDualRandom(self):
        suite = self._test_results.addTestSuite("DualRandom")
        for key, setting in self._settings.items():
            for key2, setting2 in self._settings.items():
                if key != key2:
                    self._runTest(suite, key, {key: setting.getRandomValue(), key2: setting2.getRandomValue()})
        return suite.getFailureCount()

    def testAllRandom(self, amount):
        suite = self._test_results.addTestSuite("AllRandom_%d" % (amount))
        for n in range(0, amount):
            settings = {}
            for key, setting in self._settings.items():
                settings[key] = setting.getRandomValue()
            self._runTest(suite, "Random", settings)
        return suite.getFailureCount()

    def _runTest(self, suite, class_name, settings):
        test_name = ', '.join("{!s}={!r}".format(key, val) for (key,val) in settings.items())
        for model in self._models:
            this_test_name = '%s.%s' % (os.path.basename(model), test_name)
            cmd = [self._engine, "slice", "-j", self._json_filename, "-o", "/dev/null"]
            for key, value in settings.items():
                cmd += ['-s', '%s=%s' % (key, value)]
            cmd += ["-l", model]
            error = self._runProcess(cmd)
            if error is not None:
                suite.failure(class_name, this_test_name, error)
            else:
                suite.success(class_name, this_test_name)

    def _runProcess(self, cmd):
        p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        p.error = ""
        t = threading.Thread(target=self._abortProcess, args=(p,))
        t.start()
        stdout, stderr = p.communicate()
        if p.error == "Timeout":
            return "Timeout: %s" % (' '.join(cmd))
        if p.wait() != 0:
            return "Execution failed: %s" % (' '.join(cmd))
        return None

    def _abortProcess(self, p):
        time.sleep(60)
        if p.poll() is None:
            p.terminate()
            p.error = "Timeout"

    def getResults(self):
        return self._test_results

    ##  Adds all default values for all settings to the locals.
    #
    #   The results are stored in self._locals, keyed by the setting name.
    def _addAllLocals(self):
        for key, data in self._json["settings"].items(): # top level categories
            self._addLocals(data["children"]) # the actual settings in each category

    ##  Adds the default values in a node of the setting tree to the locals.
    #
    #   The results are stored in self._locals, keyed by the setting name.
    #
    #   \param settings The JSON node of which to add the default values.
    def _addLocals(self, settings):
        for key, setting in settings.items():
            if not ("type" in setting and setting["type"] == "category"): # skip category-settings
                self._locals[key] = setting["default_value"]
            if "children" in setting:
                self._addLocals(setting["children"]) #Recursively go down the tree.

def main(engine, model_path):
    filenames = sorted(os.listdir(model_path), key=lambda filename: os.stat(os.path.join(model_path, filename)).st_size)
    filenames = list(filter(lambda filename: filename.lower().endswith(".stl"), filenames))
    for filename in filenames:
        print("Slicing: %s (%d/%d)" % (filename, filenames.index(filename), len(filenames)))
        t = time.time()
        p = subprocess.Popen([engine, "-vv", os.path.join(model_path, filename)], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        stdout, stderr = p.communicate()
        if p.wait() != 0:
            print ("Engine failed to report success on test object: %s" % (filename))
            print(stderr.decode("utf-8", "replace").split("\n")[-5:])
            sys.exit(1)
        else:
            print("Slicing took: %f" % (time.time() - t))
if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="CuraEngine testing script")
    parser.add_argument("--simple", action="store_true", help="Only run the single test, exit")
    parser.add_argument("json", type=str, help="Machine JSON file to use")
    parser.add_argument("engine", type=str, help="Engine executable")
    parser.add_argument("models", type=str, nargs="+", help="List of models to use for testing")
    args = parser.parse_args()

    et = EngineTest(args.json, args.engine, args.models)
    if et.testDefaults() == 0:
        if not args.simple:
            et.testSingleChanges()
            if et.testSingleRandom() == 0:
                et.testDualRandom()
            if et.testAllRandom(10) == 0:
                et.testAllRandom(100)
    et.getResults().saveXML("output.xml")
    if args.simple:
        if et.getResults().getFailureCount() > 0:
            sys.exit(1)<commit_msg>Don't evaluate default_value as fallback<commit_after>#!/usr/bin/python3

## runtest.py
# The runtest.py script runs regression tests on the CuraEngine.
# It parses the json file for settings to know which settings can be passed towards the engine.
# It runs the following test:
# * Defaults
# * Bounds/possible values
# * Single random value
# * All settings random

import ast #For safe function evaluation.
import math #For evaluating setting inheritance functions.
import sys
import subprocess
import os
import time
import stat
import argparse
import random
import json
import threading
from xml.etree import ElementTree

## The TestSuite class stores the test results of a single set of tests.
#  TestSuite objects are created by the TestResults class.
class TestSuite:
    def __init__(self, name):
        self._name = name
        self._successes = []
        self._failures = []

    ## Add a successful test result to the test suite.
    def success(self, class_name, test_name):
        self._successes.append((class_name, test_name))

    ## Add a failed test result to the test suite.
    def failure(self, class_name, test_name, error_message):
        self._failures.append((class_name, test_name, error_message))

    ## Return the number of tests in this test suite
    def getTestCount(self):
        return self.getSuccessCount() + self.getFailureCount()

    ## Return the number of successful tests in this test suite
    def getSuccessCount(self):
        return len(self._successes)

    ## Return the number of failed tests in this test suite
    def getFailureCount(self):
        return len(self._failures)

## The TestResults class stores a group of TestSuite objects, each TestSuite object contains failed and successful test.
#  This class can output the result of the tests in a JUnit xml format for parsing in Jenkins.
class TestResults:
    def __init__(self):
        self._testsuites = []

    ## Create a new test suite with the name.
    def addTestSuite(self, name):
        suite = TestSuite(name)
        self._testsuites.append(suite)
        return suite

    def getFailureCount(self):
        result = 0
        for testsuite in self._testsuites:
            result += testsuite.getFailureCount()
        return result

    ## Save the test results to the file given in the filename.
    def saveXML(self, filename):
        xml = ElementTree.Element("testsuites")
        xml.text = "\n"
        for testsuite in self._testsuites:
            testsuite_xml = ElementTree.SubElement(xml, "testsuite", {"name": testsuite._name, "errors": "0", "tests": str(testsuite.getTestCount()), "failures": str(testsuite.getFailureCount()), "time": "0", "timestamp": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime())})
            testsuite_xml.text = "\n"
            testsuite_xml.tail = "\n"
            for class_name, test_name in testsuite._successes:
                testcase_xml = ElementTree.SubElement(testsuite_xml, "testcase", {"classname": class_name, "name": test_name})
                testcase_xml.text = "\n"
                testcase_xml.tail = "\n"
            for class_name, test_name, error_message in testsuite._failures:
                testcase_xml = ElementTree.SubElement(testsuite_xml, "testcase", {"classname": class_name, "name": test_name})
                testcase_xml.text = "\n"
                testcase_xml.tail = "\n"
                failure = ElementTree.SubElement(testcase_xml, "failure", {"message": "test failure"})
                failure.text = error_message
                failure.tail = "\n"
        return ElementTree.ElementTree(xml).write(filename, "utf-8", True)

class Setting:
    ##  Creates a new setting from a JSON node.
    #
    #   Some parts of the setting may have to be evaluated as functions. For
    #   these, the default values of all settings are added as local variables.
    #
    #   \param key The name of the setting.
    #   \param data The JSON node of the setting, containing the default value,
    #   the setting type, the minimum value, maximum value, the minimum warning
    #   value, the maximum warning value and (for enum types) the options.
    #   \param locals The local variables for eventual function evaluation.
    def __init__(self, key, data, locals):
        self._key = key
        if "value" in data: #Evaluate "value" if we can, otherwise just take default_value.
            self._default = self._evaluateFunction(data.get("value", "0"), locals)
        else:
            self._default = data.get("default_value", 0)
        self._type = data["type"]
        self._min_value = self._evaluateFunction(data.get("minimum_value", None), locals)
        self._max_value = self._evaluateFunction(data.get("maximum_value", None), locals)
        self._min_value_warning = self._evaluateFunction(data.get("minimum_value_warning", None), locals)
        self._max_value_warning = self._evaluateFunction(data.get("maximum_value_warning", None), locals)
        self._options = data.get("options", None)
        if self._options is not None:
            self._options = list(self._options.keys())

    ##  Gets the default value of this setting, according to the source JSON.
    #
    #   \return The default value.
    def getDefault(self):
        return self._default

    ## Return a list of possible values for this setting. This list depends on the setting type.
    #  For number values it contains the minimal and maximal values.
    #  For enums and booleans it will contain the exact possible values.
    #  For string settings only the default value is returned.
    def getSettingValues(self):
        if self._type == "bool":
            return ["True", "False"]
        if self._type == "float" or self._type == "int":
            ret = [self._default]
            if self._min_value is not None:
                ret.append(self._min_value)
            else:
                ret.append(-2)
            if self._min_value_warning is not None:
                ret.append(self._min_value_warning)
            if self._max_value is not None:
                ret.append(self._max_value)
            else:
                if self._max_value_warning is None:
                    ret.append(10000)
                elif self._type == "float":
                    ret.append(float(self._max_value_warning) + 100)
                    ret.append(float(self._max_value_warning) * 10)
                elif self._type == "int":
                    ret.append(int(self._max_value_warning) + 100)
                    ret.append(int(self._max_value_warning) * 10)
                # If the type is boolean, string or enum, the warning values make no sense anyway, so don't test them.
            if self._max_value_warning is not None:
                ret.append(self._max_value_warning)

            if self._type == "int":
                for n in range(0, len(ret)):
                    ret[n] = int(ret[n])
            return ret
        if self._type == "enum":
            return self._options
        if self._type == "str":
            return self._default
        if self._type == "extruder":
            return self._default # TODO: also allow for other values below machine_extruder_count
        print("Unknown setting type:", self._type)

    ## Return a random value for this setting. The returned value will be a valid value according to the settings json file.
    def getRandomValue(self):
        if self._type == "float" or self._type == "int":
            min = -2
            if self._min_value_warning is not None:
                min = self._min_value_warning
            if self._min_value is not None:
                min = self._min_value
            max = 10000
            if self._max_value_warning is not None:
                max = self._max_value_warning
            if self._max_value is not None:
                max = self._max_value
            if self._type == "int":
                return random.randint(int(min), int(max))
            return random.uniform(float(min), float(max))
        return random.choice(self.getSettingValues())

    ##  Evaluates a setting value that is described as a function.
    #
    #   Note that this function should behave EXACTLY the same as it does in
    #   UM/Settings/Setting.py:_createFunction. The only differences should be
    #   that this evaluation always uses the default values instead of the
    #   current profile values, and that this function directly evaluates the
    #   setting instead of returning a function with which to evaluate the
    #   setting. Also, this function doesn't need to compile the list of
    #   settings that this setting depends on.
    #
    #   \param code The string to evaluate as a function of default values of
    #   other settings.
    #   \param locals The default values of other settings, as dictionary keyed
    #   by the setting names.
    #   \return The evaluated value of the setting, or None if \p code was None.
    def _evaluateFunction(self, code, locals):
        if not code: #The input was None. This setting value doesn't exist in the JSON.
            return None
        try:
            tree = ast.parse(code, "eval")
            compiled = compile(code, self._key, "eval")
        except (SyntaxError, TypeError) as e:
            print("Parse error in function (" + code + ") for setting", self._key + ":", str(e))
        except IllegalMethodError as e:
            print("Use of illegal method", str(e), "in function (" + code + ") for setting", self._key)
        except Exception as e:
            print("Exception in function (" + code + ") for setting", self._key + ":", str(e))

        return eval(compiled, globals(), locals)

class EngineTest:
    def __init__(self, json_filename, engine_filename, models):
        self._json_filename = json_filename
        self._json = json.load(open(json_filename, "r"))
        self._locals = {}
        self._addAllLocals() #Fills the _locals dictionary.
        self._engine = engine_filename
        self._models = models
        self._settings = {}
        self._test_results = TestResults()

        self._flattenAllSettings()

    def _flattenAllSettings(self):
        for key, data in self._json["settings"].items(): # top level settings are categories
            self._flattenSettings(data["children"]) # actual settings are children of top level category-settings

    def _flattenSettings(self, settings):
            for key, setting in settings.items():
                if not ("type" in setting and setting["type"] == "category"):
                    self._settings[key] = Setting(key, setting, self._locals)
                if "children" in setting:
                    self._flattenSettings(setting["children"])

    def testDefaults(self):
        suite = self._test_results.addTestSuite("Defaults")
        self._runTest(suite, "defaults", {})
        return suite.getFailureCount()

    def testSingleChanges(self):
        suite = self._test_results.addTestSuite("SingleSetting")
        for key, setting in self._settings.items():
            for value in setting.getSettingValues():
                self._runTest(suite, key, {key: value})
        return suite.getFailureCount()

    def testSingleRandom(self):
        suite = self._test_results.addTestSuite("SingleRandom")
        for key, setting in self._settings.items():
            self._runTest(suite, key, {key: setting.getRandomValue()})
        return suite.getFailureCount()

    def testDualRandom(self):
        suite = self._test_results.addTestSuite("DualRandom")
        for key, setting in self._settings.items():
            for key2, setting2 in self._settings.items():
                if key != key2:
                    self._runTest(suite, key, {key: setting.getRandomValue(), key2: setting2.getRandomValue()})
        return suite.getFailureCount()

    def testAllRandom(self, amount):
        suite = self._test_results.addTestSuite("AllRandom_%d" % (amount))
        for n in range(0, amount):
            settings = {}
            for key, setting in self._settings.items():
                settings[key] = setting.getRandomValue()
            self._runTest(suite, "Random", settings)
        return suite.getFailureCount()

    def _runTest(self, suite, class_name, settings):
        test_name = ', '.join("{!s}={!r}".format(key, val) for (key,val) in settings.items())
        for model in self._models:
            this_test_name = '%s.%s' % (os.path.basename(model), test_name)
            cmd = [self._engine, "slice", "-j", self._json_filename, "-o", "/dev/null"]
            for key, value in settings.items():
                cmd += ['-s', '%s=%s' % (key, value)]
            cmd += ["-l", model]
            error = self._runProcess(cmd)
            if error is not None:
                suite.failure(class_name, this_test_name, error)
            else:
                suite.success(class_name, this_test_name)

    def _runProcess(self, cmd):
        p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        p.error = ""
        t = threading.Thread(target=self._abortProcess, args=(p,))
        t.start()
        stdout, stderr = p.communicate()
        if p.error == "Timeout":
            return "Timeout: %s" % (' '.join(cmd))
        if p.wait() != 0:
            return "Execution failed: %s" % (' '.join(cmd))
        return None

    def _abortProcess(self, p):
        time.sleep(60)
        if p.poll() is None:
            p.terminate()
            p.error = "Timeout"

    def getResults(self):
        return self._test_results

    ##  Adds all default values for all settings to the locals.
    #
    #   The results are stored in self._locals, keyed by the setting name.
    def _addAllLocals(self):
        for key, data in self._json["settings"].items(): # top level categories
            self._addLocals(data["children"]) # the actual settings in each category

    ##  Adds the default values in a node of the setting tree to the locals.
    #
    #   The results are stored in self._locals, keyed by the setting name.
    #
    #   \param settings The JSON node of which to add the default values.
    def _addLocals(self, settings):
        for key, setting in settings.items():
            if not ("type" in setting and setting["type"] == "category"): # skip category-settings
                self._locals[key] = setting["default_value"]
            if "children" in setting:
                self._addLocals(setting["children"]) #Recursively go down the tree.

def main(engine, model_path):
    filenames = sorted(os.listdir(model_path), key=lambda filename: os.stat(os.path.join(model_path, filename)).st_size)
    filenames = list(filter(lambda filename: filename.lower().endswith(".stl"), filenames))
    for filename in filenames:
        print("Slicing: %s (%d/%d)" % (filename, filenames.index(filename), len(filenames)))
        t = time.time()
        p = subprocess.Popen([engine, "-vv", os.path.join(model_path, filename)], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        stdout, stderr = p.communicate()
        if p.wait() != 0:
            print ("Engine failed to report success on test object: %s" % (filename))
            print(stderr.decode("utf-8", "replace").split("\n")[-5:])
            sys.exit(1)
        else:
            print("Slicing took: %f" % (time.time() - t))
if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="CuraEngine testing script")
    parser.add_argument("--simple", action="store_true", help="Only run the single test, exit")
    parser.add_argument("json", type=str, help="Machine JSON file to use")
    parser.add_argument("engine", type=str, help="Engine executable")
    parser.add_argument("models", type=str, nargs="+", help="List of models to use for testing")
    args = parser.parse_args()

    et = EngineTest(args.json, args.engine, args.models)
    if et.testDefaults() == 0:
        if not args.simple:
            et.testSingleChanges()
            if et.testSingleRandom() == 0:
                et.testDualRandom()
            if et.testAllRandom(10) == 0:
                et.testAllRandom(100)
    et.getResults().saveXML("output.xml")
    if args.simple:
        if et.getResults().getFailureCount() > 0:
            sys.exit(1)<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from runner.koan import *

# Greed is a dice game where you roll up to five dice to accumulate
# points.  The following "score" function will be used calculate the
# score of a single roll of the dice.
#
# A greed roll is scored as follows:
#
# * A set of three ones is 1000 points
#
# * A set of three numbers (other than ones) is worth 100 times the
#   number. (e.g. three fives is 500 points).
#
# * A one (that is not part of a set of three) is worth 100 points.
#
# * A five (that is not part of a set of three) is worth 50 points.
#
# * Everything else is worth 0 points.
#
#
# Examples:
#
# score([1,1,1,5,1]) => 1150 points
# score([2,3,4,6,2]) => 0 points
# score([3,4,5,3,3]) => 350 points
# score([1,5,1,2,4]) => 250 points
#
# More scoring examples are given in the tests below:
#
# Your goal is to write the score method.

def score(dice):
    # You need to write this method
    pass

class AboutScoringProject(Koan):
    def test_score_of_an_empty_list_is_zero(self):
        self.assertEqual(0, score([]))

    def test_score_of_a_single_roll_of_5_is_50(self):
        self.assertEqual(50, score([5]))

    def test_score_of_a_single_roll_of_1_is_100(self):
        self.assertEqual(100, score([1]))

    def test_score_of_multiple_1s_and_5s_is_the_sum_of_individual_scores(self):
        self.assertEqual(300, score([1,5,5,1]))

    def test_score_of_single_2s_3s_4s_and_6s_are_zero(self):
        self.assertEqual(0, score([2,3,4,6]))

    def test_score_of_a_triple_1_is_1000(self):
        self.assertEqual(1000, score([1,1,1]))

    def test_score_of_other_triples_is_100x(self):
        self.assertEqual(200, score([2,2,2]))
        self.assertEqual(300, score([3,3,3]))
        self.assertEqual(400, score([4,4,4]))
        self.assertEqual(500, score([5,5,5]))
        self.assertEqual(600, score([6,6,6]))

    def test_score_of_mixed_is_sum(self):
        self.assertEqual(250, score([2,5,2,2,3]))
        self.assertEqual(550, score([5,5,5,5]))
        self.assertEqual(1150, score([1,1,1,5,1]))

    def test_ones_not_left_out(self):
        self.assertEqual(300, score([1,2,2,2]))
        self.assertEqual(350, score([1,5,2,2,2]))<commit_msg>[Python] Complete 'About Scoring Project' koans<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from runner.koan import *

# Greed is a dice game where you roll up to five dice to accumulate
# points.  The following "score" function will be used calculate the
# score of a single roll of the dice.
#
# A greed roll is scored as follows:
#
# * A set of three ones is 1000 points
#
# * A set of three numbers (other than ones) is worth 100 times the
#   number. (e.g. three fives is 500 points).
#
# * A one (that is not part of a set of three) is worth 100 points.
#
# * A five (that is not part of a set of three) is worth 50 points.
#
# * Everything else is worth 0 points.
#
#
# Examples:
#
# score([1,1,1,5,1]) => 1150 points
# score([2,3,4,6,2]) => 0 points
# score([3,4,5,3,3]) => 350 points
# score([1,5,1,2,4]) => 250 points
#
# More scoring examples are given in the tests below:
#
# Your goal is to write the score method.

def score(dice):
    # Initial data
    numbers = {}.fromkeys(range(1, 7), 0)
    points = 0

    # Fill dictionary with dice values
    for i in dice:
        numbers[i] += 1

    # Algorithm
    points += (numbers[1] // 3) * 1000
    points += (numbers[1] % 3) * 100
    points += (numbers[5] % 3) * 50
    for n in range(2, 7):
        points += (numbers[n] // 3) * 100 * n

    return points

class AboutScoringProject(Koan):
    def test_score_of_an_empty_list_is_zero(self):
        self.assertEqual(0, score([]))

    def test_score_of_a_single_roll_of_5_is_50(self):
        self.assertEqual(50, score([5]))

    def test_score_of_a_single_roll_of_1_is_100(self):
        self.assertEqual(100, score([1]))

    def test_score_of_multiple_1s_and_5s_is_the_sum_of_individual_scores(self):
        self.assertEqual(300, score([1,5,5,1]))

    def test_score_of_single_2s_3s_4s_and_6s_are_zero(self):
        self.assertEqual(0, score([2,3,4,6]))

    def test_score_of_a_triple_1_is_1000(self):
        self.assertEqual(1000, score([1,1,1]))

    def test_score_of_other_triples_is_100x(self):
        self.assertEqual(200, score([2,2,2]))
        self.assertEqual(300, score([3,3,3]))
        self.assertEqual(400, score([4,4,4]))
        self.assertEqual(500, score([5,5,5]))
        self.assertEqual(600, score([6,6,6]))

    def test_score_of_mixed_is_sum(self):
        self.assertEqual(250, score([2,5,2,2,3]))
        self.assertEqual(550, score([5,5,5,5]))
        self.assertEqual(1150, score([1,1,1,5,1]))

    def test_ones_not_left_out(self):
        self.assertEqual(300, score([1,2,2,2]))
        self.assertEqual(350, score([1,5,2,2,2]))
<|endoftext|>


<commit_before>"""Miscellaneous utility functions useful for dealing with ESIS streams."""
__version__ = '$Revision$'

import re
import string
import sys
import xml.dom.core
import xml.dom.esis_builder

_data_rx = re.compile(r"[^\\][^\\]*")

def decode(s):
    r = ''
    while s:
        m = _data_rx.match(s)
        if m:
            r = r + m.group()
            s = s[len(m.group()):]
        elif s[1] == "\\":
            r = r + "\\"
            s = s[2:]
        elif s[1] == "n":
            r = r + "\n"
            s = s[2:]
        else:
            raise ValueError, "can't handle " + `s`
    return r

_charmap = {}
for c in map(chr, range(256)):
    _charmap[c] = c
_charmap["\n"] = r"\n"
_charmap["\\"] = r"\\"
del c

def encode(s):
    return string.join(map(_charmap.get, s), '')

class ExtendedEsisBuilder(xml.dom.esis_builder.EsisBuilder):
    def __init__(self, *args, **kw):
        self.__empties = {}
        self.__is_empty = 0
        apply(xml.dom.esis_builder.EsisBuilder.__init__, (self,) + args, kw)
        self.buildFragment()

    def feed(self, data):
        for line in string.split(data, '\n'):
            if not line:
                break
            event = line[0]
            text = line[1:]
            if event == '(':
                element = self.document.createElement(text, self.attr_store)
                self.attr_store = {}
                self.push(element)
                if self.__is_empty:
                    self.__empties[text] = text
                    self.__is_empty = 0
            elif event == ')':
                self.pop()
            elif event == 'A':
                l = re.split(' ', text, 2)
                name = l[0]
                value = decode(l[2])
                self.attr_store[name] = value
            elif event == '-':
                text = self.document.createText(decode(text))
                self.push(text)
            elif event == 'C':
                return
            elif event == 'e':
                self.__is_empty = 1
            else:
                sys.stderr.write('Unknown event: %s\n' % line)

    def get_empties(self):
        return self.__empties.keys()
<commit_msg>feed():  Added support for ESIS '&' lines.<commit_after>"""Miscellaneous utility functions useful for dealing with ESIS streams."""
__version__ = '$Revision$'

import re
import string
import sys
import xml.dom.core
import xml.dom.esis_builder

_data_rx = re.compile(r"[^\\][^\\]*")

def decode(s):
    r = ''
    while s:
        m = _data_rx.match(s)
        if m:
            r = r + m.group()
            s = s[len(m.group()):]
        elif s[1] == "\\":
            r = r + "\\"
            s = s[2:]
        elif s[1] == "n":
            r = r + "\n"
            s = s[2:]
        else:
            raise ValueError, "can't handle " + `s`
    return r

_charmap = {}
for c in map(chr, range(256)):
    _charmap[c] = c
_charmap["\n"] = r"\n"
_charmap["\\"] = r"\\"
del c

def encode(s):
    return string.join(map(_charmap.get, s), '')

class ExtendedEsisBuilder(xml.dom.esis_builder.EsisBuilder):
    def __init__(self, *args, **kw):
        self.__empties = {}
        self.__is_empty = 0
        apply(xml.dom.esis_builder.EsisBuilder.__init__, (self,) + args, kw)
        self.buildFragment()

    def feed(self, data):
        for line in string.split(data, '\n'):
            if not line:
                break
            event = line[0]
            text = line[1:]
            if event == '(':
                element = self.document.createElement(text, self.attr_store)
                self.attr_store = {}
                self.push(element)
                if self.__is_empty:
                    self.__empties[text] = text
                    self.__is_empty = 0
            elif event == ')':
                self.pop()
            elif event == 'A':
                l = re.split(' ', text, 2)
                name = l[0]
                value = decode(l[2])
                self.attr_store[name] = value
            elif event == '-':
                text = self.document.createText(decode(text))
                self.push(text)
            elif event == 'C':
                return
            elif event == 'e':
                self.__is_empty = 1
            elif event == '&':
                eref = self.document.createEntityReference(text)
                self.push(eref)
            else:
                sys.stderr.write('Unknown event: %s\n' % line)

    def get_empties(self):
        return self.__empties.keys()
<|endoftext|>


<commit_before># Copyright 2020 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import abc
import contextlib
import functools
import logging
from typing import Any, Dict, List, Optional

from absl import flags
from google.cloud import secretmanager_v1
from google.longrunning import operations_pb2
from google.protobuf import json_format
from google.protobuf import text_format
from google.rpc import code_pb2
from google.rpc import error_details_pb2
from googleapiclient import discovery
import googleapiclient.errors
import googleapiclient.http
import tenacity
import yaml

import framework.helpers.highlighter

logger = logging.getLogger(__name__)
PRIVATE_API_KEY_SECRET_NAME = flags.DEFINE_string(
    "private_api_key_secret_name",
    default=None,
    help="Load Private API access key from the latest version of the secret "
    "with the given name, in the format projects/*/secrets/*")
V1_DISCOVERY_URI = flags.DEFINE_string("v1_discovery_uri",
                                       default=discovery.V1_DISCOVERY_URI,
                                       help="Override v1 Discovery URI")
V2_DISCOVERY_URI = flags.DEFINE_string("v2_discovery_uri",
                                       default=discovery.V2_DISCOVERY_URI,
                                       help="Override v2 Discovery URI")
COMPUTE_V1_DISCOVERY_FILE = flags.DEFINE_string(
    "compute_v1_discovery_file",
    default=None,
    help="Load compute v1 from discovery file")
GCP_UI_URL = flags.DEFINE_string("gcp_ui_url",
                                 default="console.cloud.google.com",
                                 help="Override GCP UI URL.")

# Type aliases
_HttpError = googleapiclient.errors.HttpError
_HttpLib2Error = googleapiclient.http.httplib2.HttpLib2Error
_HighlighterYaml = framework.helpers.highlighter.HighlighterYaml
Operation = operations_pb2.Operation
HttpRequest = googleapiclient.http.HttpRequest

class GcpApiManager:

    def __init__(self,
                 *,
                 v1_discovery_uri=None,
                 v2_discovery_uri=None,
                 compute_v1_discovery_file=None,
                 private_api_key_secret_name=None,
                 gcp_ui_url=None):
        self.v1_discovery_uri = v1_discovery_uri or V1_DISCOVERY_URI.value
        self.v2_discovery_uri = v2_discovery_uri or V2_DISCOVERY_URI.value
        self.compute_v1_discovery_file = (compute_v1_discovery_file or
                                          COMPUTE_V1_DISCOVERY_FILE.value)
        self.private_api_key_secret_name = (private_api_key_secret_name or
                                            PRIVATE_API_KEY_SECRET_NAME.value)
        self.gcp_ui_url = gcp_ui_url or GCP_UI_URL.value
        # TODO(sergiitk): add options to pass google Credentials
        self._exit_stack = contextlib.ExitStack()

    def close(self):
        self._exit_stack.close()

    @property
    @functools.lru_cache(None)
    def private_api_key(self):
        """
        Private API key.

        Return API key credential that identifies a GCP project allow-listed for
        accessing private API discovery documents.
        https://console.cloud.google.com/apis/credentials

        This method lazy-loads the content of the key from the Secret Manager.
        https://console.cloud.google.com/security/secret-manager
        """
        if not self.private_api_key_secret_name:
            raise ValueError('private_api_key_secret_name must be set to '
                             'access private_api_key.')

        secrets_api = self.secrets('v1')
        version_resource_path = secrets_api.secret_version_path(
            **secrets_api.parse_secret_path(self.private_api_key_secret_name),
            secret_version='latest')
        secret: secretmanager_v1.AccessSecretVersionResponse
        secret = secrets_api.access_secret_version(name=version_resource_path)
        return secret.payload.data.decode()

    @functools.lru_cache(None)
    def compute(self, version):
        api_name = 'compute'
        if version == 'v1':
            if self.compute_v1_discovery_file:
                return self._build_from_file(self.compute_v1_discovery_file)
            else:
                return self._build_from_discovery_v1(api_name, version)
        elif version == 'v1alpha':
            return self._build_from_discovery_v1(api_name, 'alpha')

        raise NotImplementedError(f'Compute {version} not supported')

    @functools.lru_cache(None)
    def networksecurity(self, version):
        api_name = 'networksecurity'
        if version == 'v1alpha1':
            return self._build_from_discovery_v2(
                api_name,
                version,
                api_key=self.private_api_key,
                visibility_labels=['NETWORKSECURITY_ALPHA'])
        elif version == 'v1beta1':
            return self._build_from_discovery_v2(api_name, version)

        raise NotImplementedError(f'Network Security {version} not supported')

    @functools.lru_cache(None)
    def networkservices(self, version):
        api_name = 'networkservices'
        if version == 'v1alpha1':
            return self._build_from_discovery_v2(
                api_name,
                version,
                api_key=self.private_api_key,
                visibility_labels=['NETWORKSERVICES_ALPHA'])
        elif version == 'v1beta1':
            return self._build_from_discovery_v2(api_name, version)

        raise NotImplementedError(f'Network Services {version} not supported')

    @functools.lru_cache(None)
    def secrets(self, version):
        if version == 'v1':
            return secretmanager_v1.SecretManagerServiceClient()

        raise NotImplementedError(f'Secret Manager {version} not supported')

    @functools.lru_cache(None)
    def iam(self, version: str) -> discovery.Resource:
        """Identity and Access Management (IAM) API.

        https://cloud.google.com/iam/docs/reference/rest
        https://googleapis.github.io/google-api-python-client/docs/dyn/iam_v1.html
        """
        api_name = 'iam'
        if version == 'v1':
            return self._build_from_discovery_v1(api_name, version)

        raise NotImplementedError(
            f'Identity and Access Management (IAM) {version} not supported')

    def _build_from_discovery_v1(self, api_name, version):
        api = discovery.build(api_name,
                              version,
                              cache_discovery=False,
                              discoveryServiceUrl=self.v1_discovery_uri)
        self._exit_stack.enter_context(api)
        return api

    def _build_from_discovery_v2(self,
                                 api_name,
                                 version,
                                 *,
                                 api_key: Optional[str] = None,
                                 visibility_labels: Optional[List] = None):
        params = {}
        if api_key:
            params['key'] = api_key
        if visibility_labels:
            # Dash-separated list of labels.
            params['labels'] = '_'.join(visibility_labels)

        params_str = ''
        if params:
            params_str = '&' + ('&'.join(f'{k}={v}' for k, v in params.items()))

        api = discovery.build(
            api_name,
            version,
            cache_discovery=False,
            discoveryServiceUrl=f'{self.v2_discovery_uri}{params_str}')
        self._exit_stack.enter_context(api)
        return api

    def _build_from_file(self, discovery_file):
        with open(discovery_file, 'r') as f:
            api = discovery.build_from_document(f.read())
        self._exit_stack.enter_context(api)
        return api

class Error(Exception):
    """Base error class for GCP API errors."""

class ResponseError(Error):
    """The response was not a 2xx."""
    reason: str
    uri: str
    error_details: Optional[str]
    status: Optional[int]
    cause: _HttpError

    def __init__(self, cause: _HttpError):
        # TODO(sergiitk): cleanup when we upgrade googleapiclient:
        #  - remove _get_reason()
        #  - remove error_details note
        #  - use status_code()
        self.reason = cause._get_reason().strip()  # noqa
        self.uri = cause.uri
        self.error_details = cause.error_details  # NOTE: Must after _get_reason
        self.status = None
        if cause.resp and cause.resp.status:
            self.status = cause.resp.status
        self.cause = cause
        super().__init__()

    def __repr__(self):
        return (f'<ResponseError {self.status} when requesting {self.uri} '
                f'returned "{self.reason}". Details: "{self.error_details}">')

class TransportError(Error):
    """A transport error has occurred."""
    cause: _HttpLib2Error

    def __init__(self, cause: _HttpLib2Error):
        self.cause = cause
        super().__init__()

    def __repr__(self):
        return f'<TransportError cause: {self.cause!r}>'

class OperationError(Error):
    """
    Operation was not successful.

    Assuming Operation based on Google API Style Guide:
    https://cloud.google.com/apis/design/design_patterns#long_running_operations
    https://github.com/googleapis/googleapis/blob/master/google/longrunning/operations.proto
    """

    def __init__(self, api_name, operation_response, message=None):
        self.api_name = api_name
        operation = json_format.ParseDict(
            operation_response,
            Operation(),
            ignore_unknown_fields=True,
            descriptor_pool=error_details_pb2.DESCRIPTOR.pool)
        self.name = operation.name or 'unknown'
        self.code_name = code_pb2.Code.Name(operation.error.code)
        self.error = operation.error
        # Collect error details packed as Any without parsing concrete types.
        self.error_details = [
            text_format.MessageToString(any_error, as_one_line=True)
            for any_error in self.error.details
        ]
        if message is None:
            message = (f'{api_name} operation "{self.name}" failed. Error '
                       f'code: {self.error.code} ({self.code_name}), '
                       f'message: {self.error.message}, '
                       f'details: {self.error_details}')
        self.message = message
        super().__init__(message)

class GcpProjectApiResource:
    # TODO(sergiitk): move someplace better
    _WAIT_FOR_OPERATION_SEC = 60 * 5
    _WAIT_FIXED_SEC = 2
    _GCP_API_RETRIES = 5

    def __init__(self, api: discovery.Resource, project: str):
        self.api: discovery.Resource = api
        self.project: str = project
        self._highlighter = _HighlighterYaml()

    # TODO(sergiitk): in upcoming GCP refactoring, differentiate between
    #   _execute for LRO (Long Running Operations), and immediate operations.
    def _execute(
            self,
            request: HttpRequest,
            *,
            num_retries: Optional[int] = _GCP_API_RETRIES) -> Dict[str, Any]:
        """Execute the immediate request.

        Returns:
          Unmarshalled response as a dictionary.

        Raises:
          ResponseError if the response was not a 2xx.
          TransportError if a transport error has occurred.
        """
        if num_retries is None:
            num_retries = self._GCP_API_RETRIES
        try:
            return request.execute(num_retries=num_retries)
        except _HttpError as error:
            raise ResponseError(error)
        except _HttpLib2Error as error:
            raise TransportError(error)

    def resource_pretty_format(self, body: dict) -> str:
        """Return a string with pretty-printed resource body."""
        yaml_out: str = yaml.dump(body, explicit_start=True, explicit_end=True)
        return self._highlighter.highlight(yaml_out)

    @staticmethod
    def wait_for_operation(operation_request,
                           test_success_fn,
                           timeout_sec=_WAIT_FOR_OPERATION_SEC,
                           wait_sec=_WAIT_FIXED_SEC):
        retryer = tenacity.Retrying(
            retry=(tenacity.retry_if_not_result(test_success_fn) |
                   tenacity.retry_if_exception_type()),
            wait=tenacity.wait_fixed(wait_sec),
            stop=tenacity.stop_after_delay(timeout_sec),
            after=tenacity.after_log(logger, logging.DEBUG),
            reraise=True)
        return retryer(operation_request.execute)

class GcpStandardCloudApiResource(GcpProjectApiResource, metaclass=abc.ABCMeta):
    GLOBAL_LOCATION = 'global'

    def parent(self, location: Optional[str] = GLOBAL_LOCATION):
        if location is None:
            location = self.GLOBAL_LOCATION
        return f'projects/{self.project}/locations/{location}'

    def resource_full_name(self, name, collection_name):
        return f'{self.parent()}/{collection_name}/{name}'

    def _create_resource(self, collection: discovery.Resource, body: dict,
                         **kwargs):
        logger.info("Creating %s resource:\n%s", self.api_name,
                    self.resource_pretty_format(body))
        create_req = collection.create(parent=self.parent(),
                                       body=body,
                                       **kwargs)
        self._execute(create_req)

    @property
    @abc.abstractmethod
    def api_name(self) -> str:
        raise NotImplementedError

    @property
    @abc.abstractmethod
    def api_version(self) -> str:
        raise NotImplementedError

    def _get_resource(self, collection: discovery.Resource, full_name):
        resource = collection.get(name=full_name).execute()
        logger.info('Loaded %s:\n%s', full_name,
                    self.resource_pretty_format(resource))
        return resource

    def _delete_resource(self, collection: discovery.Resource,
                         full_name: str) -> bool:
        logger.debug("Deleting %s", full_name)
        try:
            self._execute(collection.delete(name=full_name))
            return True
        except _HttpError as error:
            if error.resp and error.resp.status == 404:
                logger.info('%s not deleted since it does not exist', full_name)
            else:
                logger.warning('Failed to delete %s, %r', full_name, error)
        return False

    # TODO(sergiitk): Use ResponseError and TransportError
    def _execute(self,
                 request: HttpRequest,
                 timeout_sec=GcpProjectApiResource._WAIT_FOR_OPERATION_SEC):
        operation = request.execute(num_retries=self._GCP_API_RETRIES)
        self._wait(operation, timeout_sec)

    def _wait(self,
              operation,
              timeout_sec=GcpProjectApiResource._WAIT_FOR_OPERATION_SEC):
        op_name = operation['name']
        logger.debug('Waiting for %s operation, timeout %s sec: %s',
                     self.api_name, timeout_sec, op_name)

        op_request = self.api.projects().locations().operations().get(
            name=op_name)
        operation = self.wait_for_operation(
            operation_request=op_request,
            test_success_fn=lambda result: result['done'],
            timeout_sec=timeout_sec)

        logger.debug('Completed operation: %s', operation)
        if 'error' in operation:
            raise OperationError(self.api_name, operation)
<commit_msg>xds-k8s: Fix the issue with parsing Operation.metadata (#28736)<commit_after># Copyright 2020 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import abc
import contextlib
import functools
import json
import logging
from typing import Any, Dict, List, Optional

from absl import flags
from google.cloud import secretmanager_v1
from google.longrunning import operations_pb2
from google.protobuf import json_format
from google.rpc import code_pb2
from google.rpc import error_details_pb2
from google.rpc import status_pb2
from googleapiclient import discovery
import googleapiclient.errors
import googleapiclient.http
import tenacity
import yaml

import framework.helpers.highlighter

logger = logging.getLogger(__name__)
PRIVATE_API_KEY_SECRET_NAME = flags.DEFINE_string(
    "private_api_key_secret_name",
    default=None,
    help="Load Private API access key from the latest version of the secret "
    "with the given name, in the format projects/*/secrets/*")
V1_DISCOVERY_URI = flags.DEFINE_string("v1_discovery_uri",
                                       default=discovery.V1_DISCOVERY_URI,
                                       help="Override v1 Discovery URI")
V2_DISCOVERY_URI = flags.DEFINE_string("v2_discovery_uri",
                                       default=discovery.V2_DISCOVERY_URI,
                                       help="Override v2 Discovery URI")
COMPUTE_V1_DISCOVERY_FILE = flags.DEFINE_string(
    "compute_v1_discovery_file",
    default=None,
    help="Load compute v1 from discovery file")
GCP_UI_URL = flags.DEFINE_string("gcp_ui_url",
                                 default="console.cloud.google.com",
                                 help="Override GCP UI URL.")

# Type aliases
_HttpError = googleapiclient.errors.HttpError
_HttpLib2Error = googleapiclient.http.httplib2.HttpLib2Error
_HighlighterYaml = framework.helpers.highlighter.HighlighterYaml
Operation = operations_pb2.Operation
HttpRequest = googleapiclient.http.HttpRequest

class GcpApiManager:

    def __init__(self,
                 *,
                 v1_discovery_uri=None,
                 v2_discovery_uri=None,
                 compute_v1_discovery_file=None,
                 private_api_key_secret_name=None,
                 gcp_ui_url=None):
        self.v1_discovery_uri = v1_discovery_uri or V1_DISCOVERY_URI.value
        self.v2_discovery_uri = v2_discovery_uri or V2_DISCOVERY_URI.value
        self.compute_v1_discovery_file = (compute_v1_discovery_file or
                                          COMPUTE_V1_DISCOVERY_FILE.value)
        self.private_api_key_secret_name = (private_api_key_secret_name or
                                            PRIVATE_API_KEY_SECRET_NAME.value)
        self.gcp_ui_url = gcp_ui_url or GCP_UI_URL.value
        # TODO(sergiitk): add options to pass google Credentials
        self._exit_stack = contextlib.ExitStack()

    def close(self):
        self._exit_stack.close()

    @property
    @functools.lru_cache(None)
    def private_api_key(self):
        """
        Private API key.

        Return API key credential that identifies a GCP project allow-listed for
        accessing private API discovery documents.
        https://console.cloud.google.com/apis/credentials

        This method lazy-loads the content of the key from the Secret Manager.
        https://console.cloud.google.com/security/secret-manager
        """
        if not self.private_api_key_secret_name:
            raise ValueError('private_api_key_secret_name must be set to '
                             'access private_api_key.')

        secrets_api = self.secrets('v1')
        version_resource_path = secrets_api.secret_version_path(
            **secrets_api.parse_secret_path(self.private_api_key_secret_name),
            secret_version='latest')
        secret: secretmanager_v1.AccessSecretVersionResponse
        secret = secrets_api.access_secret_version(name=version_resource_path)
        return secret.payload.data.decode()

    @functools.lru_cache(None)
    def compute(self, version):
        api_name = 'compute'
        if version == 'v1':
            if self.compute_v1_discovery_file:
                return self._build_from_file(self.compute_v1_discovery_file)
            else:
                return self._build_from_discovery_v1(api_name, version)
        elif version == 'v1alpha':
            return self._build_from_discovery_v1(api_name, 'alpha')

        raise NotImplementedError(f'Compute {version} not supported')

    @functools.lru_cache(None)
    def networksecurity(self, version):
        api_name = 'networksecurity'
        if version == 'v1alpha1':
            return self._build_from_discovery_v2(
                api_name,
                version,
                api_key=self.private_api_key,
                visibility_labels=['NETWORKSECURITY_ALPHA'])
        elif version == 'v1beta1':
            return self._build_from_discovery_v2(api_name, version)

        raise NotImplementedError(f'Network Security {version} not supported')

    @functools.lru_cache(None)
    def networkservices(self, version):
        api_name = 'networkservices'
        if version == 'v1alpha1':
            return self._build_from_discovery_v2(
                api_name,
                version,
                api_key=self.private_api_key,
                visibility_labels=['NETWORKSERVICES_ALPHA'])
        elif version == 'v1beta1':
            return self._build_from_discovery_v2(api_name, version)

        raise NotImplementedError(f'Network Services {version} not supported')

    @functools.lru_cache(None)
    def secrets(self, version):
        if version == 'v1':
            return secretmanager_v1.SecretManagerServiceClient()

        raise NotImplementedError(f'Secret Manager {version} not supported')

    @functools.lru_cache(None)
    def iam(self, version: str) -> discovery.Resource:
        """Identity and Access Management (IAM) API.

        https://cloud.google.com/iam/docs/reference/rest
        https://googleapis.github.io/google-api-python-client/docs/dyn/iam_v1.html
        """
        api_name = 'iam'
        if version == 'v1':
            return self._build_from_discovery_v1(api_name, version)

        raise NotImplementedError(
            f'Identity and Access Management (IAM) {version} not supported')

    def _build_from_discovery_v1(self, api_name, version):
        api = discovery.build(api_name,
                              version,
                              cache_discovery=False,
                              discoveryServiceUrl=self.v1_discovery_uri)
        self._exit_stack.enter_context(api)
        return api

    def _build_from_discovery_v2(self,
                                 api_name,
                                 version,
                                 *,
                                 api_key: Optional[str] = None,
                                 visibility_labels: Optional[List] = None):
        params = {}
        if api_key:
            params['key'] = api_key
        if visibility_labels:
            # Dash-separated list of labels.
            params['labels'] = '_'.join(visibility_labels)

        params_str = ''
        if params:
            params_str = '&' + ('&'.join(f'{k}={v}' for k, v in params.items()))

        api = discovery.build(
            api_name,
            version,
            cache_discovery=False,
            discoveryServiceUrl=f'{self.v2_discovery_uri}{params_str}')
        self._exit_stack.enter_context(api)
        return api

    def _build_from_file(self, discovery_file):
        with open(discovery_file, 'r') as f:
            api = discovery.build_from_document(f.read())
        self._exit_stack.enter_context(api)
        return api

class Error(Exception):
    """Base error class for GCP API errors."""

class ResponseError(Error):
    """The response was not a 2xx."""
    reason: str
    uri: str
    error_details: Optional[str]
    status: Optional[int]
    cause: _HttpError

    def __init__(self, cause: _HttpError):
        # TODO(sergiitk): cleanup when we upgrade googleapiclient:
        #  - remove _get_reason()
        #  - remove error_details note
        #  - use status_code()
        self.reason = cause._get_reason().strip()  # noqa
        self.uri = cause.uri
        self.error_details = cause.error_details  # NOTE: Must after _get_reason
        self.status = None
        if cause.resp and cause.resp.status:
            self.status = cause.resp.status
        self.cause = cause
        super().__init__()

    def __repr__(self):
        return (f'<ResponseError {self.status} when requesting {self.uri} '
                f'returned "{self.reason}". Details: "{self.error_details}">')

class TransportError(Error):
    """A transport error has occurred."""
    cause: _HttpLib2Error

    def __init__(self, cause: _HttpLib2Error):
        self.cause = cause
        super().__init__()

    def __repr__(self):
        return f'<TransportError cause: {self.cause!r}>'

class OperationError(Error):
    """
    Operation was not successful.

    Assuming Operation based on Google API Style Guide:
    https://cloud.google.com/apis/design/design_patterns#long_running_operations
    https://github.com/googleapis/googleapis/blob/master/google/longrunning/operations.proto
    """
    api_name: str
    name: str
    metadata: str
    code_name: code_pb2.Code
    error: status_pb2.Status

    def __init__(self, api_name: str, response: dict):
        self.api_name = api_name

        # Operation.metadata field is Any specific to the API. It may not be
        # present in the default descriptor pool, and that's expected.
        # To avoid json_format.ParseError, handle it separately.
        self.metadata = response.pop('metadata', {})

        # Must be after removing metadata field.
        operation: Operation = self._parse_operation_response(response)
        self.name = operation.name or 'unknown'
        self.code_name = code_pb2.Code.Name(operation.error.code)
        self.error = operation.error
        super().__init__()

    @staticmethod
    def _parse_operation_response(operation_response: dict) -> Operation:
        try:
            return json_format.ParseDict(
                operation_response,
                Operation(),
                ignore_unknown_fields=True,
                descriptor_pool=error_details_pb2.DESCRIPTOR.pool)
        except (json_format.Error, TypeError) as e:
            # Swallow parsing errors if any. Building correct OperationError()
            # is more important than losing debug information. Details still
            # can be extracted from the warning.
            logger.warning(
                ("Can't parse response while processing OperationError: '%r', "
                 "error %r"), operation_response, e)
            return Operation()

    def __str__(self):
        indent_l1 = ' ' * 2
        indent_l2 = indent_l1 * 2

        result = (f'{self.api_name} operation "{self.name}" failed.\n'
                  f'{indent_l1}code: {self.error.code} ({self.code_name})\n'
                  f'{indent_l1}message: "{self.error.message}"')

        if self.error.details:
            result += f'\n{indent_l1}details: [\n'
            for any_error in self.error.details:
                error_str = json_format.MessageToJson(any_error)
                for line in error_str.splitlines():
                    result += indent_l2 + line + '\n'
            result += f'{indent_l1}]'

        if self.metadata:
            result += f'\n  metadata: \n'
            metadata_str = json.dumps(self.metadata, indent=2)
            for line in metadata_str.splitlines():
                result += indent_l2 + line + '\n'
            result = result.rstrip()

        return result

class GcpProjectApiResource:
    # TODO(sergiitk): move someplace better
    _WAIT_FOR_OPERATION_SEC = 60 * 5
    _WAIT_FIXED_SEC = 2
    _GCP_API_RETRIES = 5

    def __init__(self, api: discovery.Resource, project: str):
        self.api: discovery.Resource = api
        self.project: str = project
        self._highlighter = _HighlighterYaml()

    # TODO(sergiitk): in upcoming GCP refactoring, differentiate between
    #   _execute for LRO (Long Running Operations), and immediate operations.
    def _execute(
            self,
            request: HttpRequest,
            *,
            num_retries: Optional[int] = _GCP_API_RETRIES) -> Dict[str, Any]:
        """Execute the immediate request.

        Returns:
          Unmarshalled response as a dictionary.

        Raises:
          ResponseError if the response was not a 2xx.
          TransportError if a transport error has occurred.
        """
        if num_retries is None:
            num_retries = self._GCP_API_RETRIES
        try:
            return request.execute(num_retries=num_retries)
        except _HttpError as error:
            raise ResponseError(error)
        except _HttpLib2Error as error:
            raise TransportError(error)

    def resource_pretty_format(self, body: dict) -> str:
        """Return a string with pretty-printed resource body."""
        yaml_out: str = yaml.dump(body, explicit_start=True, explicit_end=True)
        return self._highlighter.highlight(yaml_out)

    @staticmethod
    def wait_for_operation(operation_request,
                           test_success_fn,
                           timeout_sec=_WAIT_FOR_OPERATION_SEC,
                           wait_sec=_WAIT_FIXED_SEC):
        retryer = tenacity.Retrying(
            retry=(tenacity.retry_if_not_result(test_success_fn) |
                   tenacity.retry_if_exception_type()),
            wait=tenacity.wait_fixed(wait_sec),
            stop=tenacity.stop_after_delay(timeout_sec),
            after=tenacity.after_log(logger, logging.DEBUG),
            reraise=True)
        return retryer(operation_request.execute)

class GcpStandardCloudApiResource(GcpProjectApiResource, metaclass=abc.ABCMeta):
    GLOBAL_LOCATION = 'global'

    def parent(self, location: Optional[str] = GLOBAL_LOCATION):
        if location is None:
            location = self.GLOBAL_LOCATION
        return f'projects/{self.project}/locations/{location}'

    def resource_full_name(self, name, collection_name):
        return f'{self.parent()}/{collection_name}/{name}'

    def _create_resource(self, collection: discovery.Resource, body: dict,
                         **kwargs):
        logger.info("Creating %s resource:\n%s", self.api_name,
                    self.resource_pretty_format(body))
        create_req = collection.create(parent=self.parent(),
                                       body=body,
                                       **kwargs)
        self._execute(create_req)

    @property
    @abc.abstractmethod
    def api_name(self) -> str:
        raise NotImplementedError

    @property
    @abc.abstractmethod
    def api_version(self) -> str:
        raise NotImplementedError

    def _get_resource(self, collection: discovery.Resource, full_name):
        resource = collection.get(name=full_name).execute()
        logger.info('Loaded %s:\n%s', full_name,
                    self.resource_pretty_format(resource))
        return resource

    def _delete_resource(self, collection: discovery.Resource,
                         full_name: str) -> bool:
        logger.debug("Deleting %s", full_name)
        try:
            self._execute(collection.delete(name=full_name))
            return True
        except _HttpError as error:
            if error.resp and error.resp.status == 404:
                logger.info('%s not deleted since it does not exist', full_name)
            else:
                logger.warning('Failed to delete %s, %r', full_name, error)
        return False

    # TODO(sergiitk): Use ResponseError and TransportError
    def _execute(self,
                 request: HttpRequest,
                 timeout_sec=GcpProjectApiResource._WAIT_FOR_OPERATION_SEC):
        operation = request.execute(num_retries=self._GCP_API_RETRIES)
        self._wait(operation, timeout_sec)

    def _wait(self,
              operation,
              timeout_sec=GcpProjectApiResource._WAIT_FOR_OPERATION_SEC):
        op_name = operation['name']
        logger.debug('Waiting for %s operation, timeout %s sec: %s',
                     self.api_name, timeout_sec, op_name)

        op_request = self.api.projects().locations().operations().get(
            name=op_name)
        operation = self.wait_for_operation(
            operation_request=op_request,
            test_success_fn=lambda result: result['done'],
            timeout_sec=timeout_sec)

        logger.debug('Completed operation: %s', operation)
        if 'error' in operation:
            raise OperationError(self.api_name, operation)
<|endoftext|>


<commit_before>__version__ = "0.4.18"
<commit_msg>bump version<commit_after>__version__ = "0.4.19"
<|endoftext|>


<commit_before>"""
Admin interface models. Automatically detected by admin.autodiscover().
"""
from django.contrib import admin
from eve_api.models import *

from eve_api.api_puller.accounts import import_eve_account

def account_api_update(modeladmin, request, queryset):
    for obj in queryset:
        import_eve_account(obj.api_key, obj.api_user_id)

account_api_update.short_description = "Update account from the EVE API"

class EVEAccountAdmin(admin.ModelAdmin):
    list_display = ('id', 'user', 'api_status', 'api_last_updated')
    search_fields = ['id']

    actions = [account_api_update]

admin.site.register(EVEAccount, EVEAccountAdmin)

class EVEPlayerCharacterAdmin(admin.ModelAdmin):
    list_display = ('id', 'name', 'corporation')
    search_fields = ['id', 'name']
admin.site.register(EVEPlayerCharacter, EVEPlayerCharacterAdmin)

class EVEPlayerCorporationInline(admin.TabularInline):
    model = EVEPlayerCorporation
    fields = ('name', 'ticker')
    extra = 0

class EVEPlayerAllianceAdmin(admin.ModelAdmin):
    list_display = ('id', 'name', 'ticker', 'member_count', 'date_founded')
    search_fields = ['name', 'ticker']
    date_hierarchy = 'date_founded'
    inlines = [EVEPlayerCorporationInline]
admin.site.register(EVEPlayerAlliance, EVEPlayerAllianceAdmin)

class EVEPlayerCorporationAdmin(admin.ModelAdmin):
    list_display = ('id', 'name', 'ticker', 'member_count', 'alliance')
    search_fields = ['name', 'ticker']
admin.site.register(EVEPlayerCorporation, EVEPlayerCorporationAdmin)
<commit_msg>Add keytype to the admin overview<commit_after>"""
Admin interface models. Automatically detected by admin.autodiscover().
"""
from django.contrib import admin
from eve_api.models import *

from eve_api.api_puller.accounts import import_eve_account

def account_api_update(modeladmin, request, queryset):
    for obj in queryset:
        import_eve_account(obj.api_key, obj.api_user_id)

account_api_update.short_description = "Update account from the EVE API"

class EVEAccountAdmin(admin.ModelAdmin):
    list_display = ('id', 'user', 'api_keytype', 'api_status', 'api_last_updated')
    search_fields = ['id']

    actions = [account_api_update]

admin.site.register(EVEAccount, EVEAccountAdmin)

class EVEPlayerCharacterAdmin(admin.ModelAdmin):
    list_display = ('id', 'name', 'corporation')
    search_fields = ['id', 'name']
admin.site.register(EVEPlayerCharacter, EVEPlayerCharacterAdmin)

class EVEPlayerCorporationInline(admin.TabularInline):
    model = EVEPlayerCorporation
    fields = ('name', 'ticker')
    extra = 0

class EVEPlayerAllianceAdmin(admin.ModelAdmin):
    list_display = ('id', 'name', 'ticker', 'member_count', 'date_founded')
    search_fields = ['name', 'ticker']
    date_hierarchy = 'date_founded'
    inlines = [EVEPlayerCorporationInline]
admin.site.register(EVEPlayerAlliance, EVEPlayerAllianceAdmin)

class EVEPlayerCorporationAdmin(admin.ModelAdmin):
    list_display = ('id', 'name', 'ticker', 'member_count', 'alliance')
    search_fields = ['name', 'ticker']
admin.site.register(EVEPlayerCorporation, EVEPlayerCorporationAdmin)
<|endoftext|>


<commit_before><commit_msg>use fillna instead of replace<commit_after><|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
Returners Directory
'''

import logging
log = logging.getLogger(__name__)

def get_returner_options(virtualname=None,
                         ret=None,
                         attrs=None,
                         **kwargs):
    '''
    Get the returner options from salt.
    '''

    if ret:
        ret_config = '{0}'.format(ret['ret_config']) if 'ret_config' in ret else ''
    else:
        ret_config = None

    profile_attr = kwargs.get('profile_attr', None)
    profile_attrs = kwargs.get('profile_attrs', None)
    defaults = kwargs.get('defaults', None)
    __salt__ = kwargs.get('__salt__', {})
    __opts__ = kwargs.get('__opts__', {})

    _options = {}

    for attr in attrs:
        if 'config.option' in __salt__:
            # Look for the configuration options in __salt__
            # most likely returner is being called from a state or module run

            # cfg is a copy of the config.option function
            cfg = __salt__['config.option']

            # c_cfg is a dictionary returned from config.option for
            # any options configured for this returner.
            c_cfg = cfg('{0}'.format(virtualname), {})
            if ret_config:
                # Using ret_config to override the default configuration key
                ret_cfg = cfg('{0}.{1}'.format(ret_config, virtualname), {})

                # Look for the configuration item in the override location
                # if not found, fall back to the default location.
                if ret_cfg.get(attrs[attr],
                               cfg('{0}.{1}.{2}'.format(ret_config,
                                                        virtualname,
                                                        attrs[attr]))):
                    _attr = ret_cfg.get(attrs[attr],
                                        cfg('{0}.{1}.{2}'.format(ret_config,
                                                                 virtualname,
                                                                 attrs[attr])))
                else:
                    _attr = c_cfg.get(attrs[attr],
                                      cfg('{0}.{1}'.format(virtualname,
                                                           attrs[attr])))
            else:
                # Using the default configuration key
                _attr = c_cfg.get(attrs[attr],
                                  cfg('{0}.{1}'.format(virtualname,
                                                       attrs[attr])))
        else:
            # __salt__ is unavailable, most likely the returner
            # is being called from the Salt scheduler so
            # look for the configuration options in __opts__

            # cfg is a copy for the __opts_ dictionary
            cfg = __opts__

            # c_cfg is a dictionary found in the cfg dictionary
            # otherwise an empty dict.
            c_cfg = cfg.get('{0}'.format(virtualname), {})

            if ret_config:
                # Using ret_config to override the default configuration key
                ret_cfg = cfg.get('{0}.{1}'.format(ret_config, virtualname), {})

                # Look for the configuration item in the override location
                # if not found, fall back to the default location.
                if ret_cfg.get(attrs[attr],
                               cfg.get('{0}.{1}.{2}'.format(ret_config,
                                                            virtualname,
                                                            attrs[attr]))):
                    _attr = ret_cfg.get(attrs[attr],
                                        cfg.get('{0}.{1}.{2}'.format(ret_config,
                                                                     virtualname,
                                                                     attrs[attr])))
                else:
                    _attr = c_cfg.get(attrs[attr],
                                      cfg.get('{0}.{1}'.format(virtualname,
                                                               attrs[attr])))
            else:
                # Using the default configuration key.
                _attr = c_cfg.get(attrs[attr],
                                  cfg.get('{0}.{1}'.format(virtualname,
                                                           attrs[attr])))
        if not _attr:
            # Attribute not found, check for a default value
            if defaults:
                if attr in defaults:
                    log.info('Using default for {0} {1}'.format(virtualname,
                                                                attr))
                    _options[attr] = defaults[attr]
                    continue
                else:
                    _options[attr] = ''
                    continue
            else:
                _options[attr] = ''
                continue
        _options[attr] = _attr

    if profile_attr:
        # Using a profile
        if profile_attr in _options:
            log.info('Using profile {0}'.format(_options[profile_attr]))
            if 'config.option' in __salt__:
                creds = cfg(_options[profile_attr])
            else:
                creds = cfg.get(_options[profile_attr])
            if creds:
                for pattr in profile_attrs:
                    _options[pattr] = creds.get('{0}.{1}'.format(virtualname,
                                                                 profile_attrs[pattr]))
    return _options
<commit_msg>Optimize attr fetching<commit_after># -*- coding: utf-8 -*-
'''
Returners Directory
'''

import logging
log = logging.getLogger(__name__)

def get_returner_options(virtualname=None,
                         ret=None,
                         attrs=None,
                         **kwargs):
    '''
    Get the returner options from salt.
    '''

    if ret:
        ret_config = '{0}'.format(ret['ret_config']) if 'ret_config' in ret else ''
    else:
        ret_config = None

    profile_attr = kwargs.get('profile_attr', None)
    profile_attrs = kwargs.get('profile_attrs', None)
    defaults = kwargs.get('defaults', None)
    __salt__ = kwargs.get('__salt__', {})
    __opts__ = kwargs.get('__opts__', {})

    _options = {}

    for attr in attrs:

        # default place for the option in the config
        default_cfg_key = '{0}.{1}'.format(virtualname, attrs[attr])

        if 'config.option' in __salt__:
            # Look for the configuration options in __salt__
            # most likely returner is being called from a state or module run

            # cfg is a copy of the config.option function
            cfg = __salt__['config.option']

            # c_cfg is a dictionary returned from config.option for
            # any options configured for this returner.
            c_cfg = cfg('{0}'.format(virtualname), {})
            if ret_config:
                # Using ret_config to override the default configuration key
                ret_cfg = cfg('{0}.{1}'.format(ret_config, virtualname), {})

                # Look for the configuration item in the override location
                # if not found, fall back to the default location.
                override_default_cfg_key = '{0}.{1}.{2}'.format(
                    ret_config,
                    virtualname,
                    attrs[attr]
                )
                override_cfg_default = cfg(override_default_cfg_key)
                ret_override_cfg = ret_cfg.get(
                    attrs[attr],
                    override_cfg_default
                )
                if ret_override_cfg:
                    _attr = ret_override_cfg
                else:
                    _attr = c_cfg.get(attrs[attr], cfg(default_cfg_key))
            else:
                # Using the default configuration key
                _attr = c_cfg.get(attrs[attr], cfg(default_cfg_key))
        else:
            # __salt__ is unavailable, most likely the returner
            # is being called from the Salt scheduler so
            # look for the configuration options in __opts__

            # cfg is a copy for the __opts_ dictionary
            cfg = __opts__

            # c_cfg is a dictionary found in the cfg dictionary
            # otherwise an empty dict.
            c_cfg = cfg.get('{0}'.format(virtualname), {})

            if ret_config:
                # Using ret_config to override the default configuration key
                ret_cfg = cfg.get('{0}.{1}'.format(ret_config, virtualname), {})

                # Look for the configuration item in the override location
                # if not found, fall back to the default location.
                if ret_cfg.get(attrs[attr],
                               cfg.get('{0}.{1}.{2}'.format(ret_config,
                                                            virtualname,
                                                            attrs[attr]))):
                    _attr = ret_cfg.get(attrs[attr],
                                        cfg.get('{0}.{1}.{2}'.format(ret_config,
                                                                     virtualname,
                                                                     attrs[attr])))
                else:
                    _attr = c_cfg.get(attrs[attr],
                                      cfg.get('{0}.{1}'.format(virtualname,
                                                               attrs[attr])))
            else:
                # Using the default configuration key.
                _attr = c_cfg.get(attrs[attr],
                                  cfg.get('{0}.{1}'.format(virtualname,
                                                           attrs[attr])))
        if not _attr:
            # Attribute not found, check for a default value
            if defaults:
                if attr in defaults:
                    log.info('Using default for {0} {1}'.format(virtualname,
                                                                attr))
                    _options[attr] = defaults[attr]
                    continue
                else:
                    _options[attr] = ''
                    continue
            else:
                _options[attr] = ''
                continue
        _options[attr] = _attr

    if profile_attr:
        # Using a profile
        if profile_attr in _options:
            log.info('Using profile {0}'.format(_options[profile_attr]))
            if 'config.option' in __salt__:
                creds = cfg(_options[profile_attr])
            else:
                creds = cfg.get(_options[profile_attr])
            if creds:
                for pattr in profile_attrs:
                    _options[pattr] = creds.get('{0}.{1}'.format(virtualname,
                                                                 profile_attrs[pattr]))
    return _options
<|endoftext|>