<commit_before>import copy
import inspect
import logging
from functools import partialmethod

from gen.exceptions import ValidationError
from pkgpanda.util import json_prettyprint

log = logging.getLogger(__name__)

def get_function_parameters(function):
    return set(inspect.signature(function).parameters)

def validate_arguments_strings(arguments: dict):
    errors = dict()
    # Validate that all keys and vlaues of arguments are strings
    for k, v in arguments.items():
        if not isinstance(k, str):
            errors[''] = "All keys in arguments must be strings. '{}' isn't.".format(k)
        if not isinstance(v, str):
            errors[k] = ("All values in arguments must be strings. Value for argument {} isn't. " +
                         "Given value: {}").format(k, v)
    if len(errors):
        raise ValidationError(errors, set())

class Setter:

    # NOTE: value may either be a function or a string.
    def __init__(self, name, value, is_optional, conditions, is_user):
        assert isinstance(conditions, list)
        self.name = name
        self.is_optional = is_optional
        self.conditions = conditions
        self.is_user = is_user

        def get_value():
            return value

        if isinstance(value, str):
            self.calc = get_value
            self.parameters = set()
        else:
            assert callable(value), "{} should be a string or callable. Got: {}".format(name, value)
            self.calc = value
            self.parameters = get_function_parameters(value)

    def __repr__(self):
        return "<Setter {}{}{}, conditions: {}{}>".format(
            self.name,
            ", optional" if self.is_optional else "",
            ", user" if self.is_user else "",
            self.conditions,
            ", parameters {}".format(self.parameters))

class Scope:
    def __init__(self, name: str, cases=None):
        self.name = name
        self.cases = cases if cases else dict()

    def add_case(self, value: str, target):
        assert isinstance(target, Target)
        self.cases[value] = target

    def __iadd__(self, other):
        assert isinstance(other, Scope), "Internal consistency error, expected Scope but got {}".format(type(other))

        # Must have the same name and same options in order to be merged (can't have a new
        # switch add new, unhandled cases to a switch already included / represented by this scope).
        assert self.name == other.name, "Internal consistency error: Trying to merge scopes with " \
            "different names: {} and {}".format(self.name, other.name)
        assert self.cases.keys() == other.cases.keys(), "Same name / switch variable introduced " \
            "with a different set of possible cases. name: {}. First options: {}, Second " \
            "options: {}".format(self.name, self.cases.keys(), other.cases.keys())

        # Merge the targets for the cases
        for name in self.cases:
            self.cases[name] += other.cases[name]

        return self

    def __eq__(self, other):
        assert isinstance(other, Scope)
        return self.name == other.name and self.cases == other.cases

class Target:

    # TODO(cmaloney): Make a better API for working with and managing sub scopes. The current
    # dictionary of dictionaries is really hard to use right.
    def __init__(self, variables=None, sub_scopes=None):
        self.variables = variables if variables else set()
        self.sub_scopes = sub_scopes if sub_scopes else dict()

    def add_variable(self, variable: str):
        self.variables.add(variable)

    def add_scope(self, scope: Scope):
        self.sub_scopes[scope.name] = scope

    def __iadd__(self, other):
        assert isinstance(other, Target), "Internal consistency error, expected Target but got {}".format(type(other))
        self.variables |= other.variables

        # Add all scopes from the other to this, merging all common sub scopes.
        for name, scope in other.sub_scopes.items():
            if name in self.sub_scopes:
                self.sub_scopes[name] += scope
            else:
                self.sub_scopes[name] = scope

        return self

    def __eq__(self, other):
        assert isinstance(other, Target)
        return self.variables == other.variables and self.sub_scopes == other.sub_scopes

class Source:
    def __init__(self, entry=None, is_user=False,):
        self.setters = dict()
        self.validate = list()
        self.is_user = is_user
        if entry:
            self.add_entry(entry, False)

    def add_setter(self, name, value, is_optional, conditions):
        self.setters.setdefault(name, list()).append(Setter(name, value, is_optional, conditions, self.is_user))

    def add_conditional_scope(self, scope, conditions):
        # TODO(cmaloney): 'defaults' are the same as 'can' and 'must' is identical to 'arguments' except
        # that one takes functions and one takes strings. Simplify to just 'can', 'must'.
        assert scope.keys() <= {'validate', 'default', 'must', 'conditional'}

        self.validate += scope.get('validate', list())

        for name, fn in scope.get('must', dict()).items():
            self.add_setter(name, fn, False, conditions)

        for name, fn in scope.get('default', dict()).items():
            self.add_setter(name, fn, True, conditions)

        for name, cond_options in scope.get('conditional', dict()).items():
            for value, sub_scope in cond_options.items():
                self.add_conditional_scope(sub_scope, conditions + [(name, value)])

    add_must = partialmethod(add_setter, is_optional=False, conditions=[])

    def add_value_dict(self, value_dict):
        for name, value in value_dict.items():
            self.add_must(name, value)

    def remove_setters(self, scope):
        def del_setter(name):
            if name in self.setters:
                del self.setters[name]

        for name in scope.get('must', dict()).keys():
            del_setter(name)

        for name in scope.get('default', dict()).keys():
            del_setter(name)

        for name, cond_options in scope.get('conditional', dict()).items():
            if name in self.setters:
                raise NotImplementedError("Should conditional setters overwrite all setters?")

    def add_entry(self, entry, replace_existing):
        if replace_existing:
            self.remove_setters(entry)

        self.add_conditional_scope(entry, [])

# NOTE: This exception should never escape the DFSArgumentCalculator
class CalculatorError(Exception):
    def __init__(self, message, chain=[]):
        assert isinstance(message, str)
        assert isinstance(chain, list)
        self.message = message
        self.chain = chain
        super().__init__(message)

# Depth first search argument calculator. Detects cycles, as well as unmet
# dependencies.
# TODO(cmaloney): Separate chain / path building when unwinding from the root
#                 error messages.
class DFSArgumentCalculator():
    def __init__(self, setters, validate_fns):
        self._setters = setters
        self._arguments = dict()
        self.__in_progress = set()
        self._errors = dict()
        self._unset = set()

        # Re-arrange the validation functions so we can more easily access them by
        # argument name.
        self._validate_by_arg = dict()
        self._multi_arg_validate = dict()

        for fn in validate_fns:
            parameters = get_function_parameters(fn)
            # Could build up the single and multi parameter validation function maps in the same
            # thing but the timing / handling of when and how we run single vs. multi-parameter
            # validation functions is fairly different, the extra bit here simplifies the later code.
            if len(parameters) == 1:
                self._validate_by_arg[parameters.pop()] = fn
                assert not parameters
            else:
                self._multi_arg_validate[frozenset(parameters)] = fn

    def _calculate_argument(self, name):
        # Filter out any setters which have predicates / conditions which are
        # satisfiably false.
        def all_conditions_met(setter):
            for condition_name, condition_value in setter.conditions:
                try:
                    if self._get(condition_name) != condition_value:
                        return False
                except CalculatorError as ex:
                    raise CalculatorError(
                        ex.message,
                        ex.chain + ['trying to test condition {}={}'.format(condition_name, condition_value)]) from ex
            return True

        # Find the right setter to calculate the argument.
        feasible = list(filter(all_conditions_met, self._setters.get(name, list())))

        if len(feasible) == 0:
            self._unset.add(name)
            raise CalculatorError("no way to set")

        # Filtier out all optional setters if there is more than one way to set.
        if len(feasible) > 1:
            final_feasible = list(filter(lambda setter: not setter.is_optional, feasible))
            assert final_feasible, "Had multiple optionals and no musts. Template internal error: {!r}".format(feasible)
            feasible = final_feasible

        # Must be calculated but user tried to provide.
        if len(feasible) == 2 and (feasible[0].is_user or feasible[1].is_user):
            self._errors[name] = ("{} must be calculated, but was explicitly set in the "
                                  "configuration. Remove it from the configuration.").format(name)
            raise CalculatorError("{} must be calculated but set twice".format(name))

        if len(feasible) > 1:
            self._errors[name] = "Internal error: Multiple ways to set {}.".format(name)
            raise CalculatorError("multiple ways to set",
                                  ["options: {}".format(feasible)])

        setter = feasible[0]

        # Get values for the parameters, then call. the setter function.
        kwargs = {}
        for parameter in setter.parameters:
            kwargs[parameter] = self._get(parameter)

        try:
            value = setter.calc(**kwargs)
        except AssertionError as ex:
            self._errors[name] = ex.args[0]
            raise CalculatorError("assertion while calc")

        if name in self._validate_by_arg:
            try:
                self._validate_by_arg[name](value)
            except AssertionError as ex:
                self._errors[name] = ex.args[0]
                raise CalculatorError("assertion while validate")

        return value

    def _get(self, name):
        if name in self._arguments:
            if self._arguments[name] is None:
                raise CalculatorError("No way to set", [name])
            return self._arguments[name]

        # Detect cycles by checking if we're in the middle of calculating the
        # argument being asked for
        if name in self.__in_progress:
            raise CalculatorError("Internal error. cycle detected. re-encountered {}".format(name))

        self.__in_progress.add(name)
        try:
            self._arguments[name] = self._calculate_argument(name)
            return self._arguments[name]
        except CalculatorError as ex:
            self._arguments[name] = None
            raise CalculatorError(ex.message, ex.chain + ['while calculating {}'.format(name)]) from ex
        except:
            self._arguments[name] = None
            raise
        finally:
            self.__in_progress.remove(name)

    def _calculate_target(self, target):
        def evaluate_var(name):
            try:
                self._get(name)
            except CalculatorError as ex:
                log.debug("Error calculating %s: %s. Chain: %s", name, ex.message, ex.chain)

        for name in target.variables:
            evaluate_var(name)

        for name, sub_scope in target.sub_scopes.items():
            if name not in self._arguments:
                evaluate_var(name)

            # If the internal arg is None, there was an error, don't check if it
            # is a legal choice.
            if self._arguments[name] is None:
                continue

            choice = self._get(name)

            if choice not in sub_scope.cases:
                self._errors[name] = "Invalid choice {}. Must choose one of {}".format(
                    choice, ", ".join(sorted(sub_scope.keys())))
                continue

            self._calculate_target(sub_scope.cases[choice])

        # Perform all multi-argument validations
        for parameter_set, validate_fn in self._multi_arg_validate.items():
            # Build up argument map for validate function. If any arguments are
            # unset then skip this validate function.
            kwargs = dict()
            skip = False
            for parameter in parameter_set:
                if (parameter not in self._arguments) or (self._arguments[parameter] is None):
                    skip = True
                    break
                kwargs[parameter] = self._arguments[parameter]
            if skip:
                continue

            # Call the validation function, catching AssertionErrors and turning them into errors in
            # the error dictionary.
            try:
                validate_fn(**kwargs)
            except AssertionError as ex:
                self._errors[parameter_set] = ex.args[0]

        # TODO(cmaloney): Return per-target results with the path to calculate each argument and the
        # full set of arguments touched included.
        return self._arguments

    # Force calculation of all arguments by accessing the arguments in this
    # scope and recursively all sub-scopes.
    def calculate(self, targets):
        for target in targets:
            self._calculate_target(target)

        if len(self._errors) or len(self._unset):
            raise ValidationError(self._errors, self._unset)

        return self._arguments

def resolve_configuration(sources: list, targets: list, user_arguments: dict):

    # Make sure all user provided arguments are strings.
    # TODO(cmaloney): Loosen this restriction  / allow arbitrary types as long
    # as they all have a gen specific string form.
    validate_arguments_strings(user_arguments)

    # Merge the sources into a big dictionary of setters + validators, ensuring
    # that all setters are either strings or functions.
    validate = list()
    setters = dict()

    # Merge all the config targets into one big group of setters for providing
    # to the calculator
    # TODO(cmaloney): The setter management / set code is very similar to that in ConfigTarget, they
    # could probably be joined.
    for source in sources:
        for name, setter_list in source.setters.items():
            setters.setdefault(name, list())
            setters[name] += setter_list
        validate += source.validate

    # Validate that targets is a list of Targets
    for target in targets:
        assert isinstance(target, Target), \
            "target should be a Target found a {} with value: {}".format(type(target), target)

    # TODO(cmaloney): Re-enable this after sorting out how to have "optional" config targets which
    # add in extra "acceptable" parameters (SSH Config, AWS Advanced Template config, etc)
    # validate_all_arguments_match_parameters(mandatory_parameters, setters, user_arguments)

    # Add in all user arguments as setters.
    # Happens last so that they are never overwritten with replace_existing=True
    user_config = Source(is_user=True)
    user_config.add_value_dict(user_arguments)

    # Merge all the seters and validate function into one uber list
    setters = copy.deepcopy(user_config.setters)
    validate = copy.deepcopy(user_config.validate)
    for source in sources:
        for name, setter_list in source.setters.items():
            # TODO(cmaloney): Make a setter manager already...
            setters.setdefault(name, list())
            setters[name] += setter_list
        validate += source.validate

    # Use setters to caluclate every required parameter
    arguments = DFSArgumentCalculator(setters, validate).calculate(targets)

    # Validate all new / calculated arguments are strings.
    validate_arguments_strings(arguments)

    log.info("Final arguments:" + json_prettyprint(arguments))

    # TODO(cmaloney) Give each config target the values for all it's parameters that were hit as
    # well as any parameters that led to those parameters.
    return arguments

def validate_configuration(sources: list, targets: list, user_arguments: dict):
    try:
        resolve_configuration(sources, targets, user_arguments)
        return {'status': 'ok'}
    except ValidationError as ex:
        messages = {}

        # Defer multi-key validation errors and noramlize them to be single-key
        # ones. The multi-key is always less important than the single-key
        # messages which is why single-key messages we never overwrite.
        # TODO(cmaloney): Teach the whole stack to be able to handle multi-key
        # validation errors.
        to_do = dict()
        for key, msg in ex.errors.items():
            if isinstance(key, frozenset):
                to_do[key] = msg
                continue

            assert isinstance(key, str)
            messages[key] = {'message': msg}

        for keys, msg in to_do.items():
            for name in keys:
                # Skip ones we already have one message for.
                if name in messages:
                    continue

                messages[name] = {'message': msg}

        return {
            'status': 'errors',
            'errors': messages,
            'unset': ex.unset
        }
<commit_msg>Fix scope merging bug in building targets from AST<commit_after>import copy
import inspect
import logging
from functools import partialmethod

from gen.exceptions import ValidationError
from pkgpanda.util import json_prettyprint

log = logging.getLogger(__name__)

def get_function_parameters(function):
    return set(inspect.signature(function).parameters)

def validate_arguments_strings(arguments: dict):
    errors = dict()
    # Validate that all keys and vlaues of arguments are strings
    for k, v in arguments.items():
        if not isinstance(k, str):
            errors[''] = "All keys in arguments must be strings. '{}' isn't.".format(k)
        if not isinstance(v, str):
            errors[k] = ("All values in arguments must be strings. Value for argument {} isn't. " +
                         "Given value: {}").format(k, v)
    if len(errors):
        raise ValidationError(errors, set())

class Setter:

    # NOTE: value may either be a function or a string.
    def __init__(self, name, value, is_optional, conditions, is_user):
        assert isinstance(conditions, list)
        self.name = name
        self.is_optional = is_optional
        self.conditions = conditions
        self.is_user = is_user

        def get_value():
            return value

        if isinstance(value, str):
            self.calc = get_value
            self.parameters = set()
        else:
            assert callable(value), "{} should be a string or callable. Got: {}".format(name, value)
            self.calc = value
            self.parameters = get_function_parameters(value)

    def __repr__(self):
        return "<Setter {}{}{}, conditions: {}{}>".format(
            self.name,
            ", optional" if self.is_optional else "",
            ", user" if self.is_user else "",
            self.conditions,
            ", parameters {}".format(self.parameters))

class Scope:
    def __init__(self, name: str, cases=None):
        self.name = name
        self.cases = cases if cases else dict()

    def add_case(self, value: str, target):
        assert isinstance(target, Target)
        self.cases[value] = target

    def __iadd__(self, other):
        assert isinstance(other, Scope), "Internal consistency error, expected Scope but got {}".format(type(other))

        # Must have the same name and same options in order to be merged (can't have a new
        # switch add new, unhandled cases to a switch already included / represented by this scope).
        assert self.name == other.name, "Internal consistency error: Trying to merge scopes with " \
            "different names: {} and {}".format(self.name, other.name)
        assert self.cases.keys() == other.cases.keys(), "Same name / switch variable introduced " \
            "with a different set of possible cases. name: {}. First options: {}, Second " \
            "options: {}".format(self.name, self.cases.keys(), other.cases.keys())

        # Merge the targets for the cases
        for name in self.cases:
            self.cases[name] += other.cases[name]

        return self

    def __eq__(self, other):
        assert isinstance(other, Scope)
        return self.name == other.name and self.cases == other.cases

    def __repr__(self):
        return "<Scope cases: {}>".format(self.cases.items())

class Target:

    # TODO(cmaloney): Make a better API for working with and managing sub scopes. The current
    # dictionary of dictionaries is really hard to use right.
    def __init__(self, variables=None, sub_scopes=None):
        self.variables = variables if variables else set()
        self.sub_scopes = sub_scopes if sub_scopes else dict()

    def add_variable(self, variable: str):
        self.variables.add(variable)

    def add_scope(self, scope: Scope):
        if scope.name in self.sub_scopes:
            self.sub_scopes[scope.name] += scope
        else:
            self.sub_scopes[scope.name] = scope

    def __iadd__(self, other):
        assert isinstance(other, Target), "Internal consistency error, expected Target but got {}".format(type(other))
        self.variables |= other.variables

        # Add all scopes from the other to this, merging all common sub scopes.
        for name, scope in other.sub_scopes.items():
            if name in self.sub_scopes:
                self.sub_scopes[name] += scope
            else:
                self.sub_scopes[name] = scope

        return self

    def __repr__(self):
        return "<Target variables: {}, sub_scopes: {}>".format(self.variables, self.sub_scopes.items())

    def __eq__(self, other):
        assert isinstance(other, Target)
        return self.variables == other.variables and self.sub_scopes == other.sub_scopes

class Source:
    def __init__(self, entry=None, is_user=False,):
        self.setters = dict()
        self.validate = list()
        self.is_user = is_user
        if entry:
            self.add_entry(entry, False)

    def add_setter(self, name, value, is_optional, conditions):
        self.setters.setdefault(name, list()).append(Setter(name, value, is_optional, conditions, self.is_user))

    def add_conditional_scope(self, scope, conditions):
        # TODO(cmaloney): 'defaults' are the same as 'can' and 'must' is identical to 'arguments' except
        # that one takes functions and one takes strings. Simplify to just 'can', 'must'.
        assert scope.keys() <= {'validate', 'default', 'must', 'conditional'}

        self.validate += scope.get('validate', list())

        for name, fn in scope.get('must', dict()).items():
            self.add_setter(name, fn, False, conditions)

        for name, fn in scope.get('default', dict()).items():
            self.add_setter(name, fn, True, conditions)

        for name, cond_options in scope.get('conditional', dict()).items():
            for value, sub_scope in cond_options.items():
                self.add_conditional_scope(sub_scope, conditions + [(name, value)])

    add_must = partialmethod(add_setter, is_optional=False, conditions=[])

    def add_value_dict(self, value_dict):
        for name, value in value_dict.items():
            self.add_must(name, value)

    def remove_setters(self, scope):
        def del_setter(name):
            if name in self.setters:
                del self.setters[name]

        for name in scope.get('must', dict()).keys():
            del_setter(name)

        for name in scope.get('default', dict()).keys():
            del_setter(name)

        for name, cond_options in scope.get('conditional', dict()).items():
            if name in self.setters:
                raise NotImplementedError("Should conditional setters overwrite all setters?")

    def add_entry(self, entry, replace_existing):
        if replace_existing:
            self.remove_setters(entry)

        self.add_conditional_scope(entry, [])

# NOTE: This exception should never escape the DFSArgumentCalculator
class CalculatorError(Exception):
    def __init__(self, message, chain=[]):
        assert isinstance(message, str)
        assert isinstance(chain, list)
        self.message = message
        self.chain = chain
        super().__init__(message)

# Depth first search argument calculator. Detects cycles, as well as unmet
# dependencies.
# TODO(cmaloney): Separate chain / path building when unwinding from the root
#                 error messages.
class DFSArgumentCalculator():
    def __init__(self, setters, validate_fns):
        self._setters = setters
        self._arguments = dict()
        self.__in_progress = set()
        self._errors = dict()
        self._unset = set()

        # Re-arrange the validation functions so we can more easily access them by
        # argument name.
        self._validate_by_arg = dict()
        self._multi_arg_validate = dict()

        for fn in validate_fns:
            parameters = get_function_parameters(fn)
            # Could build up the single and multi parameter validation function maps in the same
            # thing but the timing / handling of when and how we run single vs. multi-parameter
            # validation functions is fairly different, the extra bit here simplifies the later code.
            if len(parameters) == 1:
                self._validate_by_arg[parameters.pop()] = fn
                assert not parameters
            else:
                self._multi_arg_validate[frozenset(parameters)] = fn

    def _calculate_argument(self, name):
        # Filter out any setters which have predicates / conditions which are
        # satisfiably false.
        def all_conditions_met(setter):
            for condition_name, condition_value in setter.conditions:
                try:
                    if self._get(condition_name) != condition_value:
                        return False
                except CalculatorError as ex:
                    raise CalculatorError(
                        ex.message,
                        ex.chain + ['trying to test condition {}={}'.format(condition_name, condition_value)]) from ex
            return True

        # Find the right setter to calculate the argument.
        feasible = list(filter(all_conditions_met, self._setters.get(name, list())))

        if len(feasible) == 0:
            self._unset.add(name)
            raise CalculatorError("no way to set")

        # Filtier out all optional setters if there is more than one way to set.
        if len(feasible) > 1:
            final_feasible = list(filter(lambda setter: not setter.is_optional, feasible))
            assert final_feasible, "Had multiple optionals and no musts. Template internal error: {!r}".format(feasible)
            feasible = final_feasible

        # Must be calculated but user tried to provide.
        if len(feasible) == 2 and (feasible[0].is_user or feasible[1].is_user):
            self._errors[name] = ("{} must be calculated, but was explicitly set in the "
                                  "configuration. Remove it from the configuration.").format(name)
            raise CalculatorError("{} must be calculated but set twice".format(name))

        if len(feasible) > 1:
            self._errors[name] = "Internal error: Multiple ways to set {}.".format(name)
            raise CalculatorError("multiple ways to set",
                                  ["options: {}".format(feasible)])

        setter = feasible[0]

        # Get values for the parameters, then call. the setter function.
        kwargs = {}
        for parameter in setter.parameters:
            kwargs[parameter] = self._get(parameter)

        try:
            value = setter.calc(**kwargs)
        except AssertionError as ex:
            self._errors[name] = ex.args[0]
            raise CalculatorError("assertion while calc")

        if name in self._validate_by_arg:
            try:
                self._validate_by_arg[name](value)
            except AssertionError as ex:
                self._errors[name] = ex.args[0]
                raise CalculatorError("assertion while validate")

        return value

    def _get(self, name):
        if name in self._arguments:
            if self._arguments[name] is None:
                raise CalculatorError("No way to set", [name])
            return self._arguments[name]

        # Detect cycles by checking if we're in the middle of calculating the
        # argument being asked for
        if name in self.__in_progress:
            raise CalculatorError("Internal error. cycle detected. re-encountered {}".format(name))

        self.__in_progress.add(name)
        try:
            self._arguments[name] = self._calculate_argument(name)
            return self._arguments[name]
        except CalculatorError as ex:
            self._arguments[name] = None
            raise CalculatorError(ex.message, ex.chain + ['while calculating {}'.format(name)]) from ex
        except:
            self._arguments[name] = None
            raise
        finally:
            self.__in_progress.remove(name)

    def _calculate_target(self, target):
        def evaluate_var(name):
            try:
                self._get(name)
            except CalculatorError as ex:
                log.debug("Error calculating %s: %s. Chain: %s", name, ex.message, ex.chain)

        for name in target.variables:
            evaluate_var(name)

        for name, sub_scope in target.sub_scopes.items():
            if name not in self._arguments:
                evaluate_var(name)

            # If the internal arg is None, there was an error, don't check if it
            # is a legal choice.
            if self._arguments[name] is None:
                continue

            choice = self._get(name)

            if choice not in sub_scope.cases:
                self._errors[name] = "Invalid choice {}. Must choose one of {}".format(
                    choice, ", ".join(sorted(sub_scope.keys())))
                continue

            self._calculate_target(sub_scope.cases[choice])

        # Perform all multi-argument validations
        for parameter_set, validate_fn in self._multi_arg_validate.items():
            # Build up argument map for validate function. If any arguments are
            # unset then skip this validate function.
            kwargs = dict()
            skip = False
            for parameter in parameter_set:
                if (parameter not in self._arguments) or (self._arguments[parameter] is None):
                    skip = True
                    break
                kwargs[parameter] = self._arguments[parameter]
            if skip:
                continue

            # Call the validation function, catching AssertionErrors and turning them into errors in
            # the error dictionary.
            try:
                validate_fn(**kwargs)
            except AssertionError as ex:
                self._errors[parameter_set] = ex.args[0]

        # TODO(cmaloney): Return per-target results with the path to calculate each argument and the
        # full set of arguments touched included.
        return self._arguments

    # Force calculation of all arguments by accessing the arguments in this
    # scope and recursively all sub-scopes.
    def calculate(self, targets):
        for target in targets:
            self._calculate_target(target)

        if len(self._errors) or len(self._unset):
            raise ValidationError(self._errors, self._unset)

        return self._arguments

def resolve_configuration(sources: list, targets: list, user_arguments: dict):

    # Make sure all user provided arguments are strings.
    # TODO(cmaloney): Loosen this restriction  / allow arbitrary types as long
    # as they all have a gen specific string form.
    validate_arguments_strings(user_arguments)

    # Merge the sources into a big dictionary of setters + validators, ensuring
    # that all setters are either strings or functions.
    validate = list()
    setters = dict()

    # Merge all the config targets into one big group of setters for providing
    # to the calculator
    # TODO(cmaloney): The setter management / set code is very similar to that in ConfigTarget, they
    # could probably be joined.
    for source in sources:
        for name, setter_list in source.setters.items():
            setters.setdefault(name, list())
            setters[name] += setter_list
        validate += source.validate

    # Validate that targets is a list of Targets
    for target in targets:
        assert isinstance(target, Target), \
            "target should be a Target found a {} with value: {}".format(type(target), target)

    # TODO(cmaloney): Re-enable this after sorting out how to have "optional" config targets which
    # add in extra "acceptable" parameters (SSH Config, AWS Advanced Template config, etc)
    # validate_all_arguments_match_parameters(mandatory_parameters, setters, user_arguments)

    # Add in all user arguments as setters.
    # Happens last so that they are never overwritten with replace_existing=True
    user_config = Source(is_user=True)
    user_config.add_value_dict(user_arguments)

    # Merge all the seters and validate function into one uber list
    setters = copy.deepcopy(user_config.setters)
    validate = copy.deepcopy(user_config.validate)
    for source in sources:
        for name, setter_list in source.setters.items():
            # TODO(cmaloney): Make a setter manager already...
            setters.setdefault(name, list())
            setters[name] += setter_list
        validate += source.validate

    # Use setters to caluclate every required parameter
    arguments = DFSArgumentCalculator(setters, validate).calculate(targets)

    # Validate all new / calculated arguments are strings.
    validate_arguments_strings(arguments)

    log.info("Final arguments:" + json_prettyprint(arguments))

    # TODO(cmaloney) Give each config target the values for all it's parameters that were hit as
    # well as any parameters that led to those parameters.
    return arguments

def validate_configuration(sources: list, targets: list, user_arguments: dict):
    try:
        resolve_configuration(sources, targets, user_arguments)
        return {'status': 'ok'}
    except ValidationError as ex:
        messages = {}

        # Defer multi-key validation errors and noramlize them to be single-key
        # ones. The multi-key is always less important than the single-key
        # messages which is why single-key messages we never overwrite.
        # TODO(cmaloney): Teach the whole stack to be able to handle multi-key
        # validation errors.
        to_do = dict()
        for key, msg in ex.errors.items():
            if isinstance(key, frozenset):
                to_do[key] = msg
                continue

            assert isinstance(key, str)
            messages[key] = {'message': msg}

        for keys, msg in to_do.items():
            for name in keys:
                # Skip ones we already have one message for.
                if name in messages:
                    continue

                messages[name] = {'message': msg}

        return {
            'status': 'errors',
            'errors': messages,
            'unset': ex.unset
        }
<|endoftext|>


<commit_before># Copyright (c) 2016, Nordic Semiconductor
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice, this
#   list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
#
# * Neither the name of Nordic Semiconductor ASA nor the names of its
#   contributors may be used to endorse or promote products derived from
#   this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import os
import nrfjprog_version

from pynrfjprog import API, Hex

class NRF5x:
    """
    Common class that manages the api instance, some shared arguments amongst commands, and logging.

    """
    def __init__(self, args):
        """
        Constructor that initializes the class's properties.

        :param Object args: The arguments the command was called with.
        :return: None
        """
        try: # Quiet may not be an argument for some commands so args.quiet would be undefined causing an error.
            self.clockspeed = args.clockspeed
        except Exception:
            self.clockspeed = None

        try:
            self.quiet = args.quiet
        except Exception:
            self.quiet = None

        try:
            self.snr = args.snr
        except Exception:
            self.snr = None

        self.api = self._setup()

    def _setup(self):
        """
        Discovers the family (either NRF51 or NRF52) of the target device (through trial and error) and connects to it.

        :return API api: Instance of an API object that is initialized and connected to an nRF5x device.
        """
        device_family = API.DeviceFamily.NRF51
        api = API.API(device_family)
        api.open()
        self._connect_to_emu(api)

        try:
            device_version = api.read_device_version()
        except API.APIError as e:
            if e.err_code == API.NrfjprogdllErr.WRONG_FAMILY_FOR_DEVICE:
                device_family = API.DeviceFamily.NRF52
                api.close()
                api = API.API(device_family)
                api.open()
                self._connect_to_emu(api)
            else:
                raise e

        api.connect_to_device()
        return api

    def _connect_to_emu(self, api):
        if self.snr and self.clockspeed:
            api.connect_to_emu_with_snr(self.snr, self.clockspeed)
        elif self.snr:
            api.connect_to_emu_with_snr(self.snr)
        elif self.clockspeed:
            api.connect_to_emu_without_snr(self.clockspeed)
        else:
            api.connect_to_emu_without_snr()

    def log(self, msg):
        if self.quiet:
            pass
        else:
            print(msg)

    def cleanup(self):
        self.api.disconnect_from_emu()
        self.api.close()

"""
The functions that are called from argparse based on the command-line input.

All functions follow the same structure: initialize NRF5x, log (exactly what the help menu prints for the command, but in different tense), perform functionality, cleanup.
"""

def erase(args):
    nrf = NRF5x(args)
    nrf.log('Erasing the device.') # This should go to stderr.

    if args.erasepage:
        nrf.api.erase_page(args.erasepage)
    elif args.eraseuicr:
        nrf.api.erase_uicr()
    else:
        nrf.api.erase_all()

    nrf.cleanup()

def halt(args):
    nrf = NRF5x(args)
    nrf.log("Halting the device's CPU.")

    nrf.api.halt()

    nrf.cleanup()

def ids(args): # BUG: We should not connect to anything. _setup is different for this function.
    nrf = NRF5x(args)
    nrf.log('Displaying the serial numbers of all debuggers connected to the PC.')

    print(nrf.api.enum_emu_snr()) # This should go to stdout.

    nrf.cleanup()

def memrd(args):
    nrf = NRF5x(args)
    nrf.log("Reading the device's memory.")

    if args.length:
        length = args.length
    else:
        length = 4

    assert(length > 0), 'Length (number of bytes to read) must be nonnegative.'

    read_data = nrf.api.read(args.addr, length)
    print(read_data)

    nrf.cleanup()

def memwr(args):
    nrf = NRF5x(args)
    nrf.log("Writing the device's memory.")

    if args.flash:
        nrf.api.write_u32(args.addr, args.val, True)
    else:
        assert(args.addr >= 0x80000), "The --flash option needs to be specified when writing FLASH." # Won't catch UICR writes but better than nothing for now...
        nrf.api.write_u32(args.addr, args.val, False)

    nrf.cleanup()

def program(args):
    nrf = NRF5x(args)
    nrf.log('Programming the device.')

    if args.eraseall:
        nrf.api.erase_all()
    elif args.sectorserase:
        assert (False), "Not implemented in nrf5x.py yet."
    elif args.sectorsanduicrerase:
        assert (False), "Not implemented in nrf5x.py yet."

    if os.path.exists(args.file.name): # If user specified an absolute path.
        hex_file_path = args.file.name
    else: # Otherwise append the path the user specified to nrfjprog/.
        tmp_path = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
        tmp_path = os.path.abspath(os.path.join(tmp_path, os.pardir))
        hex_file_path = os.path.join(tmp_path, args.file.name)

    # Parse hex, program to device
    nrf.log('# Parsing hex file into segments  ')
    test_program = Hex.Hex(hex_file_path) # Parse .hex file into segments
    nrf.log('# Writing %s to device  ' % hex_file_path)
    for segment in test_program:
        read_data = nrf.api.read(segment.address, len(segment.data))
        assert (read_data == ([0xFF] * len(read_data))), 'FLASH being written to must be erased.'
        nrf.api.write(segment.address, segment.data, True)
        if args.verify:
            read_data = nrf.api.read(segment.address, len(segment.data))
            assert (read_data == segment.data), 'Verify failed. Data readback from memory does not match data written.'

    if args.verify:
        nrf.log('Programming verified.')

    if args.debugreset:
        nrf.api.debug_reset()
    elif args.pinreset:
        nrf.api.pin_reset()
    elif args.systemreset: # BUG: doesn't seem to be working.
        nrf.api.sys_reset()

    nrf.cleanup()

def readregs(args):
    nrf = NRF5x(args)
    nrf.log('Reading the CPU registers.')

    for reg in API.CpuRegister:
        print(nrf.api.read_cpu_register(reg))

    nrf.cleanup()

def readtofile(args):
    nrf = NRF5x(args)
    nrf.log("Reading and storing the device's memory.")

    #TODO
    assert (False), "Not implemented in nrf5x.py yet."

    nrf.cleanup()

def recover(args):
    nrf = NRF5x(args)
    nrf.log("Erasing all user FLASH and RAM and disabling any readback protection mechanisms that are enabled.")

    nrf.api.recover()

    nrf.cleanup()

def reset(args):
    nrf = NRF5x(args)
    nrf.log('Resetting the device.')

    _reset(nrf, args)

    nrf.api.go()
    nrf.cleanup()

def run(args): # TODO: run should accept pc and sp as input.
    nrf = NRF5x(args)
    nrf.log("Running the device's CPU.")

    nrf.api.go()

    nrf.cleanup()

def verify(args):
    nrf = NRF5x(args)
    nrf.log("Verifying that the device's memory contains the correct data.")

    hex_file_path = args.file.name # TODO: fix file name

    test_program = Hex.Hex(hex_file_path) # Parse .hex file into segments
    for segment in test_program:
        read_data = nrf.api.read(segment.address, len(segment.data))
        assert (read_data == segment.data), 'Verify failed. Data readback from memory does not match data written.'

    nrf.log('Programming verified.')

    nrf.cleanup()

def version(args):
    nrf = NRF5x(args)
    nrf.log('Displaying the nrfjprog and JLinkARM DLL versions.')

    jlink_arm_dll_version = nrf.api.dll_version()
    print(jlink_arm_dll_version)
    print(nrfjprog_version.NRFJPROG_VERSION)

    nrf.cleanup()

def _reset(nrf, args):
    if args.debugreset:
        nrf.api.debug_reset()
    elif args.pinreset:
        nrf.api.pin_reset()
    else:
        nrf.api.sys_reset()
<commit_msg>Improve the way we find the --file.<commit_after># Copyright (c) 2016, Nordic Semiconductor
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice, this
#   list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
#
# * Neither the name of Nordic Semiconductor ASA nor the names of its
#   contributors may be used to endorse or promote products derived from
#   this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import os
import nrfjprog_version

from pynrfjprog import API, Hex

class NRF5x:
    """
    Common class that manages the api instance, some shared arguments amongst commands, and logging.

    """
    def __init__(self, args):
        """
        Constructor that initializes the class's properties.

        :param Object args: The arguments the command was called with.
        :return: None
        """
        try: # Quiet may not be an argument for some commands so args.quiet would be undefined causing an error.
            self.clockspeed = args.clockspeed
        except Exception:
            self.clockspeed = None

        try:
            self.quiet = args.quiet
        except Exception:
            self.quiet = None

        try:
            self.snr = args.snr
        except Exception:
            self.snr = None

        self.api = self._setup()

    def _setup(self):
        """
        Discovers the family (either NRF51 or NRF52) of the target device (through trial and error) and connects to it.

        :return API api: Instance of an API object that is initialized and connected to an nRF5x device.
        """
        device_family = API.DeviceFamily.NRF51
        api = API.API(device_family)
        api.open()
        self._connect_to_emu(api)

        try:
            device_version = api.read_device_version()
        except API.APIError as e:
            if e.err_code == API.NrfjprogdllErr.WRONG_FAMILY_FOR_DEVICE:
                device_family = API.DeviceFamily.NRF52
                api.close()
                api = API.API(device_family)
                api.open()
                self._connect_to_emu(api)
            else:
                raise e

        api.connect_to_device()
        return api

    def _connect_to_emu(self, api):
        if self.snr and self.clockspeed:
            api.connect_to_emu_with_snr(self.snr, self.clockspeed)
        elif self.snr:
            api.connect_to_emu_with_snr(self.snr)
        elif self.clockspeed:
            api.connect_to_emu_without_snr(self.clockspeed)
        else:
            api.connect_to_emu_without_snr()

    def log(self, msg):
        if self.quiet:
            pass
        else:
            print(msg)

    def cleanup(self):
        self.api.disconnect_from_emu()
        self.api.close()

"""
The functions that are called from argparse based on the command-line input.

All functions follow the same structure: initialize NRF5x, log (exactly what the help menu prints for the command, but in different tense), perform functionality, cleanup.
"""

def erase(args):
    nrf = NRF5x(args)
    nrf.log('Erasing the device.') # This should go to stderr.

    if args.erasepage:
        nrf.api.erase_page(args.erasepage)
    elif args.eraseuicr:
        nrf.api.erase_uicr()
    else:
        nrf.api.erase_all()

    nrf.cleanup()

def halt(args):
    nrf = NRF5x(args)
    nrf.log("Halting the device's CPU.")

    nrf.api.halt()

    nrf.cleanup()

def ids(args): # BUG: We should not connect to anything. _setup is different for this function.
    nrf = NRF5x(args)
    nrf.log('Displaying the serial numbers of all debuggers connected to the PC.')

    print(nrf.api.enum_emu_snr()) # This should go to stdout.

    nrf.cleanup()

def memrd(args):
    nrf = NRF5x(args)
    nrf.log("Reading the device's memory.")

    if args.length:
        length = args.length
    else:
        length = 4

    assert(length > 0), 'Length (number of bytes to read) must be nonnegative.'

    read_data = nrf.api.read(args.addr, length)
    print(read_data)

    nrf.cleanup()

def memwr(args):
    nrf = NRF5x(args)
    nrf.log("Writing the device's memory.")

    if args.flash:
        nrf.api.write_u32(args.addr, args.val, True)
    else:
        assert(args.addr >= 0x80000), "The --flash option needs to be specified when writing FLASH." # Won't catch UICR writes but better than nothing for now...
        nrf.api.write_u32(args.addr, args.val, False)

    nrf.cleanup()

def program(args):
    nrf = NRF5x(args)
    nrf.log('Programming the device.')

    if args.eraseall:
        nrf.api.erase_all()
    elif args.sectorserase:
        assert (False), "Not implemented in nrf5x.py yet."
    elif args.sectorsanduicrerase:
        assert (False), "Not implemented in nrf5x.py yet."

    hex_file_path = _get_file_path(args.file.name)

    # Parse hex, program to device
    nrf.log('# Parsing hex file into segments  ')
    test_program = Hex.Hex(hex_file_path) # Parse .hex file into segments
    nrf.log('# Writing %s to device  ' % hex_file_path)
    for segment in test_program:
        read_data = nrf.api.read(segment.address, len(segment.data))
        assert (read_data == ([0xFF] * len(read_data))), 'FLASH being written to must be erased.'
        nrf.api.write(segment.address, segment.data, True)
        if args.verify:
            read_data = nrf.api.read(segment.address, len(segment.data))
            assert (read_data == segment.data), 'Verify failed. Data readback from memory does not match data written.'

    if args.verify:
        nrf.log('Programming verified.')

    if args.debugreset:
        nrf.api.debug_reset()
    elif args.pinreset:
        nrf.api.pin_reset()
    elif args.systemreset: # BUG: doesn't seem to be working.
        nrf.api.sys_reset()

    nrf.cleanup()

def readregs(args):
    nrf = NRF5x(args)
    nrf.log('Reading the CPU registers.')

    for reg in API.CpuRegister:
        print(nrf.api.read_cpu_register(reg))

    nrf.cleanup()

def readtofile(args):
    nrf = NRF5x(args)
    nrf.log("Reading and storing the device's memory.")

    #TODO
    assert (False), "Not implemented in nrf5x.py yet."

    nrf.cleanup()

def recover(args):
    nrf = NRF5x(args)
    nrf.log("Erasing all user FLASH and RAM and disabling any readback protection mechanisms that are enabled.")

    nrf.api.recover()

    nrf.cleanup()

def reset(args):
    nrf = NRF5x(args)
    nrf.log('Resetting the device.')

    _reset(nrf, args)

    nrf.api.go()
    nrf.cleanup()

def run(args): # TODO: run should accept pc and sp as input.
    nrf = NRF5x(args)
    nrf.log("Running the device's CPU.")

    nrf.api.go()

    nrf.cleanup()

def verify(args):
    nrf = NRF5x(args)
    nrf.log("Verifying that the device's memory contains the correct data.")

    hex_file_path = _get_file_path(args.file.name)

    test_program = Hex.Hex(hex_file_path) # Parse .hex file into segments
    for segment in test_program:
        read_data = nrf.api.read(segment.address, len(segment.data))
        assert (read_data == segment.data), 'Verify failed. Data readback from memory does not match data written.'

    nrf.log('Programming verified.')

    nrf.cleanup()

def version(args):
    nrf = NRF5x(args)
    nrf.log('Displaying the nrfjprog and JLinkARM DLL versions.')

    jlink_arm_dll_version = nrf.api.dll_version()
    print(jlink_arm_dll_version)
    print(nrfjprog_version.NRFJPROG_VERSION)

    nrf.cleanup()

"""
Helper functions.

"""

def _get_file_path(user_specified_path):
    if os.path.exists(user_specified_path): # If user specified an absolute path.
        return user_specified_path
    else: # Otherwise append the path user specified to nrfjprog/.
        tmp_path = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
        tmp_path = os.path.abspath(os.path.join(tmp_path, os.pardir))
        return os.path.join(tmp_path, user_specified_path)

def _reset(nrf, args):
    if args.debugreset:
        nrf.api.debug_reset()
    elif args.pinreset:
        nrf.api.pin_reset()
    else:
        nrf.api.sys_reset()
<|endoftext|>


<commit_before># TestSwiftHashedContainerEnum.py

"""
Test combinations of hashed swift containers with enums as keys/values
"""

import lldb
from lldbsuite.test.lldbtest import *
from lldbsuite.test.decorators import *
import lldbsuite.test.lldbutil as lldbutil
import os
import unittest2

class TestSwiftHashedContainerEnum(TestBase):

    mydir = TestBase.compute_mydir(__file__)

    @swiftTest
    @expectedFailureAll(debug_info="dwarf", bugnumber="rdar://problem/44689915")
    @add_test_categories(["swiftpr"])
    def test_any_object_type(self):
        """Test combinations of hashed swift containers with enums"""
        self.build()
        self.do_test()

    def setUp(self):
        TestBase.setUp(self)
        self.main_source = "main.swift"
        self.main_source_spec = lldb.SBFileSpec(self.main_source)

    def do_test(self):
        """Test combinations of hashed swift containers with enums"""
        exe_name = "a.out"
        exe = self.getBuildArtifact(exe_name)

        # Create the target
        target = self.dbg.CreateTarget(exe)
        self.assertTrue(target, VALID_TARGET)

        # Set the breakpoints
        breakpoint = target.BreakpointCreateBySourceRegex(
            '// break here', self.main_source_spec)
        self.assertTrue(breakpoint.GetNumLocations() > 0, VALID_BREAKPOINT)

        # Launch the process, and do not stop at the entry point.
        process = target.LaunchSimple(None, None, os.getcwd())

        self.assertTrue(process, PROCESS_IS_VALID)

        # Frame #0 should be at our breakpoint.
        threads = lldbutil.get_threads_stopped_at_breakpoint(
            process, breakpoint)

        self.assertTrue(len(threads) == 1)
        self.thread = threads[0]
        self.frame = self.thread.frames[0]
        self.assertTrue(self.frame, "Frame 0 is valid.")

        self.expect(
            'frame variable -d run -- testA',
            substrs=[
                'key = c',
                'value = 1',
                'key = b',
                'value = 2'])
        self.expect(
            'expr -d run -- testA',
            substrs=[
                'key = c',
                'value = 1',
                'key = b',
                'value = 2'])

        self.expect(
            'frame variable -d run -- testB',
            substrs=[
                'key = "a", value = 1',
                'key = "b", value = 2'])
        self.expect(
            'expr -d run -- testB',
            substrs=[
                'key = "a", value = 1',
                'key = "b", value = 2'])

        self.expect(
            'frame variable -d run -- testC',
            substrs=['key = b', 'value = 2'])
        self.expect(
            'expr -d run -- testC',
            substrs=['key = b', 'value = 2'])

        self.expect(
            'frame variable -d run -- testD',
            substrs=['[0] = c'])
        self.expect(
            'expr -d run -- testD',
            substrs=['[0] = c'])
<commit_msg>Revert "XFAIL test for DWARF (rdar://problem/44689915)"<commit_after># TestSwiftHashedContainerEnum.py

"""
Test combinations of hashed swift containers with enums as keys/values
"""

import lldb
from lldbsuite.test.lldbtest import *
from lldbsuite.test.decorators import *
import lldbsuite.test.lldbutil as lldbutil
import os
import unittest2

class TestSwiftHashedContainerEnum(TestBase):

    mydir = TestBase.compute_mydir(__file__)

    @swiftTest
    @add_test_categories(["swiftpr"])
    def test_any_object_type(self):
        """Test combinations of hashed swift containers with enums"""
        self.build()
        self.do_test()

    def setUp(self):
        TestBase.setUp(self)
        self.main_source = "main.swift"
        self.main_source_spec = lldb.SBFileSpec(self.main_source)

    def do_test(self):
        """Test combinations of hashed swift containers with enums"""
        exe_name = "a.out"
        exe = self.getBuildArtifact(exe_name)

        # Create the target
        target = self.dbg.CreateTarget(exe)
        self.assertTrue(target, VALID_TARGET)

        # Set the breakpoints
        breakpoint = target.BreakpointCreateBySourceRegex(
            '// break here', self.main_source_spec)
        self.assertTrue(breakpoint.GetNumLocations() > 0, VALID_BREAKPOINT)

        # Launch the process, and do not stop at the entry point.
        process = target.LaunchSimple(None, None, os.getcwd())

        self.assertTrue(process, PROCESS_IS_VALID)

        # Frame #0 should be at our breakpoint.
        threads = lldbutil.get_threads_stopped_at_breakpoint(
            process, breakpoint)

        self.assertTrue(len(threads) == 1)
        self.thread = threads[0]
        self.frame = self.thread.frames[0]
        self.assertTrue(self.frame, "Frame 0 is valid.")

        self.expect(
            'frame variable -d run -- testA',
            substrs=[
                'key = c',
                'value = 1',
                'key = b',
                'value = 2'])
        self.expect(
            'expr -d run -- testA',
            substrs=[
                'key = c',
                'value = 1',
                'key = b',
                'value = 2'])

        self.expect(
            'frame variable -d run -- testB',
            substrs=[
                'key = "a", value = 1',
                'key = "b", value = 2'])
        self.expect(
            'expr -d run -- testB',
            substrs=[
                'key = "a", value = 1',
                'key = "b", value = 2'])

        self.expect(
            'frame variable -d run -- testC',
            substrs=['key = b', 'value = 2'])
        self.expect(
            'expr -d run -- testC',
            substrs=['key = b', 'value = 2'])

        self.expect(
            'frame variable -d run -- testD',
            substrs=['[0] = c'])
        self.expect(
            'expr -d run -- testD',
            substrs=['[0] = c'])
<|endoftext|>


<commit_before>"""Command to import PMG's constitiency office data

PMG are undertaking an exercise to visit all the constituency
offices and collect accurate locations and photographs.
The data from this is available at

https://app.m4jam.com/app/campaigns/2298/export/

This script parses the resulting CSV file and updates our data
on constituency offices to have the correct location.
"""

import csv
import re

from django.core.management.base import BaseCommand, CommandError
from django.contrib.gis.geos import Point

from pombola.core.models import (
    Organisation,
    )

# Matches the format of the locations in the CSV we get from PMG.
location_re = re.compile(
    r'SRID=4326;POINT\((-?[0-9]+\.[0-9]+) (-?[0-9]+\.[0-9]+)\)')

class Command(BaseCommand):
    args = '<path to csv file>'
    help = 'Updates constituency offices based on the supplied CSV file.'

    def handle(self, *args, **options):
        if len(args) != 1:
            raise CommandError(
                "You must provide the path to the CSV file as an argument."
                )

        constituency_offices = Organisation.objects.filter(
            kind__slug='constituency-office')

        with open(args[0]) as csvfile:
            reader = csv.DictReader(csvfile)

            for row in reader:
                poi_ref = row['POI_REFERENCE']
                party, name = (
                    re.match(
                        '(.*) Constituency Office (.*)', poi_ref
                        ).groups()
                    )
                name = re.sub('\s+', ' ', name)

                qs = constituency_offices.filter(
                    name__regex=r'^{}.+: {}$'.format(party, name))

                try:
                    org = qs.get()
                except Organisation.MultipleObjectsReturned:
                    print "Skipping {} as multiple orgs returned: {}".format(
                        poi_ref,
                        repr(qs),
                        )
                    continue
                except Organisation.DoesNotExist:
                    # Fall back to searching for the name and the party in the
                    # constituency office name
                    qs = (
                        Organisation.objects
                        .filter(kind__slug='constituency-office')
                        .filter(name__contains=name)
                        .filter(name__contains=party)
                        )

                    org = qs.get()

                place = (
                    org.place_set
                    .filter(name__contains='Approximate position of')
                    .filter(kind__slug='constituency-office')
                    .get()  # There should be only one.
                    )

                lon, lat = location_re.match(row['Constituency_Pin']).groups()

                place.location = Point(float(lon), float(lat))
                place.save()
<commit_msg>Store images for constituency offices.<commit_after>"""Command to import PMG's constitiency office data

PMG are undertaking an exercise to visit all the constituency
offices and collect accurate locations and photographs.
The data from this is available at

https://app.m4jam.com/app/campaigns/2298/export/

This script parses the resulting CSV file and updates our data
on constituency offices to have the correct location.
"""

import csv
import os.path
import re
from StringIO import StringIO

import requests

from django.conf import settings

from django.core.files.base import ContentFile
from django.core.files.storage import FileSystemStorage
from django.core.management.base import BaseCommand, CommandError
from django.contrib.gis.geos import Point

from pombola.core.models import (
    Organisation,
    )
from pombola.core.utils import mkdir_p

from pombola.images.models import Image

# Matches the format of the locations in the CSV we get from PMG.
location_re = re.compile(
    r'SRID=4326;POINT\((-?[0-9]+\.[0-9]+) (-?[0-9]+\.[0-9]+)\)')

class Command(BaseCommand):
    args = '<path to csv file>'
    help = 'Updates constituency offices based on the supplied CSV file.'

    def handle(self, *args, **options):
        if len(args) != 1:
            raise CommandError(
                "You must provide the path to the CSV file as an argument."
                )

        constituency_offices = Organisation.objects.filter(
            kind__slug='constituency-office')

        storage = FileSystemStorage()

        storage_path = os.path.join('images', 'constituency-offices')
        images_directory = os.path.join(
            settings.MEDIA_ROOT,
            storage_path
            )
        mkdir_p(images_directory)

        with open(args[0]) as csvfile:
            reader = csv.DictReader(csvfile)

            for row in reader:
                poi_ref = row['POI_REFERENCE']
                party, name = (
                    re.match(
                        '(.*) Constituency Office (.*)', poi_ref
                        ).groups()
                    )
                name = re.sub('\s+', ' ', name)

                qs = constituency_offices.filter(
                    name__regex=r'^{}.+: {}$'.format(party, name))

                try:
                    org = qs.get()
                except Organisation.MultipleObjectsReturned:
                    print "Skipping {} as multiple orgs returned: {}".format(
                        poi_ref,
                        repr(qs),
                        )
                    continue
                except Organisation.DoesNotExist:
                    # Fall back to searching for the name and the party in the
                    # constituency office name
                    qs = (
                        Organisation.objects
                        .filter(kind__slug='constituency-office')
                        .filter(name__contains=name)
                        .filter(name__contains=party)
                        )

                    org = qs.get()

                place = (
                    org.place_set
                    .filter(name__contains='Approximate position of')
                    .filter(kind__slug='constituency-office')
                    .get()  # There should be only one.
                    )

                lon, lat = location_re.match(row['Constituency_Pin']).groups()

                place.location = Point(float(lon), float(lat))
                place.save()

                # Now get the photo
                photo_url = row.get('Constituency_Photo')

                if photo_url:
                    response = requests.get(photo_url)
                    image_filename = '{}'.format(org.slug)

                    desired_storage_path = os.path.join(storage_path, image_filename)
                    storage_filename = storage.save(desired_storage_path, StringIO(response.content))

                    image = Image(
                        content_object=org,
                        source=photo_url,
                        is_primary=True,
                        image=storage_filename,
                        )
                    image.save()
<|endoftext|>


<commit_before>"""Leetcode 700. Search in a Binary Search Tree
Easy

URL: https://leetcode.com/problems/search-in-a-binary-search-tree/

Given the root node of a binary search tree (BST) and a value.
You need to find the node in the BST that the node's value equals the given value.
Return the subtree rooted with that node.
If such node doesn't exist, you should return NULL.

For example,
Given the tree:
        4
       / \
      2   7
     / \
    1   3
And the value to search: 2
You should return this subtree:
      2
     / \
    1   3
In the example above, if we want to search the value 5, since there is no node
with value 5, we should return NULL.

Note that an empty tree is represented by NULL, therefore you would see the
expected output (serialized tree format) as [], not null.
"""

# Definition for a binary tree node.
class TreeNode(object):
    def __init__(self, val):
        self.val = val
        self.left = None
        self.right = None

class Solution(object):
    def searchBST(self, root, val):
        """
        :type root: TreeNode
        :type val: int
        :rtype: TreeNode
        """
        pass

def main():
    pass

if __name__ == '__main__':
    main()
<commit_msg>Complete recur sol<commit_after>"""Leetcode 700. Search in a Binary Search Tree
Easy

URL: https://leetcode.com/problems/search-in-a-binary-search-tree/

Given the root node of a binary search tree (BST) and a value.
You need to find the node in the BST that the node's value equals the given value.
Return the subtree rooted with that node.
If such node doesn't exist, you should return NULL.

For example,
Given the tree:
        4
       / \
      2   7
     / \
    1   3
And the value to search: 2
You should return this subtree:
      2
     / \
    1   3
In the example above, if we want to search the value 5, since there is no node
with value 5, we should return NULL.

Note that an empty tree is represented by NULL, therefore you would see the
expected output (serialized tree format) as [], not null.
"""

# Definition for a binary tree node.
class TreeNode(object):
    def __init__(self, val):
        self.val = val
        self.left = None
        self.right = None

class SolutionRecur(object):
    def searchBST(self, root, val):
        """
        :type root: TreeNode
        :type val: int
        :rtype: TreeNode

        Time complexity: O(logn).
        Space complexity: O(logn), since stack calls.
        """
        if not root:
            return None

        if root.val == val:
            return root

        if val < root.val:
            return self.searchBST(root.left, val)
        else:
            return self.searchBST(root.right, val)

def main():
    # Given the tree:
    #         4
    #        / \
    #       2   7
    #      / \
    #     1   3
    root = TreeNode(4)
    root.left = TreeNode(2)
    root.right = TreeNode(7)
    root.left.left = TreeNode(1)
    root.left.right = TreeNode(3)

    # Output: 2
    val = 2
    print SolutionRecur().searchBST(root, val).val

    # Output: None
    val = 5
    print SolutionRecur().searchBST(root, val)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import matplotlib.pyplot as plt
from matplotlib.patches import Rectangle, Ellipse, Circle
# register Axes3D class with matplotlib by importing Axes3D
from mpl_toolkits.mplot3d import art3d
import numpy as np

def set_windtunnel_bounds(ax):
    ax.set_xlim(0, 1)
    ax.set_ylim(-0.5, 0.5)
    ax.set_zlim(-0.5, 0.5)
    ax.invert_xaxis()  # fix for y convention

    ax.set_ylabel("Crosswind/$y$ (meters)", fontsize=14)  # remember! x,y switched in plot() above!
    ax.set_xlabel("Upwind/$x$ (meters)", fontsize=14)
    ax.set_zlabel("Elevation/$z$ (meters)", fontsize=14)

    ax.grid(False)  # no grid
    ax.patch.set_visible(False)  # white background
    for a in (ax.w_xaxis, ax.w_yaxis, ax.w_zaxis):
        a.pane.set_visible(False)

def draw_windtunnel_border(ax):
    x_min = 0
    x_max = 1
    z_min = 0
    z_max = 0.254
    y_min = -0.127
    y_max = 0.127
    draw_rectangular_prism(ax, x_min, x_max, y_min, y_max, z_min, z_max)
    plt.draw()

def draw_heaters(ax, windtunnel):
    draw_heater(ax, windtunnel.heater_l)
    draw_heater(ax, windtunnel.heater_r)

def draw_heater(ax, heater):
    x_center, y_center = heater.x_position, heater.y_position
    elevation, height = heater.zmin, heater.zmax - heater.zmin
    radius = heater.diam / 2
    resolution = 101
    color = heater.color

    plot_3D_cylinder(ax, radius, height, elevation=elevation, resolution=resolution, color=color, x_center=x_center,
                     y_center=y_center)

def draw_rectangular_prism(ax, x_min, x_max, y_min, y_max, z_min, z_max):
    alpha = 1
    back = Rectangle((y_min, z_min), y_max - y_min, z_max, alpha=alpha, fill=None, linestyle='dotted')
    ax.add_patch(back)
    art3d.pathpatch_2d_to_3d(back, z=x_min, zdir="x")

    front = Rectangle((y_min, z_min), y_max - y_min, z_max, alpha=alpha, fill=None, linestyle='dotted')
    ax.add_patch(front)
    art3d.pathpatch_2d_to_3d(front, z=x_max, zdir="x")

    floor = Rectangle((x_min, y_min), x_max, z_max, alpha=alpha, fill=None, linestyle='dotted')
    ax.add_patch(floor)
    art3d.pathpatch_2d_to_3d(floor, z=z_min, zdir="z")

    ceiling = Rectangle((x_min, y_min), x_max, z_max, alpha=alpha, fill=None, linestyle='dotted')
    ax.add_patch(ceiling)
    art3d.pathpatch_2d_to_3d(ceiling, z=z_max, zdir="z")

def draw_bool_plume(plume, ax=None):
    """Note: tried  to add ells to Pathcollection and then plot that, but kept having a ValueError
    Also tried using an EclipseCollection but the patchcollection_2d_to_3d function does not work on it

    # xy is actually the yz plane
    # val headers: [u'x_position', u'z_position', u'small_radius', u'y_position']
    """
    if plume.condition in 'controlControlCONTROL':
        print "No plume in ", plume.condition
        return

    ells = [Ellipse(xy=(val[3], val[1]),
                    width=2 * val[2],
                    height=2 * val[2] * 3,
                    angle=0,
                    # linestyle='dotted',
                facecolor='none',
                    alpha=0.05,
                    edgecolor='r')
            for i, val in plume.data.iterrows()]

    for i, val in plume.data['x_position'].iteritems():
        ell = ells[i]
        ax.add_patch(ell)
        art3d.patch_2d_to_3d(ell, z=val, zdir="x")

    art3d.patch_collection_2d_to_3d(ells)

    plt.show()

def plot_3D_cylinder(ax, radius, height, elevation=0, resolution=200, color='r', x_center=0, y_center=0):
    x = np.linspace(x_center - radius, x_center + radius, resolution)
    z = np.linspace(elevation, elevation + height, resolution)
    X, Z = np.meshgrid(x, z)

    Y = np.sqrt(radius ** 2 - (X - x_center) ** 2) + y_center  # Pythagorean theorem

    ax.plot_surface(X, Y, Z, linewidth=0, color=color)
    ax.plot_surface(X, (2 * y_center - Y), Z, linewidth=0, color=color)

    floor = Circle((x_center, y_center), radius, color=color)
    ax.add_patch(floor)
    art3d.pathpatch_2d_to_3d(floor, z=elevation, zdir="z")

    ceiling = Circle((x_center, y_center), radius, color=color)
    ax.add_patch(ceiling)
    art3d.pathpatch_2d_to_3d(ceiling, z=elevation + height, zdir="z")

def plot_windtunnel(windtunnel, title=''):
    fig = plt.figure()
    ax = fig.add_subplot(111, projection='3d')
    ax.set_aspect('equal')
    # ax.axis('off')
    set_windtunnel_bounds(ax)
    draw_windtunnel_border(ax)
    ax.set_title(title, fontsize=20)  # FIXME parent functions aren't passing titles yet
    draw_heaters(ax, windtunnel)

    return fig, ax

def draw_trajectory(ax, trajectory, **kwargs):
    """TODO: red inside windtunnel"""
    highlight = kwargs.get("highlight_inside_plume")
    ax.plot(trajectory.position_x, trajectory.position_y, trajectory.position_z)
    if highlight:
        inside_pts = trajectory.loc[trajectory['inPlume'] == True]
        ax.scatter(inside_pts.position_x, inside_pts.position_y, inside_pts.position_z, c='r')

def plot_plume_recordings_scatter(plume_data, ax):
    ax.scatter(plume_data.x, plume_data.y, plume_data.z, c=plume_data.avg_temp, cmap='inferno')

def plot_plume_gradient(plume, ax, thresh):  # TODO: plot inside windtunnel as in draw_bool_plume
    """
    Plot a quiverplot of the gradient
    Parameters
    ----------
    plume
        (gradient plume object)
    thresh
        (float)
        filters out plotting of gradient arrows smaller than this threshold
    """

    filtered = plume.data[plume.data.gradient_mag > thresh]

    ax.quiver(filtered.x, filtered.y, filtered.z, filtered.gradient_x, filtered.gradient_y, filtered.gradient_z, length=0.01)
    # ax.set_xlim3d(0, 1)
    # ax.set_ylim3d(-0.127, 0.127)
    # ax.set_zlim3d(0, 0.254)

    plt.title("Temperature gradient of interpolated time-averaged thermocouple recordings")
    plt.xlabel("Upwind/downwind")
    plt.ylabel("Crosswind")
    plt.clabel("Elevation")

    plt.show()

if __name__ is '__main__':
    ax = plot_windtunnel()<commit_msg>renamed mag to norm<commit_after>import matplotlib.pyplot as plt
from matplotlib.patches import Rectangle, Ellipse, Circle
# register Axes3D class with matplotlib by importing Axes3D
from mpl_toolkits.mplot3d import art3d
import numpy as np

def set_windtunnel_bounds(ax):
    ax.set_xlim(0, 1)
    ax.set_ylim(-0.5, 0.5)
    ax.set_zlim(-0.5, 0.5)
    ax.invert_xaxis()  # fix for y convention

    ax.set_ylabel("Crosswind/$y$ (meters)", fontsize=14)  # remember! x,y switched in plot() above!
    ax.set_xlabel("Upwind/$x$ (meters)", fontsize=14)
    ax.set_zlabel("Elevation/$z$ (meters)", fontsize=14)

    ax.grid(False)  # no grid
    ax.patch.set_visible(False)  # white background
    for a in (ax.w_xaxis, ax.w_yaxis, ax.w_zaxis):
        a.pane.set_visible(False)

def draw_windtunnel_border(ax):
    x_min = 0
    x_max = 1
    z_min = 0
    z_max = 0.254
    y_min = -0.127
    y_max = 0.127
    draw_rectangular_prism(ax, x_min, x_max, y_min, y_max, z_min, z_max)
    plt.draw()

def draw_heaters(ax, windtunnel):
    draw_heater(ax, windtunnel.heater_l)
    draw_heater(ax, windtunnel.heater_r)

def draw_heater(ax, heater):
    x_center, y_center = heater.x_position, heater.y_position
    elevation, height = heater.zmin, heater.zmax - heater.zmin
    radius = heater.diam / 2
    resolution = 101
    color = heater.color

    plot_3D_cylinder(ax, radius, height, elevation=elevation, resolution=resolution, color=color, x_center=x_center,
                     y_center=y_center)

def draw_rectangular_prism(ax, x_min, x_max, y_min, y_max, z_min, z_max):
    alpha = 1
    back = Rectangle((y_min, z_min), y_max - y_min, z_max, alpha=alpha, fill=None, linestyle='dotted')
    ax.add_patch(back)
    art3d.pathpatch_2d_to_3d(back, z=x_min, zdir="x")

    front = Rectangle((y_min, z_min), y_max - y_min, z_max, alpha=alpha, fill=None, linestyle='dotted')
    ax.add_patch(front)
    art3d.pathpatch_2d_to_3d(front, z=x_max, zdir="x")

    floor = Rectangle((x_min, y_min), x_max, z_max, alpha=alpha, fill=None, linestyle='dotted')
    ax.add_patch(floor)
    art3d.pathpatch_2d_to_3d(floor, z=z_min, zdir="z")

    ceiling = Rectangle((x_min, y_min), x_max, z_max, alpha=alpha, fill=None, linestyle='dotted')
    ax.add_patch(ceiling)
    art3d.pathpatch_2d_to_3d(ceiling, z=z_max, zdir="z")

def draw_bool_plume(plume, ax=None):
    """Note: tried  to add ells to Pathcollection and then plot that, but kept having a ValueError
    Also tried using an EclipseCollection but the patchcollection_2d_to_3d function does not work on it

    # xy is actually the yz plane
    # val headers: [u'x_position', u'z_position', u'small_radius', u'y_position']
    """
    if plume.condition in 'controlControlCONTROL':
        print "No plume in ", plume.condition
        return

    ells = [Ellipse(xy=(val[3], val[1]),
                    width=2 * val[2],
                    height=2 * val[2] * 3,
                    angle=0,
                    # linestyle='dotted',
                facecolor='none',
                    alpha=0.05,
                    edgecolor='r')
            for i, val in plume.data.iterrows()]

    for i, val in plume.data['x_position'].iteritems():
        ell = ells[i]
        ax.add_patch(ell)
        art3d.patch_2d_to_3d(ell, z=val, zdir="x")

    art3d.patch_collection_2d_to_3d(ells)

    plt.show()

def plot_3D_cylinder(ax, radius, height, elevation=0, resolution=200, color='r', x_center=0, y_center=0):
    x = np.linspace(x_center - radius, x_center + radius, resolution)
    z = np.linspace(elevation, elevation + height, resolution)
    X, Z = np.meshgrid(x, z)

    Y = np.sqrt(radius ** 2 - (X - x_center) ** 2) + y_center  # Pythagorean theorem

    ax.plot_surface(X, Y, Z, linewidth=0, color=color)
    ax.plot_surface(X, (2 * y_center - Y), Z, linewidth=0, color=color)

    floor = Circle((x_center, y_center), radius, color=color)
    ax.add_patch(floor)
    art3d.pathpatch_2d_to_3d(floor, z=elevation, zdir="z")

    ceiling = Circle((x_center, y_center), radius, color=color)
    ax.add_patch(ceiling)
    art3d.pathpatch_2d_to_3d(ceiling, z=elevation + height, zdir="z")

def plot_windtunnel(windtunnel, title=''):
    fig = plt.figure()
    ax = fig.add_subplot(111, projection='3d')
    ax.set_aspect('equal')
    # ax.axis('off')
    set_windtunnel_bounds(ax)
    draw_windtunnel_border(ax)
    ax.set_title(title, fontsize=20)  # FIXME parent functions aren't passing titles yet
    draw_heaters(ax, windtunnel)

    return fig, ax

def draw_trajectory(ax, trajectory, **kwargs):
    """TODO: red inside windtunnel"""
    highlight = kwargs.get("highlight_inside_plume")
    ax.plot(trajectory.position_x, trajectory.position_y, trajectory.position_z)
    if highlight:
        inside_pts = trajectory.loc[trajectory['inPlume'] == True]
        ax.scatter(inside_pts.position_x, inside_pts.position_y, inside_pts.position_z, c='r')

def plot_plume_recordings_scatter(plume_data, ax):
    ax.scatter(plume_data.x, plume_data.y, plume_data.z, c=plume_data.avg_temp, cmap='inferno')

def plot_plume_gradient(plume, ax, thresh):  # TODO: plot inside windtunnel as in draw_bool_plume
    """
    Plot a quiverplot of the gradient
    Parameters
    ----------
    plume
        (gradient plume object)
    thresh
        (float)
        filters out plotting of gradient arrows smaller than this threshold
    """

    filtered = plume.data[plume.data.gradient_norm > thresh]

    ax.quiver(filtered.x, filtered.y, filtered.z, filtered.gradient_x, filtered.gradient_y, filtered.gradient_z, length=0.01)
    # ax.set_xlim3d(0, 1)
    # ax.set_ylim3d(-0.127, 0.127)
    # ax.set_zlim3d(0, 0.254)

    plt.title("Temperature gradient of interpolated time-averaged thermocouple recordings")
    plt.xlabel("Upwind/downwind")
    plt.ylabel("Crosswind")
    plt.clabel("Elevation")

    plt.show()

if __name__ is '__main__':
    ax = plot_windtunnel()<|endoftext|>


<commit_before>import json

from flask import request, session, render_template, Blueprint

from lexos.helpers import constants as constants
from lexos.managers import utility, session_manager as session_manager
from lexos.interfaces.base_interface import detect_active_docs

# this is a flask blue print
# it helps us to manage groups of views
# see here for more detail:
# http://exploreflask.com/en/latest/blueprints.html
# http://flask.pocoo.org/docs/0.12/blueprints/
viz_view = Blueprint('viz', __name__)

# Tells Flask to load this function when someone is at '/viz'
@viz_view.route("/viz", methods=["GET", "POST"])
def viz():
    """Handles the functionality on the alternate bubbleViz page with performance
    improvements.

    :returns: a response object (often a render_template call) to flask and
     eventually to the browser.
    """
    # Detect the number of active documents.
    num_active_docs = detect_active_docs()
    file_manager = utility.load_file_manager()
    labels = file_manager.get_active_labels()
    from collections import OrderedDict
    from natsort import natsorted
    labels = OrderedDict(natsorted(labels.items(), key=lambda x: x[1]))
    if request.method == "GET":
        # "GET" request occurs when the page is first loaded.
        if 'cloudoption' not in session:
            session['cloudoption'] = constants.DEFAULT_CLOUD_OPTIONS
        if 'bubblevisoption' not in session:
            session['bubblevisoption'] = constants.DEFAULT_BUBBLEVIZ_OPTIONS
        return render_template(
            'viz.html',
            JSONObj="",
            labels=labels,
            itm="bubbleviz",
            numActiveDocs=num_active_docs)
    if request.method == "POST":
        # "POST" request occur when html form is submitted
        # (i.e. 'Get Dendrogram', 'Download...')
        # Legacy function
        # json_obj = utility.generateJSONForD3(file_manager, mergedSet=True)
        # Get the file manager, sorted labels, and tokenization options
        file_manager = utility.load_file_manager()
        if 'analyoption' not in session:
            session['analyoption'] = constants.DEFAULT_ANALYZE_OPTIONS
        token_type = session['analyoption']['tokenType']
        token_size = int(session['analyoption']['tokenSize'])
        # Limit docs to those selected or to active docs
        chosen_doc_ids = [int(x) for x in request.form.getlist('segmentlist')]
        active_docs = []
        if chosen_doc_ids:
            for ID in chosen_doc_ids:
                active_docs.append(ID)
        else:
            for lFile in file_manager.files.values():
                if lFile.active:
                    active_docs.append(lFile.id)
        # Get the contents of all selected/active docs
        all_contents = []
        for ID in active_docs:
            if file_manager.files[ID].active:
                content = file_manager.files[ID].load_contents()
                all_contents.append(content)
        # Generate a DTM
        dtm, vocab = utility.simple_vectorizer(
            all_contents, token_type, token_size)
        # Convert the DTM to a pandas dataframe with the terms as column
        # headers
        import pandas as pd
        df = pd.DataFrame(dtm, columns=vocab)
        # Get the Minimum Token Length and Maximum Term Settings
        minimum_length = int(
            request.form['minlength']) if 'minlength' in request.form else 0
        if 'maxwords' in request.form:
            # Make sure there is a number in the input form
            check_for_value = request.form['maxwords']
            if check_for_value == "":
                max_num_words = 100
            else:
                max_num_words = int(request.form['maxwords'])
        # Filter words that don't meet the minimum length from the dataframe
        for term in vocab:
            if len(term) < minimum_length:
                del df[term]
        # Extract a dictionary of term count sums
        sums_dict = df.sum(axis=0).to_dict()
        # Create a new dataframe of sums and sort it by counts, then terms
        """ Warning!!! This is not natsort. Multiple terms at the edge of
            the maximum number of words limit may be cut off in abitrary
            order. We need to implement natsort for dataframes.
        """
        f = pd.DataFrame(list(sums_dict.items()), columns=['term', 'count'])
        f.sort_values(by=['count', 'term'], axis=0,
                      ascending=[False, True], inplace=True)
        # Convert the dataframe head to a dict for use below
        f = f.head(n=max_num_words).to_dict()
        # Build the JSON object for d3.js
        termslist = []
        countslist = []
        children = []
        for item in f['term'].items():
            termslist.append(item[1])
        for item in f['count'].items():
            countslist.append(item[1])
        for k, v in enumerate(termslist):
            children.append({"name": v, "size": str(countslist[k])})
        json_obj = {"name": "tokens", "children": children}
        # Turn the JSON object into a JSON string for the front end
        json_str = json.dumps(json_obj)
        session_manager.cache_cloud_option()
        session_manager.cache_bubble_viz_option()
        return render_template(
            'viz.html',
            JSONObj=json_str,
            labels=labels,
            itm="bubbleviz",
            numActiveDocs=num_active_docs)
<commit_msg>fix documentation bubble_viz_interface<commit_after>import json

from flask import request, session, render_template, Blueprint

from lexos.helpers import constants as constants
from lexos.managers import utility, session_manager as session_manager
from lexos.interfaces.base_interface import detect_active_docs

# this is a flask blue print
# it helps us to manage groups of views
# see here for more detail:
# http://exploreflask.com/en/latest/blueprints.html
# http://flask.pocoo.org/docs/0.12/blueprints/
viz_view = Blueprint('viz', __name__)

# Tells Flask to load this function when someone is at '/viz'
@viz_view.route("/viz", methods=["GET", "POST"])
def viz():
    """Handles the functionality on the alternate bubbleViz page with
    performance improvements.

    :returns: a response object (often a render_template call) to flask and
    eventually to the browser.
    """
    # Detect the number of active documents.
    num_active_docs = detect_active_docs()
    file_manager = utility.load_file_manager()
    labels = file_manager.get_active_labels()
    from collections import OrderedDict
    from natsort import natsorted
    labels = OrderedDict(natsorted(labels.items(), key=lambda x: x[1]))
    if request.method == "GET":
        # "GET" request occurs when the page is first loaded.
        if 'cloudoption' not in session:
            session['cloudoption'] = constants.DEFAULT_CLOUD_OPTIONS
        if 'bubblevisoption' not in session:
            session['bubblevisoption'] = constants.DEFAULT_BUBBLEVIZ_OPTIONS
        return render_template(
            'viz.html',
            JSONObj="",
            labels=labels,
            itm="bubbleviz",
            numActiveDocs=num_active_docs)
    if request.method == "POST":
        # "POST" request occur when html form is submitted
        # (i.e. 'Get Dendrogram', 'Download...')
        # Legacy function
        # json_obj = utility.generateJSONForD3(file_manager, mergedSet=True)
        # Get the file manager, sorted labels, and tokenization options
        file_manager = utility.load_file_manager()
        if 'analyoption' not in session:
            session['analyoption'] = constants.DEFAULT_ANALYZE_OPTIONS
        token_type = session['analyoption']['tokenType']
        token_size = int(session['analyoption']['tokenSize'])
        # Limit docs to those selected or to active docs
        chosen_doc_ids = [int(x) for x in request.form.getlist('segmentlist')]
        active_docs = []
        if chosen_doc_ids:
            for ID in chosen_doc_ids:
                active_docs.append(ID)
        else:
            for lFile in file_manager.files.values():
                if lFile.active:
                    active_docs.append(lFile.id)
        # Get the contents of all selected/active docs
        all_contents = []
        for ID in active_docs:
            if file_manager.files[ID].active:
                content = file_manager.files[ID].load_contents()
                all_contents.append(content)
        # Generate a DTM
        dtm, vocab = utility.simple_vectorizer(
            all_contents, token_type, token_size)
        # Convert the DTM to a pandas dataframe with the terms as column
        # headers
        import pandas as pd
        df = pd.DataFrame(dtm, columns=vocab)
        # Get the Minimum Token Length and Maximum Term Settings
        minimum_length = int(
            request.form['minlength']) if 'minlength' in request.form else 0
        if 'maxwords' in request.form:
            # Make sure there is a number in the input form
            check_for_value = request.form['maxwords']
            if check_for_value == "":
                max_num_words = 100
            else:
                max_num_words = int(request.form['maxwords'])
        # Filter words that don't meet the minimum length from the dataframe
        for term in vocab:
            if len(term) < minimum_length:
                del df[term]
        # Extract a dictionary of term count sums
        sums_dict = df.sum(axis=0).to_dict()
        # Create a new dataframe of sums and sort it by counts, then terms
        # Warning!!! This is not natsort. Multiple terms at the edge of
        # the maximum number of words limit may be cut off in abitrary
        # order. We need to implement natsort for dataframes.
        f = pd.DataFrame(list(sums_dict.items()), columns=['term', 'count'])
        f.sort_values(by=['count', 'term'], axis=0,
                      ascending=[False, True], inplace=True)
        # Convert the dataframe head to a dict for use below
        f = f.head(n=max_num_words).to_dict()
        # Build the JSON object for d3.js
        termslist = []
        countslist = []
        children = []
        for item in f['term'].items():
            termslist.append(item[1])
        for item in f['count'].items():
            countslist.append(item[1])
        for k, v in enumerate(termslist):
            children.append({"name": v, "size": str(countslist[k])})
        json_obj = {"name": "tokens", "children": children}
        # Turn the JSON object into a JSON string for the front end
        json_str = json.dumps(json_obj)
        session_manager.cache_cloud_option()
        session_manager.cache_bubble_viz_option()
        return render_template(
            'viz.html',
            JSONObj=json_str,
            labels=labels,
            itm="bubbleviz",
            numActiveDocs=num_active_docs)
<|endoftext|>