<commit_before>########
# Copyright (c) 2015 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.

import networkx as nx

from dsl_parser import exceptions
from dsl_parser.framework import elements
from dsl_parser.framework.requirements import Requirement

class SchemaAPIValidator(object):

    def validate(self, element_cls):
        self._traverse_element_cls(element_cls)

    def _traverse_element_cls(self, element_cls):
        try:
            if not issubclass(element_cls, elements.Element):
                raise exceptions.DSLParsingSchemaAPIException(1)
        except TypeError:
            raise exceptions.DSLParsingSchemaAPIException(1)
        self._traverse_schema(element_cls.schema)

    def _traverse_schema(self, schema, list_nesting=0):
        if isinstance(schema, dict):
            for key, value in schema.items():
                if not isinstance(key, basestring):
                    raise exceptions.DSLParsingSchemaAPIException(1)
                self._traverse_element_cls(value)
        elif isinstance(schema, list):
            if list_nesting > 0:
                raise exceptions.DSLParsingSchemaAPIException(1)
            if len(schema) == 0:
                raise exceptions.DSLParsingSchemaAPIException(1)
            for value in schema:
                self._traverse_schema(value, list_nesting+1)
        elif isinstance(schema, elements.ElementType):
            if isinstance(schema, elements.Leaf):
                if not isinstance(schema.type, (type, list, tuple)):
                    raise exceptions.DSLParsingSchemaAPIException(1)
                if (isinstance(schema.type, (list, tuple)) and
                    (not schema.type or
                     not all([isinstance(i, type) for i in schema.type]))):
                    raise exceptions.DSLParsingSchemaAPIException(1)
            elif isinstance(schema, elements.Dict):
                self._traverse_element_cls(schema.type)
            elif isinstance(schema, elements.List):
                self._traverse_element_cls(schema.type)
            else:
                raise exceptions.DSLParsingSchemaAPIException(1)
        else:
            raise exceptions.DSLParsingSchemaAPIException(1)

_schema_validator = SchemaAPIValidator()

class Context(object):

    def __init__(self,
                 value,
                 element_cls,
                 element_name,
                 inputs):
        self.inputs = inputs or {}
        self.element_type_to_elements = {}
        self._root_element = None
        self._element_tree = nx.DiGraph()
        self._element_graph = nx.DiGraph()
        self._traverse_element_cls(element_cls=element_cls,
                                   name=element_name,
                                   value=value,
                                   parent_element=None)
        self._calculate_element_graph()

    @property
    def parsed_value(self):
        return self._root_element.value if self._root_element else None

    def child_elements_iter(self, element):
        return self._element_tree.successors_iter(element)

    def ancestors_iter(self, element):
        current_element = element
        while True:
            predecessors = self._element_tree.predecessors(current_element)
            if not predecessors:
                return
            if len(predecessors) > 1:
                raise exceptions.DSLParsingFormatException(
                    1, 'More than 1 parent found for {0}'
                       .format(element))
            current_element = predecessors[0]
            yield current_element

    def descendants(self, element):
        return nx.descendants(self._element_tree, element)

    def _add_element(self, element, parent=None):
        element_type = type(element)
        if element_type not in self.element_type_to_elements:
            self.element_type_to_elements[element_type] = []
        self.element_type_to_elements[element_type].append(element)

        self._element_tree.add_node(element)
        if parent:
            self._element_tree.add_edge(parent, element)
        else:
            self._root_element = element

    def _traverse_element_cls(self,
                              element_cls,
                              name,
                              value,
                              parent_element):
        element = element_cls(name=name,
                              initial_value=value,
                              context=self)
        self._add_element(element, parent=parent_element)
        self._traverse_schema(schema=element_cls.schema,
                              parent_element=element)

    def _traverse_schema(self, schema, parent_element):
        if isinstance(schema, dict):
            self._traverse_dict_schema(schema=schema,
                                       parent_element=parent_element)
        elif isinstance(schema, elements.ElementType):
            self._traverse_element_type_schema(
                schema=schema,
                parent_element=parent_element)
        elif isinstance(schema, list):
            self._traverse_list_schema(schema=schema,
                                       parent_element=parent_element)
        elif isinstance(schema, elements.UnknownSchema):
            pass
        else:
            raise ValueError('Illegal state should have been identified'
                             ' by schema API validation')

    def _traverse_dict_schema(self, schema,  parent_element):
        if not isinstance(parent_element.initial_value, dict):
            return

        parsed_names = set()
        for name, element_cls in schema.items():
            if name not in parent_element.initial_value_holder:
                value = None
            else:
                name, value = \
                    parent_element.initial_value_holder.get_item(name)
                parsed_names.add(name.value)
            self._traverse_element_cls(element_cls=element_cls,
                                       name=name,
                                       value=value,
                                       parent_element=parent_element)
        for k_holder, v_holder in parent_element.initial_value_holder.value.\
                iteritems():
            if k_holder.value not in parsed_names:
                self._traverse_element_cls(element_cls=elements.UnknownElement,
                                           name=k_holder, value=v_holder,
                                           parent_element=parent_element)

    def _traverse_element_type_schema(self, schema, parent_element):
        if isinstance(schema, elements.Leaf):
            return

        element_cls = schema.type
        if isinstance(schema, elements.Dict):
            if not isinstance(parent_element.initial_value, dict):
                return
            for name_holder, value_holder in parent_element.\
                    initial_value_holder.value.items():
                self._traverse_element_cls(element_cls=element_cls,
                                           name=name_holder,
                                           value=value_holder,
                                           parent_element=parent_element)
        elif isinstance(schema, elements.List):
            if not isinstance(parent_element.initial_value, list):
                return
            for index, value_holder in enumerate(
                    parent_element.initial_value_holder.value):
                self._traverse_element_cls(element_cls=element_cls,
                                           name=index,
                                           value=value_holder,
                                           parent_element=parent_element)
        else:
            raise ValueError('Illegal state should have been identified'
                             ' by schema API validation')

    def _traverse_list_schema(self, schema, parent_element):
        for schema_item in schema:
            self._traverse_schema(schema=schema_item,
                                  parent_element=parent_element)

    def _calculate_element_graph(self):
        self.element_graph = nx.DiGraph(self._element_tree)
        for element_type, _elements in self.element_type_to_elements.items():
            requires = element_type.requires
            for requirement, requirement_values in requires.items():
                requirement_values = [
                    Requirement(r) if isinstance(r, basestring)
                    else r for r in requirement_values]
                if requirement == 'inputs':
                    continue
                if requirement == 'self':
                    requirement = element_type
                dependencies = self.element_type_to_elements.get(
                    requirement, [])
                for dependency in dependencies:
                    for element in _elements:
                        predicates = [r.predicate for r in requirement_values
                                      if r.predicate is not None]
                        add_dependency = not predicates or all([
                            predicate(element, dependency)
                            for predicate in predicates])
                        if add_dependency:
                            self.element_graph.add_edge(element, dependency)
        # we reverse the graph because only netorkx 1.9.1 has the reverse
        # flag in the topological sort function, it is only used by it
        # so this should be good
        self.element_graph.reverse(copy=False)

    def elements_graph_topological_sort(self):
        try:
            return nx.topological_sort(self.element_graph)
        except nx.NetworkXUnfeasible:
            # Cycle detected
            cycle = nx.recursive_simple_cycles(self.element_graph)[0]
            names = [str(e.name) for e in cycle]
            names.append(str(names[0]))
            ex = exceptions.DSLParsingLogicException(
                exceptions.ERROR_CODE_CYCLE,
                'Parsing failed. Circular dependency detected: {0}'
                .format(' --> '.join(names)))
            ex.circular_dependency = names
            raise ex

class Parser(object):

    def parse(self,
              value,
              element_cls,
              element_name='root',
              inputs=None,
              strict=True):
        context = Context(
            value=value,
            element_cls=element_cls,
            element_name=element_name,
            inputs=inputs)
        for element in context.elements_graph_topological_sort():
            try:
                self._validate_element_schema(element, strict=strict)
                self._process_element(element)
            except exceptions.DSLParsingException as e:
                if not e.element:
                    e.element = element
                raise
        return context.parsed_value

    @staticmethod
    def _validate_element_schema(element, strict):
        value = element.initial_value
        if element.required and value is None:
            raise exceptions.DSLParsingFormatException(
                1, "'{0}' key is required but it is currently missing"
                   .format(element.name))

        def validate_schema(schema):
            if isinstance(schema, (dict, elements.Dict)):
                if not isinstance(value, dict):
                    raise exceptions.DSLParsingFormatException(
                        1, _expected_type_message(value, dict))
                for key in value.keys():
                    if not isinstance(key, basestring):
                        raise exceptions.DSLParsingFormatException(
                            1, "Dict keys must be strings but"
                               " found '{0}' of type '{1}'"
                               .format(key, _py_type_to_user_type(type(key))))

            if strict and isinstance(schema, dict):
                for key in value.keys():
                    if key not in schema:
                        ex = exceptions.DSLParsingFormatException(
                            1, "'{0}' is not in schema. "
                               "Valid schema values: {1}"
                               .format(key, schema.keys()))
                        for child_element in element.children():
                            if child_element.name == key:
                                ex.element = child_element
                                break
                        raise ex

            if (isinstance(schema, elements.List) and
                    not isinstance(value, list)):
                raise exceptions.DSLParsingFormatException(
                    1, _expected_type_message(value, list))

            if (isinstance(schema, elements.Leaf) and
                    not isinstance(value, schema.type)):
                raise exceptions.DSLParsingFormatException(
                    1, _expected_type_message(value, schema.type))
        if value is not None:
            if isinstance(element.schema, list):
                validated = False
                last_error = None
                for schema_item in element.schema:
                    try:
                        validate_schema(schema_item)
                    except exceptions.DSLParsingFormatException as e:
                        last_error = e
                    else:
                        validated = True
                        break
                if not validated:
                    if not last_error:
                        raise ValueError('Illegal state should have been '
                                         'identified by schema API validation')
                    else:
                        raise last_error
            else:
                validate_schema(element.schema)

    def _process_element(self, element):
        required_args = self._extract_element_requirements(element)
        element.validate(**required_args)
        element.value = element.parse(**required_args)
        element.provided = element.calculate_provided(**required_args)

    @staticmethod
    def _extract_element_requirements(element):
        context = element.context
        required_args = {}
        for required_type, requirements in element.requires.items():
            requirements = [Requirement(r) if isinstance(r, basestring)
                            else r for r in requirements]
            if not requirements:
                # only set required type as a logical dependency
                pass
            elif required_type == 'inputs':
                for input in requirements:
                    if input.name not in context.inputs and input.required:
                        raise exceptions.DSLParsingFormatException(
                            1, "Missing required input '{0}'. "
                               "Existing inputs: "
                               .format(input.name, context.inputs.keys()))
                    required_args[input.name] = context.inputs.get(input.name)
            else:
                if required_type == 'self':
                    required_type = type(element)
                required_type_elements = context.element_type_to_elements.get(
                    required_type, [])
                for requirement in requirements:
                    result = []
                    for required_element in required_type_elements:
                        if requirement.predicate and not requirement.predicate(
                                element, required_element):
                            continue
                        if requirement.parsed:
                            result.append(required_element.value)
                        else:
                            if (requirement.name not in
                                    required_element.provided):
                                provided = required_element.provided.keys()
                                if requirement.required:
                                    raise exceptions.DSLParsingFormatException(
                                        1,
                                        "Required value '{0}' is not "
                                        "provided by '{1}'. Provided values "
                                        "are: {2}"
                                        .format(requirement.name,
                                                required_element.name,
                                                provided))
                                else:
                                    continue
                            result.append(required_element.provided[
                                requirement.name])

                    if len(result) != 1 and not requirement.multiple_results:
                        if requirement.required:
                            raise exceptions.DSLParsingFormatException(
                                1, "Expected exactly one result for "
                                   "requirement '{0}' but found {1}"
                                   .format(requirement.name,
                                           'none' if not result else result))
                        elif not result:
                            result = [None]
                        else:
                            raise ValueError('Illegal state')

                    if not requirement.multiple_results:
                        result = result[0]
                    required_args[requirement.name] = result

        return required_args

_parser = Parser()

def validate_schema_api(element_cls):
    _schema_validator.validate(element_cls)

def parse(value,
          element_cls,
          element_name='root',
          inputs=None,
          strict=True):
    validate_schema_api(element_cls)
    return _parser.parse(value=value,
                         element_cls=element_cls,
                         element_name=element_name,
                         inputs=inputs,
                         strict=strict)

def _expected_type_message(value, expected_type):
    return ("Expected '{0}' type but found '{1}' type"
            .format(_py_type_to_user_type(expected_type),
                    _py_type_to_user_type(type(value))))

def _py_type_to_user_type(_type):
    if isinstance(_type, tuple):
        return list(set(_py_type_to_user_type(t) for t in _type))
    elif issubclass(_type, basestring):
        return 'string'
    elif issubclass(_type, bool):
        return 'boolean'
    elif issubclass(_type, int) or issubclass(_type, long):
        return 'integer'
    elif issubclass(_type, float):
        return 'float'
    elif issubclass(_type, dict):
        return 'dict'
    elif issubclass(_type, list):
        return 'list'
    else:
        raise ValueError('Unexpected type: {0}'.format(_type))
<commit_msg>CFY-7806 Batch dependencies between element types (#129)<commit_after>########
# Copyright (c) 2015 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.

import networkx as nx

from dsl_parser import exceptions
from dsl_parser.framework import elements
from dsl_parser.framework.requirements import Requirement

class SchemaAPIValidator(object):

    def validate(self, element_cls):
        self._traverse_element_cls(element_cls)

    def _traverse_element_cls(self, element_cls):
        try:
            if not issubclass(element_cls, elements.Element):
                raise exceptions.DSLParsingSchemaAPIException(1)
        except TypeError:
            raise exceptions.DSLParsingSchemaAPIException(1)
        self._traverse_schema(element_cls.schema)

    def _traverse_schema(self, schema, list_nesting=0):
        if isinstance(schema, dict):
            for key, value in schema.items():
                if not isinstance(key, basestring):
                    raise exceptions.DSLParsingSchemaAPIException(1)
                self._traverse_element_cls(value)
        elif isinstance(schema, list):
            if list_nesting > 0:
                raise exceptions.DSLParsingSchemaAPIException(1)
            if len(schema) == 0:
                raise exceptions.DSLParsingSchemaAPIException(1)
            for value in schema:
                self._traverse_schema(value, list_nesting + 1)
        elif isinstance(schema, elements.ElementType):
            if isinstance(schema, elements.Leaf):
                if not isinstance(schema.type, (type, list, tuple)):
                    raise exceptions.DSLParsingSchemaAPIException(1)
                if (isinstance(schema.type, (list, tuple)) and
                    (not schema.type or
                     not all([isinstance(i, type) for i in schema.type]))):
                    raise exceptions.DSLParsingSchemaAPIException(1)
            elif isinstance(schema, elements.Dict):
                self._traverse_element_cls(schema.type)
            elif isinstance(schema, elements.List):
                self._traverse_element_cls(schema.type)
            else:
                raise exceptions.DSLParsingSchemaAPIException(1)
        else:
            raise exceptions.DSLParsingSchemaAPIException(1)

_schema_validator = SchemaAPIValidator()

class Context(object):

    def __init__(self,
                 value,
                 element_cls,
                 element_name,
                 inputs):
        self.inputs = inputs or {}
        self.element_type_to_elements = {}
        self._root_element = None
        self._element_tree = nx.DiGraph()
        self._element_graph = nx.DiGraph()
        self._traverse_element_cls(element_cls=element_cls,
                                   name=element_name,
                                   value=value,
                                   parent_element=None)
        self._calculate_element_graph()

    @property
    def parsed_value(self):
        return self._root_element.value if self._root_element else None

    def child_elements_iter(self, element):
        return self._element_tree.successors_iter(element)

    def ancestors_iter(self, element):
        current_element = element
        while True:
            predecessors = self._element_tree.predecessors(current_element)
            if not predecessors:
                return
            if len(predecessors) > 1:
                raise exceptions.DSLParsingFormatException(
                    1, 'More than 1 parent found for {0}'
                       .format(element))
            current_element = predecessors[0]
            yield current_element

    def descendants(self, element):
        return nx.descendants(self._element_tree, element)

    def _add_element(self, element, parent=None):
        element_type = type(element)
        if element_type not in self.element_type_to_elements:
            self.element_type_to_elements[element_type] = []
        self.element_type_to_elements[element_type].append(element)

        self._element_tree.add_node(element)
        if parent:
            self._element_tree.add_edge(parent, element)
        else:
            self._root_element = element

    def _traverse_element_cls(self,
                              element_cls,
                              name,
                              value,
                              parent_element):
        element = element_cls(name=name,
                              initial_value=value,
                              context=self)
        self._add_element(element, parent=parent_element)
        self._traverse_schema(schema=element_cls.schema,
                              parent_element=element)

    def _traverse_schema(self, schema, parent_element):
        if isinstance(schema, dict):
            self._traverse_dict_schema(schema=schema,
                                       parent_element=parent_element)
        elif isinstance(schema, elements.ElementType):
            self._traverse_element_type_schema(
                schema=schema,
                parent_element=parent_element)
        elif isinstance(schema, list):
            self._traverse_list_schema(schema=schema,
                                       parent_element=parent_element)
        elif isinstance(schema, elements.UnknownSchema):
            pass
        else:
            raise ValueError('Illegal state should have been identified'
                             ' by schema API validation')

    def _traverse_dict_schema(self, schema, parent_element):
        if not isinstance(parent_element.initial_value, dict):
            return

        parsed_names = set()
        for name, element_cls in schema.items():
            if name not in parent_element.initial_value_holder:
                value = None
            else:
                name, value = \
                    parent_element.initial_value_holder.get_item(name)
                parsed_names.add(name.value)
            self._traverse_element_cls(element_cls=element_cls,
                                       name=name,
                                       value=value,
                                       parent_element=parent_element)
        for k_holder, v_holder in parent_element.initial_value_holder.value.\
                iteritems():
            if k_holder.value not in parsed_names:
                self._traverse_element_cls(element_cls=elements.UnknownElement,
                                           name=k_holder, value=v_holder,
                                           parent_element=parent_element)

    def _traverse_element_type_schema(self, schema, parent_element):
        if isinstance(schema, elements.Leaf):
            return

        element_cls = schema.type
        if isinstance(schema, elements.Dict):
            if not isinstance(parent_element.initial_value, dict):
                return
            for name_holder, value_holder in parent_element.\
                    initial_value_holder.value.items():
                self._traverse_element_cls(element_cls=element_cls,
                                           name=name_holder,
                                           value=value_holder,
                                           parent_element=parent_element)
        elif isinstance(schema, elements.List):
            if not isinstance(parent_element.initial_value, list):
                return
            for index, value_holder in enumerate(
                    parent_element.initial_value_holder.value):
                self._traverse_element_cls(element_cls=element_cls,
                                           name=index,
                                           value=value_holder,
                                           parent_element=parent_element)
        else:
            raise ValueError('Illegal state should have been identified'
                             ' by schema API validation')

    def _traverse_list_schema(self, schema, parent_element):
        for schema_item in schema:
            self._traverse_schema(schema=schema_item,
                                  parent_element=parent_element)

    def _calculate_element_graph(self):
        self.element_graph = nx.DiGraph(self._element_tree)
        for element_type, _elements in self.element_type_to_elements.items():
            requires = element_type.requires
            for requirement, requirement_values in requires.items():
                requirement_values = [
                    Requirement(r) if isinstance(r, basestring)
                    else r for r in requirement_values]
                if requirement == 'inputs':
                    continue
                if requirement == 'self':
                    requirement = element_type
                dependencies = self.element_type_to_elements.get(
                    requirement, [])
                predicates = [r.predicate for r in requirement_values
                              if r.predicate is not None]

                if not predicates:
                    dep = _BatchDependency(element_type, requirement)
                    for dependency in dependencies:
                        self.element_graph.add_edge(dep, dependency)
                    for element in _elements:
                        self.element_graph.add_edge(element, dep)
                    continue

                for dependency in dependencies:
                    for element in _elements:
                        add_dependency = all([
                            predicate(element, dependency)
                            for predicate in predicates])
                        if add_dependency:
                            self.element_graph.add_edge(element, dependency)
        # we reverse the graph because only netorkx 1.9.1 has the reverse
        # flag in the topological sort function, it is only used by it
        # so this should be good
        self.element_graph.reverse(copy=False)

    def elements_graph_topological_sort(self):
        try:
            return nx.topological_sort(self.element_graph)
        except nx.NetworkXUnfeasible:
            # Cycle detected
            cycle = nx.recursive_simple_cycles(self.element_graph)[0]
            names = [str(e.name) for e in cycle]
            names.append(str(names[0]))
            ex = exceptions.DSLParsingLogicException(
                exceptions.ERROR_CODE_CYCLE,
                'Parsing failed. Circular dependency detected: {0}'
                .format(' --> '.join(names)))
            ex.circular_dependency = names
            raise ex

class _BatchDependency(object):
    """Marker object to represent dependencies between types of elements.

    To force traversing the graph in order, we add edges between types
    of elements.
    This is used if all elements of one type must come before all
    elements of another type.
    """
    def __init__(self, dependent_type, dependency_type):
        self._dependent_type = dependent_type
        self._dependency_type = dependency_type

class Parser(object):

    def parse(self,
              value,
              element_cls,
              element_name='root',
              inputs=None,
              strict=True):
        context = Context(
            value=value,
            element_cls=element_cls,
            element_name=element_name,
            inputs=inputs)

        for element in context.elements_graph_topological_sort():

            if isinstance(element, _BatchDependency):
                continue
            try:
                self._validate_element_schema(element, strict=strict)
                self._process_element(element)
            except exceptions.DSLParsingException as e:
                if not e.element:
                    e.element = element
                raise
        return context.parsed_value

    @staticmethod
    def _validate_element_schema(element, strict):
        value = element.initial_value
        if element.required and value is None:
            raise exceptions.DSLParsingFormatException(
                1, "'{0}' key is required but it is currently missing"
                   .format(element.name))

        def validate_schema(schema):
            if isinstance(schema, (dict, elements.Dict)):
                if not isinstance(value, dict):
                    raise exceptions.DSLParsingFormatException(
                        1, _expected_type_message(value, dict))
                for key in value.keys():
                    if not isinstance(key, basestring):
                        raise exceptions.DSLParsingFormatException(
                            1, "Dict keys must be strings but"
                               " found '{0}' of type '{1}'"
                               .format(key, _py_type_to_user_type(type(key))))

            if strict and isinstance(schema, dict):
                for key in value.keys():
                    if key not in schema:
                        ex = exceptions.DSLParsingFormatException(
                            1, "'{0}' is not in schema. "
                               "Valid schema values: {1}"
                               .format(key, schema.keys()))
                        for child_element in element.children():
                            if child_element.name == key:
                                ex.element = child_element
                                break
                        raise ex

            if (isinstance(schema, elements.List) and
                    not isinstance(value, list)):
                raise exceptions.DSLParsingFormatException(
                    1, _expected_type_message(value, list))

            if (isinstance(schema, elements.Leaf) and
                    not isinstance(value, schema.type)):
                raise exceptions.DSLParsingFormatException(
                    1, _expected_type_message(value, schema.type))
        if value is not None:
            if isinstance(element.schema, list):
                validated = False
                last_error = None
                for schema_item in element.schema:
                    try:
                        validate_schema(schema_item)
                    except exceptions.DSLParsingFormatException as e:
                        last_error = e
                    else:
                        validated = True
                        break
                if not validated:
                    if not last_error:
                        raise ValueError('Illegal state should have been '
                                         'identified by schema API validation')
                    else:
                        raise last_error
            else:
                validate_schema(element.schema)

    def _process_element(self, element):
        required_args = self._extract_element_requirements(element)
        element.validate(**required_args)
        element.value = element.parse(**required_args)
        element.provided = element.calculate_provided(**required_args)

    @staticmethod
    def _extract_element_requirements(element):
        context = element.context
        required_args = {}
        for required_type, requirements in element.requires.items():
            requirements = [Requirement(r) if isinstance(r, basestring)
                            else r for r in requirements]
            if not requirements:
                # only set required type as a logical dependency
                pass
            elif required_type == 'inputs':
                for input in requirements:
                    if input.name not in context.inputs and input.required:
                        raise exceptions.DSLParsingFormatException(
                            1, "Missing required input '{0}'. "
                               "Existing inputs: "
                               .format(input.name, context.inputs.keys()))
                    required_args[input.name] = context.inputs.get(input.name)
            else:
                if required_type == 'self':
                    required_type = type(element)
                required_type_elements = context.element_type_to_elements.get(
                    required_type, [])
                for requirement in requirements:
                    result = []
                    for required_element in required_type_elements:
                        if requirement.predicate and not requirement.predicate(
                                element, required_element):
                            continue
                        if requirement.parsed:
                            result.append(required_element.value)
                        else:
                            if (requirement.name not in
                                    required_element.provided):
                                provided = required_element.provided.keys()
                                if requirement.required:
                                    raise exceptions.DSLParsingFormatException(
                                        1,
                                        "Required value '{0}' is not "
                                        "provided by '{1}'. Provided values "
                                        "are: {2}"
                                        .format(requirement.name,
                                                required_element.name,
                                                provided))
                                else:
                                    continue
                            result.append(required_element.provided[
                                requirement.name])

                    if len(result) != 1 and not requirement.multiple_results:
                        if requirement.required:
                            raise exceptions.DSLParsingFormatException(
                                1, "Expected exactly one result for "
                                   "requirement '{0}' but found {1}"
                                   .format(requirement.name,
                                           'none' if not result else result))
                        elif not result:
                            result = [None]
                        else:
                            raise ValueError('Illegal state')

                    if not requirement.multiple_results:
                        result = result[0]
                    required_args[requirement.name] = result

        return required_args

_parser = Parser()

def validate_schema_api(element_cls):
    _schema_validator.validate(element_cls)

def parse(value,
          element_cls,
          element_name='root',
          inputs=None,
          strict=True):
    validate_schema_api(element_cls)
    return _parser.parse(value=value,
                         element_cls=element_cls,
                         element_name=element_name,
                         inputs=inputs,
                         strict=strict)

def _expected_type_message(value, expected_type):
    return ("Expected '{0}' type but found '{1}' type"
            .format(_py_type_to_user_type(expected_type),
                    _py_type_to_user_type(type(value))))

def _py_type_to_user_type(_type):
    if isinstance(_type, tuple):
        return list(set(_py_type_to_user_type(t) for t in _type))
    elif issubclass(_type, basestring):
        return 'string'
    elif issubclass(_type, bool):
        return 'boolean'
    elif issubclass(_type, int) or issubclass(_type, long):
        return 'integer'
    elif issubclass(_type, float):
        return 'float'
    elif issubclass(_type, dict):
        return 'dict'
    elif issubclass(_type, list):
        return 'list'
    else:
        raise ValueError('Unexpected type: {0}'.format(_type))
<|endoftext|>


<commit_before># SfePy version
__version__ = '2015.2'

# "Minimal" supported versions.
NUMPY_MIN_VERSION = '1.3'
SCIPY_MIN_VERSION = '0.7'
MATPLOTLIB_MIN_VERSION = '0.99.0'
PYPARSING_MIN_VERSION = '1.5.0'
PYTABLES_MIN_VERSION = '2.1.2'
IPYTHON_MIN_VERSION = '0.10.0'
MAYAVI_MIN_VERSION = '3.3.0'
SYMPY_MIN_VERSION = '0.6.7'
IGAKIT_MIN_VERSION = '0.1'
PETSC4PY_MIN_VERSION = '3.4'
MPI4PY_MIN_VERSION = '1.3.1'
PYMETIS_MIN_VERSION = '2014.1'

CYTHON_MIN_VERSION = '0.14.1'

def get_basic_info(version=__version__):
    """
    Return SfePy installation directory information. Append current git
    commit hash to `version`.
    """
    import os.path as op
    from sfepy import Config

    # If installed, up_dir is '.', otherwise (in (git) source directory) '..'.
    for up_dir in ['..', '.']:
        top_dir = op.normpath(op.realpath(op.join(op.dirname(__file__),
                                                  up_dir)))
        aux = op.join(top_dir, 'README')
        if op.isfile(aux):
            break
    else:
        raise RuntimeError('cannot determine SfePy top level directory!')

    config = Config()
    if not config.is_release():
        # Append current git commit hash to __version__.
        master = op.join(top_dir, '.git/refs/heads/master')
        if op.isfile(master):
            fd = open(master, 'r')
            version += '-git-%s' % fd.readline().strip()
            fd.close()

    in_source_tree = up_dir == '..'

    return version, top_dir, in_source_tree

__version__, top_dir, in_source_tree = get_basic_info(__version__)
<commit_msg>remove unused IPYTHON_MIN_VERSION<commit_after># SfePy version
__version__ = '2015.2'

# "Minimal" supported versions.
NUMPY_MIN_VERSION = '1.3'
SCIPY_MIN_VERSION = '0.7'
MATPLOTLIB_MIN_VERSION = '0.99.0'
PYPARSING_MIN_VERSION = '1.5.0'
PYTABLES_MIN_VERSION = '2.1.2'
MAYAVI_MIN_VERSION = '3.3.0'
SYMPY_MIN_VERSION = '0.6.7'
IGAKIT_MIN_VERSION = '0.1'
PETSC4PY_MIN_VERSION = '3.4'
MPI4PY_MIN_VERSION = '1.3.1'
PYMETIS_MIN_VERSION = '2014.1'

CYTHON_MIN_VERSION = '0.14.1'

def get_basic_info(version=__version__):
    """
    Return SfePy installation directory information. Append current git
    commit hash to `version`.
    """
    import os.path as op
    from sfepy import Config

    # If installed, up_dir is '.', otherwise (in (git) source directory) '..'.
    for up_dir in ['..', '.']:
        top_dir = op.normpath(op.realpath(op.join(op.dirname(__file__),
                                                  up_dir)))
        aux = op.join(top_dir, 'README')
        if op.isfile(aux):
            break
    else:
        raise RuntimeError('cannot determine SfePy top level directory!')

    config = Config()
    if not config.is_release():
        # Append current git commit hash to __version__.
        master = op.join(top_dir, '.git/refs/heads/master')
        if op.isfile(master):
            fd = open(master, 'r')
            version += '-git-%s' % fd.readline().strip()
            fd.close()

    in_source_tree = up_dir == '..'

    return version, top_dir, in_source_tree

__version__, top_dir, in_source_tree = get_basic_info(__version__)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Regression example for JDOT
"""

# Author: Remi Flamary <remi.flamary@unice.fr>
#         Nicolas Courty <ncourty@irisa.fr>
#
# License: MIT License

import numpy as np
import pylab as pl

import jdot

#from sklearn import datasets
import sklearn
from scipy.spatial.distance import cdist
import ot

#%% data generation

seed=1985
np.random.seed(seed)

n = 200
ntest=200
nz=.3

theta=0.8

n2=int(n/2)
sigma=0.05

xs=np.random.randn(n,1)+2
xs[:n2,:]-=4
ys=sigma*np.random.randn(n,1)+np.sin(xs/2)

fs_s = lambda x: np.sin(x/2)

xt=np.random.randn(n,1)+2
xt[:n2,:]/=2
xt[:n2,:]-=3

gauss = lambda x,s,m: np.exp((x-m)**2/(2*s*s))/(s*np.sqrt(2*np.pi))

mus_x = lambda x: gauss(x,1,2)/2+gauss(x,1,-2)/2

yt=sigma*np.random.randn(n,1)+np.sin(xt/2)
xt+=2

fs_t = lambda x: np.sin((x-2)/2)

mut_x = lambda x: gauss(x,1,2)/2+gauss(x,1./2,-4)/2

xvisu=np.linspace(-4,6.5,100)

pl.figure(1)
pl.clf()

pl.subplot()
pl.scatter(xs,ys,label='Source samples',edgecolors='k')
pl.scatter(xt,yt,label='Target samples',edgecolors='k')
pl.plot(xvisu,fs_s(xvisu),'b',label='Source model')
pl.plot(xvisu,fs_t(xvisu),'g',label='Target model')
pl.xlabel('x')

pl.ylabel('y')
pl.legend()
pl.title('Toy regression example')
#pl.savefig('imgs/visu_data_reg.eps')

#%% TLOT
lambd0=1e1
itermax=15
gamma=1e-1
alpha=1e0/4
C0=cdist(xs,xt,metric='sqeuclidean')
#print np.max(C0)
C0=C0/np.median(C0)
fcost = cdist(ys,yt,metric='sqeuclidean')
C=alpha*C0+fcost
G0=ot.emd(ot.unif(n),ot.unif(n),C)

model,loss = jdot.jdot_krr(xs,ys,xt,gamma_g=gamma,numIterBCD = 10, alpha=alpha, lambd=lambd0,ktype='rbf')

K=sklearn.metrics.pairwise.rbf_kernel(xt,xt,gamma=gamma)
Kvisu=sklearn.metrics.pairwise.rbf_kernel(xvisu.reshape((-1,1)),xt,gamma=gamma)
ypred=model.predict(Kvisu)
ypred0=model.predict(K)

# compute true OT
C0=cdist(xs,xt,metric='sqeuclidean')
#print np.max(C0)
C0=C0/np.median(C0)
fcost = cdist(ys,ypred0,metric='sqeuclidean')
C=alpha*C0+fcost
G=ot.emd(ot.unif(n),ot.unif(n),C)

pl.figure(2)
pl.clf()
pl.scatter(xs,ys,label='Source samples',edgecolors='k')
pl.scatter(xt,yt,label='Target samples',edgecolors='k')
pl.plot(xvisu,fs_s(xvisu),'b',label='Source model')
pl.plot(xvisu,fs_t(xvisu),'g',label='Target model')
pl.plot(xvisu,ypred,'r',label='Tloss model')
pl.xlabel('x')

pl.ylabel('y')
pl.legend()
pl.title('Toy regression example')

#%%
seed=1985
np.random.seed(seed)
idv=np.random.permutation(n)
fs=17
nb=15
pl.figure(3,(10,7))
pl.clf()

pl.subplot(2,2,1)
pl.scatter(xs,ys,label='Source samples',edgecolors='k')
pl.scatter(xt,yt,label='Target samples',edgecolors='k')
pl.plot(xvisu,fs_s(xvisu),label='Source model')
pl.plot(xvisu,fs_t(xvisu),label='Target model')
#pl.xlabel('x')

pl.ylabel('y')
pl.legend(loc=4,fontsize=.7*fs)
pl.title('Toy regression example',fontsize=fs)

pl.subplot(2,2,2)
pl.scatter(xs,ys,edgecolors='k')
pl.scatter(xt,yt,edgecolors='k')
#pl.plot(xvisu,fs_s(xvisu),'b',label='Source model')
#pl.plot(xvisu,fs_t(xvisu),'g',label='Target model')
for i in range(nb):
    idt=G0[idv[i],:].argmax()
    if not i:
        pl.plot([xs[idv[i]],xt[idt]],[ys[idv[i]],yt[idt]],'k',label='OT matrix link')
    else:
        pl.plot([xs[idv[i]],xt[idt]],[ys[idv[i]],yt[idt]],'k')

#pl.xlabel('x')

pl.ylabel('y')
pl.legend(loc=4,fontsize=.7*fs)
pl.title('OT matrix on joint distribution',fontsize=fs)

pl.subplot(2,2,3)
pl.scatter(xs,ys,edgecolors='k')
pl.scatter(xt,yt,edgecolors='k')
#pl.plot(xvisu,fs_s(xvisu),'b',label='Source model')
#pl.plot(xvisu,fs_t(xvisu),'g',label='Target model')
for i in range(nb):
    idt=G[idv[i],:].argmax()

    pl.plot([xs[idv[i]],xt[idt]],[ys[idv[i]],yt[idt]],'k')

pl.xlabel('x')

pl.ylabel('y')
pl.legend(loc=4)
pl.title('OT matrix of JDOT',fontsize=fs)

pl.subplot(2,2,4)
pl.scatter(xs,ys,edgecolors='k')
pl.scatter(xt,yt,edgecolors='k')
pl.plot(xvisu,fs_s(xvisu),label='Source model')
pl.plot(xvisu,fs_t(xvisu),label='Target model')
pl.plot(xvisu,ypred,'g',label='JDOT model')
pl.xlabel('x')

pl.ylabel('y')
pl.legend(loc=4,fontsize=.7*fs)
pl.title('Model estimated with JDOT',fontsize=fs)

pl.tight_layout()
#pl.savefig('imgs/visu_reg.eps')

#%%
#seed=1985
#np.random.seed(seed)
#idv=np.random.permutation(n)
#fs=12
#nb=15
#pl.figure(3,(12,3))
#pl.clf()
#
#pl.subplot(1,4,1)
#pl.scatter(xs,ys,label='Source samples',edgecolors='k')
#pl.scatter(xt,yt,label='Target samples',edgecolors='k')
#pl.plot(xvisu,fs_s(xvisu),label='Source model')
#pl.plot(xvisu,fs_t(xvisu),label='Target model')
##pl.xlabel('x')
#pl.xlabel('x')
#pl.ylabel('y')
#pl.legend(loc=4,fontsize=.7*fs)
#pl.title('Toy regression example',fontsize=fs)
#
#pl.subplot(1,4,2)
#pl.scatter(xs,ys,edgecolors='k')
#pl.scatter(xt,yt,edgecolors='k')
##pl.plot(xvisu,fs_s(xvisu),'b',label='Source model')
##pl.plot(xvisu,fs_t(xvisu),'g',label='Target model')
#for i in range(nb):
#    idt=G0[idv[i],:].argmax()
#    if not i:
#        pl.plot([xs[idv[i]],xt[idt]],[ys[idv[i]],yt[idt]],'k',label='OT matrix link')
#    else:
#        pl.plot([xs[idv[i]],xt[idt]],[ys[idv[i]],yt[idt]],'k')
#
##pl.xlabel('x')
#
##pl.ylabel('y')
#pl.legend(loc=4,fontsize=.7*fs)
#pl.title('OT matrix on joint distribution',fontsize=fs)
#pl.xlabel('x')
#pl.subplot(1,4,3)
#pl.scatter(xs,ys,edgecolors='k')
#pl.scatter(xt,yt,edgecolors='k')
##pl.plot(xvisu,fs_s(xvisu),'b',label='Source model')
##pl.plot(xvisu,fs_t(xvisu),'g',label='Target model')
#for i in range(nb):
#    idt=G[idv[i],:].argmax()
#
#    pl.plot([xs[idv[i]],xt[idt]],[ys[idv[i]],yt[idt]],'k')
#
#pl.xlabel('x')
#
##pl.ylabel('y')
#pl.legend(loc=4)
#pl.title('OT matrix of JDOT',fontsize=fs)
#
#pl.subplot(1,4,4)
#pl.scatter(xs,ys,edgecolors='k')
#pl.scatter(xt,yt,edgecolors='k')
#pl.plot(xvisu,fs_s(xvisu),label='Source model')
#pl.plot(xvisu,fs_t(xvisu),label='Target model')
#pl.plot(xvisu,ypred,'g',label='JDOT model')
#pl.xlabel('x')
#
##pl.ylabel('y')
#pl.legend(loc=4,fontsize=.7*fs)
#pl.title('Model estimated with JDOT',fontsize=fs)
#
#pl.tight_layout(pad=00,h_pad=0)
##pl.savefig('imgs/visu_reg2.eps')
##pl.savefig('imgs/visu_reg2.pdf')<commit_msg>add visu data for regression<commit_after># -*- coding: utf-8 -*-
"""
Regression example for JDOT
"""

# Author: Remi Flamary <remi.flamary@unice.fr>
#         Nicolas Courty <ncourty@irisa.fr>
#
# License: MIT License

import numpy as np
import pylab as pl

import jdot

#from sklearn import datasets
import sklearn
from scipy.spatial.distance import cdist
import ot

#%% data generation

seed=1985
np.random.seed(seed)

n = 200
ntest=200
nz=.3

theta=0.8

n2=int(n/2)
sigma=0.05

xs=np.random.randn(n,1)+2
xs[:n2,:]-=4
ys=sigma*np.random.randn(n,1)+np.sin(xs/2)

fs_s = lambda x: np.sin(x/2)

xt=np.random.randn(n,1)+2
xt[:n2,:]/=2
xt[:n2,:]-=3

gauss = lambda x,s,m: np.exp((x-m)**2/(2*s*s))/(s*np.sqrt(2*np.pi))

mus_x = lambda x: gauss(x,1,2)/2+gauss(x,1,-2)/2

yt=sigma*np.random.randn(n,1)+np.sin(xt/2)
xt+=2

fs_t = lambda x: np.sin((x-2)/2)

mut_x = lambda x: gauss(x,1,2)/2+gauss(x,1./2,-4)/2

xvisu=np.linspace(-4,6.5,100)

pl.figure(1)
pl.clf()

pl.subplot()
pl.scatter(xs,ys,label='Source samples',edgecolors='k')
pl.scatter(xt,yt,label='Target samples',edgecolors='k')
pl.plot(xvisu,fs_s(xvisu),'b',label='Source model')
pl.plot(xvisu,fs_t(xvisu),'g',label='Target model')
pl.xlabel('x')

pl.ylabel('y')
pl.legend()
pl.title('Toy regression example')
#pl.savefig('imgs/visu_data_reg.eps')

#%% TLOT
lambd0=1e1
itermax=15
gamma=1e-1
alpha=1e0/4
C0=cdist(xs,xt,metric='sqeuclidean')
#print np.max(C0)
C0=C0/np.median(C0)
fcost = cdist(ys,yt,metric='sqeuclidean')
C=alpha*C0+fcost
G0=ot.emd(ot.unif(n),ot.unif(n),C)

model,loss = jdot.jdot_krr(xs,ys,xt,gamma_g=gamma,numIterBCD = 10, alpha=alpha, lambd=lambd0,ktype='rbf')

K=sklearn.metrics.pairwise.rbf_kernel(xt,xt,gamma=gamma)
Kvisu=sklearn.metrics.pairwise.rbf_kernel(xvisu.reshape((-1,1)),xt,gamma=gamma)
ypred=model.predict(Kvisu)
ypred0=model.predict(K)

# compute true OT
C0=cdist(xs,xt,metric='sqeuclidean')
#print np.max(C0)
C0=C0/np.median(C0)
fcost = cdist(ys,ypred0,metric='sqeuclidean')
C=alpha*C0+fcost
G=ot.emd(ot.unif(n),ot.unif(n),C)

pl.figure(2)
pl.clf()
pl.scatter(xs,ys,label='Source samples',edgecolors='k')
pl.scatter(xt,yt,label='Target samples',edgecolors='k')
pl.plot(xvisu,fs_s(xvisu),'b',label='Source model')
pl.plot(xvisu,fs_t(xvisu),'g',label='Target model')
pl.plot(xvisu,ypred,'r',label='Tloss model')
pl.xlabel('x')

pl.ylabel('y')
pl.legend()
pl.title('Toy regression example')

#%%
seed=1985
np.random.seed(seed)
idv=np.random.permutation(n)
fs=17
nb=15
pl.figure(3,(10,7))
pl.clf()

pl.subplot(2,2,1)
pl.scatter(xs,ys,label='Source samples',edgecolors='k')
pl.scatter(xt,yt,label='Target samples',edgecolors='k')
pl.plot(xvisu,fs_s(xvisu),label='Source model')
pl.plot(xvisu,fs_t(xvisu),label='Target model')
#pl.xlabel('x')

pl.ylabel('y')
pl.legend(loc=4,fontsize=.7*fs)
pl.title('Toy regression example',fontsize=fs)

pl.subplot(2,2,2)
pl.scatter(xs,ys,edgecolors='k')
pl.scatter(xt,yt,edgecolors='k')
#pl.plot(xvisu,fs_s(xvisu),'b',label='Source model')
#pl.plot(xvisu,fs_t(xvisu),'g',label='Target model')
for i in range(nb):
    idt=G0[idv[i],:].argmax()
    if not i:
        pl.plot([xs[idv[i]],xt[idt]],[ys[idv[i]],yt[idt]],'k',label='OT matrix link')
    else:
        pl.plot([xs[idv[i]],xt[idt]],[ys[idv[i]],yt[idt]],'k')

#pl.xlabel('x')

pl.ylabel('y')
pl.legend(loc=4,fontsize=.7*fs)
pl.title('OT matrix on joint distribution',fontsize=fs)

pl.subplot(2,2,3)
pl.scatter(xs,ys,edgecolors='k')
pl.scatter(xt,yt,edgecolors='k')
#pl.plot(xvisu,fs_s(xvisu),'b',label='Source model')
#pl.plot(xvisu,fs_t(xvisu),'g',label='Target model')
for i in range(nb):
    idt=G[idv[i],:].argmax()

    pl.plot([xs[idv[i]],xt[idt]],[ys[idv[i]],yt[idt]],'k')

pl.xlabel('x')

pl.ylabel('y')
pl.legend(loc=4)
pl.title('OT matrix of JDOT',fontsize=fs)

pl.subplot(2,2,4)
pl.scatter(xs,ys,edgecolors='k')
pl.scatter(xt,yt,edgecolors='k')
pl.plot(xvisu,fs_s(xvisu),label='Source model')
pl.plot(xvisu,fs_t(xvisu),label='Target model')
pl.plot(xvisu,ypred,'g',label='JDOT model')
pl.xlabel('x')

pl.ylabel('y')
pl.legend(loc=4,fontsize=.7*fs)
pl.title('Model estimated with JDOT',fontsize=fs)

pl.tight_layout()
#pl.savefig('imgs/visu_reg.eps')

#%% visu joint distributions
from matplotlib import gridspec
seed=1985
np.random.seed(seed)

n = 2000000
ntest=2000000
nz=.3

theta=0.8

n2=int(n/2)
sigma=0.05

xs=np.random.randn(n,1)+2
xs[:n2,:]-=4
ys=sigma*np.random.randn(n,1)+np.sin(xs/2)

fs_s = lambda x: np.sin(x/2)

xt=np.random.randn(n,1)+2
xt[:n2,:]/=2
xt[:n2,:]-=3

gauss = lambda x,s,m: np.exp((x-m)**2/(2*s*s))/(s*np.sqrt(2*np.pi))

mus_x = lambda x: gauss(x,1,2)/2+gauss(x,1,-2)/2

yt=sigma*np.random.randn(n,1)+np.sin(xt/2)
xt+=2

fs_t = lambda x: np.sin((x-2)/2)

mut_x = lambda x: gauss(x,1,2)/2+gauss(x,1./2,-4)/2

xvisu=np.linspace(-4,6.5,100)

#%% data visu
pl.figure(5,(10,10))
pl.clf()
from matplotlib import gridspec
gs = gridspec.GridSpec(3, 3)
ax1 = pl.subplot(gs[0, 1:])

#pl.hist(xs.ravel(),bins=200,histtype='step')
#pl.hist(xt.ravel(),bins=200,histtype='step')

nbin=100
nbvisu=500

iss=np.random.permutation(n)[:nbvisu]
ist=np.random.permutation(ntest)[:nbvisu]

hys,pos=np.histogram(xs,nbin)
hyt,pot=np.histogram(xt,nbin)
pl.plot(pos[1:],hys)
pl.plot(pot[1:],hyt)

pl.yticks(())
pl.ylim([0,pl.ylim()[1]])

xl=pl.xlim()
pl.title('x marginals')

ax2 = pl.subplot(gs[1:, 0])

#pl.hist(ys.ravel(),bins=200,histtype='step')
#pl.hist(yt.ravel(),bins=200,histtype='step')
pl.gca().invert_xaxis()
#pl.gca().invert_yaxis()

hys,pos=np.histogram(ys,nbin)
hyt,pot=np.histogram(yt,nbin)
pl.plot(hys,pos[1:])
pl.plot(hyt,pot[1:])

pl.xticks(())
yl=pl.ylim()
pl.xlim([pl.xlim()[0],0])
rg=[xl, np.sort(yl)]
pl.ylabel('y marginals')

pl.subplot(gs[1:, 1:], sharex=ax1, sharey=ax2)

pl.scatter(xs[iss],ys[iss],edgecolors='k')
pl.scatter(xt[ist],yt[ist],edgecolors='k')
pl.plot(xvisu,fs_s(xvisu),label='Source model')
pl.plot(xvisu,fs_t(xvisu),label='Target model')
#pl.hist2d(xs.ravel(),ys.ravel(),cmap='Blues',bins=200,alpha=0.99,range=rg)
#pl.hist2d(xt.ravel(),yt.ravel(),cmap='Oranges',bins=200,alpha=0.7,range=rg)

pl.xticks(())
pl.yticks(())
pl.savefig('imgs/visu_data_reg.png')
#
#gs = gridspe

#%%
#seed=1985
#np.random.seed(seed)
#idv=np.random.permutation(n)
#fs=12
#nb=15
#pl.figure(3,(12,3))
#pl.clf()
#
#pl.subplot(1,4,1)
#pl.scatter(xs,ys,label='Source samples',edgecolors='k')
#pl.scatter(xt,yt,label='Target samples',edgecolors='k')
#pl.plot(xvisu,fs_s(xvisu),label='Source model')
#pl.plot(xvisu,fs_t(xvisu),label='Target model')
##pl.xlabel('x')
#pl.xlabel('x')
#pl.ylabel('y')
#pl.legend(loc=4,fontsize=.7*fs)
#pl.title('Toy regression example',fontsize=fs)
#
#pl.subplot(1,4,2)
#pl.scatter(xs,ys,edgecolors='k')
#pl.scatter(xt,yt,edgecolors='k')
##pl.plot(xvisu,fs_s(xvisu),'b',label='Source model')
##pl.plot(xvisu,fs_t(xvisu),'g',label='Target model')
#for i in range(nb):
#    idt=G0[idv[i],:].argmax()
#    if not i:
#        pl.plot([xs[idv[i]],xt[idt]],[ys[idv[i]],yt[idt]],'k',label='OT matrix link')
#    else:
#        pl.plot([xs[idv[i]],xt[idt]],[ys[idv[i]],yt[idt]],'k')
#
##pl.xlabel('x')
#
##pl.ylabel('y')
#pl.legend(loc=4,fontsize=.7*fs)
#pl.title('OT matrix on joint distribution',fontsize=fs)
#pl.xlabel('x')
#pl.subplot(1,4,3)
#pl.scatter(xs,ys,edgecolors='k')
#pl.scatter(xt,yt,edgecolors='k')
##pl.plot(xvisu,fs_s(xvisu),'b',label='Source model')
##pl.plot(xvisu,fs_t(xvisu),'g',label='Target model')
#for i in range(nb):
#    idt=G[idv[i],:].argmax()
#
#    pl.plot([xs[idv[i]],xt[idt]],[ys[idv[i]],yt[idt]],'k')
#
#pl.xlabel('x')
#
##pl.ylabel('y')
#pl.legend(loc=4)
#pl.title('OT matrix of JDOT',fontsize=fs)
#
#pl.subplot(1,4,4)
#pl.scatter(xs,ys,edgecolors='k')
#pl.scatter(xt,yt,edgecolors='k')
#pl.plot(xvisu,fs_s(xvisu),label='Source model')
#pl.plot(xvisu,fs_t(xvisu),label='Target model')
#pl.plot(xvisu,ypred,'g',label='JDOT model')
#pl.xlabel('x')
#
##pl.ylabel('y')
#pl.legend(loc=4,fontsize=.7*fs)
#pl.title('Model estimated with JDOT',fontsize=fs)
#
#pl.tight_layout(pad=00,h_pad=0)
##pl.savefig('imgs/visu_reg2.eps')
##pl.savefig('imgs/visu_reg2.pdf')<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import sys
import subprocess

import boto3

from drift.utils import get_config

def get_options(parser):
    parser.add_argument("service", help="Service or deployable to connect to", nargs='?')

def run_command(args):
    service = args.service
    conf = get_config()
    tier_name = conf.tier['tier_name']
    region = conf.tier['aws']['region']
    ssh_key_name = conf.tier['aws']['ssh_key']
    deployables = conf.table_store.get_table('deployables').find({"tier_name": tier_name})
    deployables = {depl["deployable_name"]: depl for depl in deployables}
    if service is None:
        print "Select an instance to connect to:"
        for k in sorted(deployables.keys()):
            print "   ", k
        return

    elif service not in deployables:
        print "Service or deployable '{}' not one of {}.".format(service, deployables.keys())
        return

    ssh_key_file = '~/.ssh/{}.pem'.format(ssh_key_name)

    # Get IP address of any instance of this deployable.
    sess = boto3.session.Session(region_name=region)
    ec2 = sess.client("ec2")
    filters = [{"Name": "instance-state-name", "Values": ["running"]},
               {"Name": "tag:tier", "Values": [tier_name]},
               {"Name": "tag:service-name", "Values": [service]},
               ]
    print "Getting a list of EC2's from AWS matching the following criteria:"
    for criteria in filters:
        print "   {} = {}".format(criteria["Name"], criteria["Values"][0])

    ret = ec2.describe_instances(Filters=filters)
    instances = []
    for res in ret["Reservations"]:
        instances += res["Instances"]

    if not instances:
        print "No instance found which matches the criteria."
        return

    print "Instances:"
    inst = instances[0]
    for i, ins in enumerate(instances):
        lb = [tag["Value"] for tag in ins["Tags"] if tag["Key"] == "launched-by"] or ["n/a"]
        print "  {}: {} at {} launched by {} on {}".format(
            i + 1, ins["InstanceId"], ins["PrivateIpAddress"], lb[0], ins["LaunchTime"])

    if len(instances) > 1:
        which = raw_input("Select an instance to connect to (or press enter for first one): ")
        if which:
            inst = instances[int(which) - 1]
    else:
        print "Only one instance available. Connecting to it immediately.."

    ip_address = inst["PrivateIpAddress"]
    cd_cmd = ""
    if service in deployables:
        cd_cmd = 'cd /usr/local/bin/{}; exec bash --login'.format(service)
    cmd = ["ssh", "ubuntu@{}".format(ip_address), "-i", ssh_key_file, "-t", cd_cmd]
    print "\nSSH command:", " ".join(cmd)
    p = subprocess.Popen(cmd)
    stdout, _ = p.communicate()
    if p.returncode != 0:
        print stdout
        sys.exit(p.returncode)
<commit_msg>Enabling ssh command even though service name is not defined in deployables. Support ~ in ssh key names.<commit_after># -*- coding: utf-8 -*-

import sys
import os
import subprocess

import boto3

from drift.utils import get_config

def get_options(parser):
    parser.add_argument("service", help="Service or deployable to connect to", nargs='?')

def run_command(args):
    service = args.service
    conf = get_config()
    tier_name = conf.tier['tier_name']
    region = conf.tier['aws']['region']
    ssh_key_name = conf.tier['aws']['ssh_key']
    deployables = conf.table_store.get_table('deployables').find({"tier_name": tier_name})
    deployables = {depl["deployable_name"]: depl for depl in deployables}
    if service is None:
        print "Select an instance to connect to:"
        for k in sorted(deployables.keys()):
            print "   ", k
        return

    elif service not in deployables:
        print "Warning! Service or deployable '{}' not one of {}.".format(service, ", ".join(deployables.keys()))

    ssh_key_file = '~/.ssh/{}.pem'.format(ssh_key_name)
    ssh_key_file = ssh_key_file.replace('~', os.path.expanduser('~'))

    # Get IP address of any instance of this deployable.
    sess = boto3.session.Session(region_name=region)
    ec2 = sess.client("ec2")
    filters = [{"Name": "instance-state-name", "Values": ["running"]},
               {"Name": "tag:tier", "Values": [tier_name]},
               {"Name": "tag:service-name", "Values": [service]},
               ]
    print "Getting a list of EC2's from AWS matching the following criteria:"
    for criteria in filters:
        print "   {} = {}".format(criteria["Name"], criteria["Values"][0])

    ret = ec2.describe_instances(Filters=filters)
    instances = []
    for res in ret["Reservations"]:
        instances += res["Instances"]

    if not instances:
        print "No instance found which matches the criteria."
        return

    print "Instances:"
    inst = instances[0]
    for i, ins in enumerate(instances):
        lb = [tag["Value"] for tag in ins["Tags"] if tag["Key"] == "launched-by"] or ["n/a"]
        print "  {}: {} at {} launched by {} on {}".format(
            i + 1, ins["InstanceId"], ins["PrivateIpAddress"], lb[0], ins["LaunchTime"])

    if len(instances) > 1:
        which = raw_input("Select an instance to connect to (or press enter for first one): ")
        if which:
            inst = instances[int(which) - 1]
    else:
        print "Only one instance available. Connecting to it immediately.."

    ip_address = inst["PrivateIpAddress"]
    cd_cmd = ""
    if service in deployables:
        cd_cmd = 'cd /usr/local/bin/{}; exec bash --login'.format(service)
    cmd = ["ssh", "ubuntu@{}".format(ip_address), "-i", ssh_key_file, "-t", cd_cmd]
    print "\nSSH command:", " ".join(cmd)
    p = subprocess.Popen(cmd)
    stdout, _ = p.communicate()
    if p.returncode != 0:
        print stdout
        sys.exit(p.returncode)
<|endoftext|>


<commit_before>"""API that generates auto-complete suggestions for the search bar in the header of seqr pages"""

import collections
import logging

from django.contrib.auth.decorators import login_required
from django.db.models import Q
from django.db.models.functions import Length

from django.views.decorators.http import require_GET
from guardian.shortcuts import get_objects_for_user

from seqr.views.apis.auth_api import API_LOGIN_REQUIRED_URL
from seqr.views.utils.json_utils import create_json_response
from reference_data.models import GencodeGene
from seqr.models import Project, Family, Individual, CAN_VIEW

logger = logging.getLogger(__name__)

MAX_RESULTS_PER_CATEGORY = 10
MAX_STRING_LENGTH = 100

def _get_matching_projects(user, query):
    """Returns projects that match the given query string, and that the user can view.

    Args:
        user: Django user
        query: String typed into the awesomebar
    Returns:
        Sorted list of matches where each match is a dictionary of strings
    """
    project_permissions_check = Q()
    if not user.is_staff:
        projects_can_view = [p.id for p in get_objects_for_user(user, CAN_VIEW, Project)]
        project_permissions_check = Q(id__in=projects_can_view)

    matching_projects = Project.objects.filter(
        project_permissions_check & (
            Q(deprecated_project_id__icontains=query) |
            Q(name__icontains=query)
        )  # Q(description__icontains=query) | Q(project_category__icontains=query)
    )

    projects_result = []
    for p in matching_projects[:MAX_RESULTS_PER_CATEGORY]:
        title = p.name
        projects_result.append({
            'title': title[:MAX_STRING_LENGTH],
            'description': '',  # '('+p.description+')' if p.description else '',
            'href': '/project/'+p.deprecated_project_id,
        })

    projects_result.sort(key=lambda f: len(f.get('title', '')))

    return projects_result

def _get_matching_families(user, query):
    """Returns families that match the given query string, and that the user can view.

    Args:
        user: Django user
        query: String typed into the awesomebar
    Returns:
        Sorted list of matches where each match is a dictionary of strings
    """
    family_permissions_check = Q()
    if not user.is_staff:
        projects_can_view = [p.id for p in get_objects_for_user(user, CAN_VIEW, Project)]
        family_permissions_check = Q(project__id__in=projects_can_view)

    matching_families = Family.objects.select_related('project').filter(
        family_permissions_check & (Q(family_id__icontains=query) | Q(display_name__icontains=query))
    )

    families_result = []
    for f in matching_families[:MAX_RESULTS_PER_CATEGORY]:
        title = f.display_name
        families_result.append({
            'title': title[:MAX_STRING_LENGTH],
            'description': ('(%s)' % f.project.name) if f.project else '',
            'href': '/project/'+f.project.deprecated_project_id+'/family/'+f.family_id,
        })

    families_result.sort(key=lambda f: len(f.get('title', '')))

    return families_result

def _get_matching_individuals(user, query):
    """Returns individuals that match the given query string, and that the user can view.

    Args:
        user: Django user
        query: String typed into the awesomebar
    Returns:
        Sorted list of matches where each match is a dictionary of strings
    """
    individual_permissions_check = Q()
    if not user.is_staff:
        projects_can_view = [p.id for p in get_objects_for_user(user, CAN_VIEW, Project)]
        individual_permissions_check = Q(family__project__id__in=projects_can_view)

    matching_individuals = Individual.objects.select_related('family__project').filter(
        individual_permissions_check & (Q(individual_id__icontains=query) | Q(display_name__icontains=query))
    )

    individuals_result = []
    for i in matching_individuals[:MAX_RESULTS_PER_CATEGORY]:
        title = i.display_name
        f = i.family
        individuals_result.append({
            'title': title[:MAX_STRING_LENGTH],
            'description': ('(%s : family %s)' % (f.project.name, f.display_name)) if f.project else '',
            'href': '/project/'+f.project.deprecated_project_id+'/family/'+f.family_id,
        })

    individuals_result.sort(key=lambda i: len(i.get('title', '')))

    return individuals_result

def _get_matching_genes(user, query):
    """Returns genes that match the given query string, and that the user can view.

    Args:
       user: Django user
       query: String typed into the awesomebar
    Returns:
       Sorted list of matches where each match is a dictionary of strings
    """
    result = []
    matching_genes = GencodeGene.objects.filter(Q(gene_id__icontains=query) | Q(gene_name__icontains=query)).order_by(Length('gene_name').asc())
    for g in matching_genes[:MAX_RESULTS_PER_CATEGORY]:
        if query.lower() in g.gene_id.lower():
            title = g.gene_id
            description = g.gene_name
        else:
            title = g.gene_name
            description = g.gene_id

        result.append({
            'title': title,
            'description': '('+description+')' if description else '',
            'href': '/gene/'+g.gene_id,
        })

    return result

@login_required(login_url=API_LOGIN_REQUIRED_URL)
@require_GET
def awesomebar_autocomplete(request):
    """Accepts HTTP GET request with q=.. url arg, and returns suggestions"""

    query = request.GET.get('q')
    if query is None:
        raise ValueError("missing ?q=<prefix> url arg")

    results = collections.OrderedDict()
    if len(query) > 0:
        projects = _get_matching_projects(request.user, query)
        if projects:
            results['projects'] = {'name': 'Projects', 'results': projects}

        families = _get_matching_families(request.user, query)
        if families:
            results['families'] = {'name': 'Families', 'results': families}

        individuals = _get_matching_individuals(request.user, query)
        if individuals:
            results['individuals'] = {'name': 'Individuals', 'results': individuals}

        genes = _get_matching_genes(request.user, query)
        if genes:
            results['genes'] = {'name': 'Genes', 'results': genes}

    return create_json_response({'matches': results})
<commit_msg>make sure title is never null<commit_after>"""API that generates auto-complete suggestions for the search bar in the header of seqr pages"""

import collections
import logging

from django.contrib.auth.decorators import login_required
from django.db.models import Q
from django.db.models.functions import Length

from django.views.decorators.http import require_GET
from guardian.shortcuts import get_objects_for_user

from seqr.views.apis.auth_api import API_LOGIN_REQUIRED_URL
from seqr.views.utils.json_utils import create_json_response
from reference_data.models import GencodeGene
from seqr.models import Project, Family, Individual, CAN_VIEW

logger = logging.getLogger(__name__)

MAX_RESULTS_PER_CATEGORY = 10
MAX_STRING_LENGTH = 100

def _get_matching_projects(user, query):
    """Returns projects that match the given query string, and that the user can view.

    Args:
        user: Django user
        query: String typed into the awesomebar
    Returns:
        Sorted list of matches where each match is a dictionary of strings
    """
    project_permissions_check = Q()
    if not user.is_staff:
        projects_can_view = [p.id for p in get_objects_for_user(user, CAN_VIEW, Project)]
        project_permissions_check = Q(id__in=projects_can_view)

    matching_projects = Project.objects.filter(
        project_permissions_check & (
            Q(deprecated_project_id__icontains=query) |
            Q(name__icontains=query)
        )  # Q(description__icontains=query) | Q(project_category__icontains=query)
    )

    projects_result = []
    for p in matching_projects[:MAX_RESULTS_PER_CATEGORY]:
        title = p.name or p.guid  # TODO make sure all projects & families have a name
        projects_result.append({
            'title': title[:MAX_STRING_LENGTH],
            'description': '',  # '('+p.description+')' if p.description else '',
            'href': '/project/%s/project_page' % p.guid,
        })

    projects_result.sort(key=lambda f: len(f.get('title', '')))

    return projects_result

def _get_matching_families(user, query):
    """Returns families that match the given query string, and that the user can view.

    Args:
        user: Django user
        query: String typed into the awesomebar
    Returns:
        Sorted list of matches where each match is a dictionary of strings
    """
    family_permissions_check = Q()
    if not user.is_staff:
        projects_can_view = [p.id for p in get_objects_for_user(user, CAN_VIEW, Project)]
        family_permissions_check = Q(project__id__in=projects_can_view)

    matching_families = Family.objects.select_related('project').filter(
        family_permissions_check & (Q(family_id__icontains=query) | Q(display_name__icontains=query))
    )

    families_result = []
    for f in matching_families[:MAX_RESULTS_PER_CATEGORY]:
        title = f.display_name or f.family_id or f.guid
        families_result.append({
            'title': title[:MAX_STRING_LENGTH],
            'description': ('(%s)' % f.project.name) if f.project else '',
            'href': '/project/'+f.project.deprecated_project_id+'/family/'+f.family_id,
        })

    families_result.sort(key=lambda f: len(f.get('title', '')))

    return families_result

def _get_matching_individuals(user, query):
    """Returns individuals that match the given query string, and that the user can view.

    Args:
        user: Django user
        query: String typed into the awesomebar
    Returns:
        Sorted list of matches where each match is a dictionary of strings
    """
    individual_permissions_check = Q()
    if not user.is_staff:
        projects_can_view = [p.id for p in get_objects_for_user(user, CAN_VIEW, Project)]
        individual_permissions_check = Q(family__project__id__in=projects_can_view)

    matching_individuals = Individual.objects.select_related('family__project').filter(
        individual_permissions_check & (Q(individual_id__icontains=query) | Q(display_name__icontains=query))
    )

    individuals_result = []
    for i in matching_individuals[:MAX_RESULTS_PER_CATEGORY]:
        title = i.display_name or i.individual_id or i.guid
        f = i.family
        individuals_result.append({
            'title': title[:MAX_STRING_LENGTH],
            'description': ('(%s : family %s)' % (f.project.name, f.display_name)) if f.project else '',
            'href': '/project/'+f.project.deprecated_project_id+'/family/'+f.family_id,
        })

    individuals_result.sort(key=lambda i: len(i.get('title', '')))

    return individuals_result

def _get_matching_genes(user, query):
    """Returns genes that match the given query string, and that the user can view.

    Args:
       user: Django user
       query: String typed into the awesomebar
    Returns:
       Sorted list of matches where each match is a dictionary of strings
    """
    result = []
    matching_genes = GencodeGene.objects.filter(Q(gene_id__icontains=query) | Q(gene_name__icontains=query)).order_by(Length('gene_name').asc())
    for g in matching_genes[:MAX_RESULTS_PER_CATEGORY]:
        if query.lower() in g.gene_id.lower():
            title = g.gene_id
            description = g.gene_name
        else:
            title = g.gene_name
            description = g.gene_id

        result.append({
            'title': title,
            'description': '('+description+')' if description else '',
            'href': '/gene/'+g.gene_id,
        })

    return result

@login_required(login_url=API_LOGIN_REQUIRED_URL)
@require_GET
def awesomebar_autocomplete(request):
    """Accepts HTTP GET request with q=.. url arg, and returns suggestions"""

    query = request.GET.get('q')
    if query is None:
        raise ValueError("missing ?q=<prefix> url arg")

    results = collections.OrderedDict()
    if len(query) > 0:
        projects = _get_matching_projects(request.user, query)
        if projects:
            results['projects'] = {'name': 'Projects', 'results': projects}

        families = _get_matching_families(request.user, query)
        if families:
            results['families'] = {'name': 'Families', 'results': families}

        individuals = _get_matching_individuals(request.user, query)
        if individuals:
            results['individuals'] = {'name': 'Individuals', 'results': individuals}

        genes = _get_matching_genes(request.user, query)
        if genes:
            results['genes'] = {'name': 'Genes', 'results': genes}

    return create_json_response({'matches': results})
<|endoftext|>


<commit_before>import asyncio
import time

import pytest # noqa

from ..cache import staticcache

@staticcache
def fake_func(dale=1, **kwargs):
    import datetime
    return datetime.datetime.now()

def test_decorator_bare():
    first = fake_func()
    second = fake_func()
    assert first == second

    third = fake_func(3)
    assert first != third

    fourth = fake_func(1)
    assert first == fourth

def test_decorator_kwargs():
    """
    Make sure kwargs works since it's a nested dict in the invoked args.
    Just make sure it's hashing.
    """
    first = fake_func(bob=1)
    second = fake_func(bob=1)
    assert first == second

    third = fake_func(3)
    assert first != third

    fourth = fake_func(bob=3)
    assert first != fourth

def test_decorator_async():
    loop = asyncio.new_event_loop()

    @staticcache
    async def asyncfunc(dale=1):
        await asyncio.sleep(1, loop=loop)
        return time.monotonic()

    try:
        asyncio.set_event_loop(loop)

        # make sure this takes a second
        before = time.monotonic()
        res1 = loop.run_until_complete(asyncfunc(3))
        after = time.monotonic()
        assert after - before >= 1

        # this one should cache
        before = time.monotonic()
        res2 = loop.run_until_complete(asyncfunc(3))
        after = time.monotonic()
        assert after - before < .001

        assert res1 == res2

        # use new arg. cache is cold
        before = time.monotonic()
        res3 = loop.run_until_complete(asyncfunc(2))
        after = time.monotonic()
        assert after - before >= 1
        assert res3 != res1

    finally:
        loop.close()
<commit_msg>flake8 :/<commit_after>import asyncio
import time

import pytest # noqa

from ..cache import staticcache

@staticcache
def fake_func(dale=1, **kwargs):
    import datetime
    return datetime.datetime.now()

def test_decorator_bare():
    first = fake_func()
    second = fake_func()
    assert first == second

    third = fake_func(3)
    assert first != third

    fourth = fake_func(1)
    assert first == fourth

def test_decorator_kwargs():
    """
    Make sure kwargs works since it's a nested dict in the invoked args.
    Just make sure it's hashing.
    """
    first = fake_func(bob=1)
    second = fake_func(bob=1)
    assert first == second

    third = fake_func(3)
    assert first != third

    fourth = fake_func(bob=3)
    assert first != fourth

def test_decorator_async():
    loop = asyncio.new_event_loop()

    @staticcache
    async def asyncfunc(dale=1):
        await asyncio.sleep(1, loop=loop)
        return time.monotonic()

    try:
        asyncio.set_event_loop(loop)

        # make sure this takes a second
        before = time.monotonic()
        res1 = loop.run_until_complete(asyncfunc(3))
        after = time.monotonic()
        assert after - before >= 1

        # this one should cache
        before = time.monotonic()
        res2 = loop.run_until_complete(asyncfunc(3))
        after = time.monotonic()
        assert after - before < .001

        assert res1 == res2

        # use new arg. cache is cold
        before = time.monotonic()
        res3 = loop.run_until_complete(asyncfunc(2))
        after = time.monotonic()
        assert after - before >= 1
        assert res3 != res1

    finally:
        loop.close()
<|endoftext|>


<commit_before>#! python
#
# Python Serial Port Extension for Win32, Linux, BSD, Jython
# see __init__.py
#
# This module implements a special URL handler that wraps an other port,
# print the traffic for debugging purposes. With this, it is possible
# to debug the serial port traffic on every application that uses
# serial_for_url.
#
# (C) 2015 Chris Liechti <cliechti@gmx.net>
#
# SPDX-License-Identifier:    BSD-3-Clause
#
# URL format:    spy://port[?option[=value][&option[=value]]]
# options:
# - dev=X   a file or device to write to
# - color   use escape code to colorize output
# - raw     forward raw bytes instead of hexdump
#
# example:
#   redirect output to an other terminal window on Posix (Linux):
#   python -m serial.tools.miniterm spy:///dev/ttyUSB0?dev=/dev/pts/14\&color

import sys
import time

import serial

try:
    import urlparse
except ImportError:
    import urllib.parse as urlparse

def sixteen(data):
    """\
    yield tuples of hex and ASCII display in multiples of 16. Includes a
    space after 8 bytes and (None, None) after 16 bytes and at the end.
    """
    n = 0
    for b in serial.iterbytes(data):
        yield ('{:02X} '.format(ord(b)), b.decode('ascii') if b' ' <= b < b'\x7f' else '.')
        n += 1
        if n == 8:
            yield (' ', ' ')
        elif n >= 16:
            yield (None, None)
            n = 0
    if n > 0:
        while n < 16:
            n += 1
            if n == 8:
                yield (' ', ' ')
            yield ('   ', ' ')
        yield (None, None)

def hexdump(data):
    """yield lines with hexdump of data"""
    values = []
    ascii = []
    offset = 0
    for h, a in sixteen(data):
        if h is None:
            yield (offset, ' '.join([
                    ''.join(values),
                    ''.join(ascii)]))
            del values[:]
            del ascii[:]
            offset += 0x10
        else:
            values.append(h)
            ascii.append(a)

class FormatRaw(object):
    """forward rx and tx data to output"""

    def __init__(self, output, color):
        self.output = output
        self.color = color
        self.rx_color = '\x1b[32m'
        self.tx_color = '\x1b[31m'

    def rx(self, data):
        if self.color:
            self.output.write(self.rx_color)
        self.output.write(data)
        self.output.flush()

    def tx(self, data):
        if self.color:
            self.output.write(self.tx_color)
        self.output.write(data)
        self.output.flush()

    def control(self, name, value):
        pass

class FormatHexdump(object):
    """\
    Create a hex dump of RX ad TX data, show when control lines are read or
    written.

    output example::

        000000.000 FLSH flushInput
        000002.469 RTS  inactive
        000002.773 RTS  active
        000003.001 TX   48 45 4C 4C 4F                                    HELLO
        000003.102 RX   48 45 4C 4C 4F                                    HELLO

    """

    def __init__(self, output, color):
        self.start_time = time.time()
        self.output = output
        self.color = color
        self.rx_color = '\x1b[32m'
        self.tx_color = '\x1b[31m'
        self.control_color = '\x1b[37m'

    def write_line(self, timestamp, label, value, value2=''):
        self.output.write('{:010.3f} {:4} {}{}\n'.format(timestamp, label, value, value2))
        self.output.flush()

    def rx(self, data):
        if self.color:
            self.output.write(self.rx_color)
        if data:
            for offset, row in hexdump(data):
                self.write_line(time.time() - self.start_time, 'RX', '{:04X}  '.format(offset), row)
        else:
            self.write_line(time.time() - self.start_time, 'RX', '<empty>')

    def tx(self, data):
        if self.color:
            self.output.write(self.tx_color)
        for offset, row in hexdump(data):
            self.write_line(time.time() - self.start_time, 'TX', '{:04X}  '.format(offset), row)

    def control(self, name, value):
        if self.color:
            self.output.write(self.control_color)
        self.write_line(time.time() - self.start_time, name, value)

class Serial(serial.Serial):
    """Just inherit the native Serial port implementation and patch the port property."""

    def __init__(self, *args, **kwargs):
        super(Serial, self).__init__(*args, **kwargs)
        self.formatter = None
        self.show_all = False

    @serial.Serial.port.setter
    def port(self, value):
        if value is not None:
            serial.Serial.port.__set__(self, self.fromURL(value))

    def fromURL(self, url):
        """extract host and port from an URL string"""
        parts = urlparse.urlsplit(url)
        if parts.scheme != 'spy':
            raise serial.SerialException('expected a string in the form "spy://port[?option[=value][&option[=value]]]": not starting with spy:// (%r)' % (parts.scheme,))
        # process options now, directly altering self
        formatter = FormatHexdump
        color = False
        output = sys.stderr
        try:
            for option, values in urlparse.parse_qs(parts.query, True).items():
                if option == 'file':
                    output = open(values[0], 'w')
                elif option == 'color':
                    color = True
                elif option == 'raw':
                    formatter = FormatRaw
                elif option == 'all':
                    self.show_all = True
                else:
                    raise ValueError('unknown option: %r' % (option,))
        except ValueError as e:
            raise serial.SerialException('expected a string in the form "spy://port[?option[=value][&option[=value]]]": %s' % e)
        self.formatter = formatter(output, color)
        return ''.join([parts.netloc, parts.path])

    def write(self, tx):
        self.formatter.tx(tx)
        return super(Serial, self).write(tx)

    def read(self, size=1):
        rx = super(Serial, self).read(size)
        if rx or self.show_all:
            self.formatter.rx(rx)
        return rx

    def inWaiting(self):
        n = super(Serial, self).inWaiting()
        if self.show_all:
            self.formatter.control('QRX', 'inWaiting -> {}'.format(n))
        return n

    def flush(self):
        self.formatter.control('QTX', 'flush')
        super(Serial, self).flush()

    def flushInput(self):
        self.formatter.control('QRX', 'flushInput')
        super(Serial, self).flushInput()

    def flushOutput(self):
        self.formatter.control('QTX', 'flushOutput')
        super(Serial, self).flushOutput()

    def sendBreak(self, duration=0.25):
        self.formatter.control('BRK', 'sendBreak {}s'.format(duration))
        super(Serial, self).sendBreak(duration)

    def setBreak(self, level=1):
        self.formatter.control('BRK', 'active' if level else 'inactive')
        super(Serial, self).setBreak(level)

    def setRTS(self, level=1):
        self.formatter.control('RTS', 'active' if level else 'inactive')
        super(Serial, self).setRTS(level)

    def setDTR(self, level=1):
        self.formatter.control('DTR', 'active' if level else 'inactive')
        super(Serial, self).setDTR(level)

    def getCTS(self):
        level = super(Serial, self).getCTS()
        self.formatter.control('CTS', 'active' if level else 'inactive')
        return level

    def getDSR(self):
        level = super(Serial, self).getDSR()
        self.formatter.control('DSR', 'active' if level else 'inactive')
        return level

    def getRI(self):
        level = super(Serial, self).getRI()
        self.formatter.control('RI', 'active' if level else 'inactive')
        return level

    def getCD(self):
        level = super(Serial, self).getCD()
        self.formatter.control('CD', 'active' if level else 'inactive')
        return level

# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
if __name__ == '__main__':
    s = Serial(None)
    s.port = 'spy:///dev/ttyS0'
    print(s)

<commit_msg>spy: tweak output<commit_after>#! python
#
# Python Serial Port Extension for Win32, Linux, BSD, Jython
# see __init__.py
#
# This module implements a special URL handler that wraps an other port,
# print the traffic for debugging purposes. With this, it is possible
# to debug the serial port traffic on every application that uses
# serial_for_url.
#
# (C) 2015 Chris Liechti <cliechti@gmx.net>
#
# SPDX-License-Identifier:    BSD-3-Clause
#
# URL format:    spy://port[?option[=value][&option[=value]]]
# options:
# - dev=X   a file or device to write to
# - color   use escape code to colorize output
# - raw     forward raw bytes instead of hexdump
#
# example:
#   redirect output to an other terminal window on Posix (Linux):
#   python -m serial.tools.miniterm spy:///dev/ttyUSB0?dev=/dev/pts/14\&color

import sys
import time

import serial

try:
    import urlparse
except ImportError:
    import urllib.parse as urlparse

def sixteen(data):
    """\
    yield tuples of hex and ASCII display in multiples of 16. Includes a
    space after 8 bytes and (None, None) after 16 bytes and at the end.
    """
    n = 0
    for b in serial.iterbytes(data):
        yield ('{:02X} '.format(ord(b)), b.decode('ascii') if b' ' <= b < b'\x7f' else '.')
        n += 1
        if n == 8:
            yield (' ', '')
        elif n >= 16:
            yield (None, None)
            n = 0
    if n > 0:
        while n < 16:
            n += 1
            if n == 8:
                yield (' ', '')
            yield ('   ', ' ')
        yield (None, None)

def hexdump(data):
    """yield lines with hexdump of data"""
    values = []
    ascii = []
    offset = 0
    for h, a in sixteen(data):
        if h is None:
            yield (offset, ' '.join([
                    ''.join(values),
                    ''.join(ascii)]))
            del values[:]
            del ascii[:]
            offset += 0x10
        else:
            values.append(h)
            ascii.append(a)

class FormatRaw(object):
    """Forward only RX and TX data to output."""

    def __init__(self, output, color):
        self.output = output
        self.color = color
        self.rx_color = '\x1b[32m'
        self.tx_color = '\x1b[31m'

    def rx(self, data):
        if self.color:
            self.output.write(self.rx_color)
        self.output.write(data)
        self.output.flush()

    def tx(self, data):
        if self.color:
            self.output.write(self.tx_color)
        self.output.write(data)
        self.output.flush()

    def control(self, name, value):
        pass

class FormatHexdump(object):
    """\
    Create a hex dump of RX ad TX data, show when control lines are read or
    written.

    output example::

        000000.000 Q-RX flushInput
        000002.469 RTS  inactive
        000002.773 RTS  active
        000003.001 TX   48 45 4C 4C 4F                                    HELLO
        000003.102 RX   48 45 4C 4C 4F                                    HELLO

    """

    def __init__(self, output, color):
        self.start_time = time.time()
        self.output = output
        self.color = color
        self.rx_color = '\x1b[32m'
        self.tx_color = '\x1b[31m'
        self.control_color = '\x1b[37m'

    def write_line(self, timestamp, label, value, value2=''):
        self.output.write('{:010.3f} {:4} {}{}\n'.format(timestamp, label, value, value2))
        self.output.flush()

    def rx(self, data):
        if self.color:
            self.output.write(self.rx_color)
        if data:
            for offset, row in hexdump(data):
                self.write_line(time.time() - self.start_time, 'RX', '{:04X}  '.format(offset), row)
        else:
            self.write_line(time.time() - self.start_time, 'RX', '<empty>')

    def tx(self, data):
        if self.color:
            self.output.write(self.tx_color)
        for offset, row in hexdump(data):
            self.write_line(time.time() - self.start_time, 'TX', '{:04X}  '.format(offset), row)

    def control(self, name, value):
        if self.color:
            self.output.write(self.control_color)
        self.write_line(time.time() - self.start_time, name, value)

class Serial(serial.Serial):
    """Just inherit the native Serial port implementation and patch the port property."""

    def __init__(self, *args, **kwargs):
        super(Serial, self).__init__(*args, **kwargs)
        self.formatter = None
        self.show_all = False

    @serial.Serial.port.setter
    def port(self, value):
        if value is not None:
            serial.Serial.port.__set__(self, self.fromURL(value))

    def fromURL(self, url):
        """extract host and port from an URL string"""
        parts = urlparse.urlsplit(url)
        if parts.scheme != 'spy':
            raise serial.SerialException('expected a string in the form "spy://port[?option[=value][&option[=value]]]": not starting with spy:// (%r)' % (parts.scheme,))
        # process options now, directly altering self
        formatter = FormatHexdump
        color = False
        output = sys.stderr
        try:
            for option, values in urlparse.parse_qs(parts.query, True).items():
                if option == 'file':
                    output = open(values[0], 'w')
                elif option == 'color':
                    color = True
                elif option == 'raw':
                    formatter = FormatRaw
                elif option == 'all':
                    self.show_all = True
                else:
                    raise ValueError('unknown option: %r' % (option,))
        except ValueError as e:
            raise serial.SerialException('expected a string in the form "spy://port[?option[=value][&option[=value]]]": %s' % e)
        self.formatter = formatter(output, color)
        return ''.join([parts.netloc, parts.path])

    def write(self, tx):
        self.formatter.tx(tx)
        return super(Serial, self).write(tx)

    def read(self, size=1):
        rx = super(Serial, self).read(size)
        if rx or self.show_all:
            self.formatter.rx(rx)
        return rx

    def inWaiting(self):
        n = super(Serial, self).inWaiting()
        if self.show_all:
            self.formatter.control('Q-RX', 'inWaiting -> {}'.format(n))
        return n

    def flush(self):
        self.formatter.control('Q-TX', 'flush')
        super(Serial, self).flush()

    def flushInput(self):
        self.formatter.control('Q-RX', 'flushInput')
        super(Serial, self).flushInput()

    def flushOutput(self):
        self.formatter.control('Q-TX', 'flushOutput')
        super(Serial, self).flushOutput()

    def sendBreak(self, duration=0.25):
        self.formatter.control('BRK', 'sendBreak {}s'.format(duration))
        super(Serial, self).sendBreak(duration)

    def setBreak(self, level=1):
        self.formatter.control('BRK', 'active' if level else 'inactive')
        super(Serial, self).setBreak(level)

    def setRTS(self, level=1):
        self.formatter.control('RTS', 'active' if level else 'inactive')
        super(Serial, self).setRTS(level)

    def setDTR(self, level=1):
        self.formatter.control('DTR', 'active' if level else 'inactive')
        super(Serial, self).setDTR(level)

    def getCTS(self):
        level = super(Serial, self).getCTS()
        self.formatter.control('CTS', 'active' if level else 'inactive')
        return level

    def getDSR(self):
        level = super(Serial, self).getDSR()
        self.formatter.control('DSR', 'active' if level else 'inactive')
        return level

    def getRI(self):
        level = super(Serial, self).getRI()
        self.formatter.control('RI', 'active' if level else 'inactive')
        return level

    def getCD(self):
        level = super(Serial, self).getCD()
        self.formatter.control('CD', 'active' if level else 'inactive')
        return level

# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
if __name__ == '__main__':
    s = Serial(None)
    s.port = 'spy:///dev/ttyS0'
    print(s)

<|endoftext|>