<commit_before># Copyright (c) 2014, The MITRE Corporation. All rights reserved.
# See LICENSE.txt for complete terms.

# builtin
import os
import collections

# external
from lxml import etree

# internal
import sdv.errors as errors
import sdv.utils as utils
from sdv.validators.base import (ValidationError, ValidationResults)

NS_XML_SCHEMA_INSTANCE = "http://www.w3.org/2001/XMLSchema-instance"
NS_XML_SCHEMA = "http://www.w3.org/2001/XMLSchema"
TAG_XS_INCLUDE = "{%s}include" % (NS_XML_SCHEMA)
TAG_XS_IMPORT = "{%s}import" % (NS_XML_SCHEMA)

class XmlSchemaError(ValidationError):
    """Represents an XML Schema validation error.

    Args:
        error: An error returned from ``etree`` XML Schema validation error
            log.

    Attributes:
        message: The XML validation error message.

    """
    def __init__(self, error):
        super(XmlSchemaError, self).__init__()
        self.message = str(error)

    @property
    def line(self):
        """Returns the line number associated with the error."""
        if not self.message:
            return None

        try:
            error = self.message
            tokenized = error.split(":")
            return tokenized[1]
        except:
            return "UNKNOWN"

    def as_dict(self):
        """Returns a dictionary representation.

        Keys:
            * ``'message'``: The error message
            * ``'line'``: The line number associated with the error
        """
        return dict(message=self.message, line=self.line)

    def __unicode__(self):
        return unicode(self.message)

    def __str__(self):
        return unicode(self).encode("utf-8")

class XmlValidationResults(ValidationResults):
    """Results of XML schema validation. Returned from
    :meth:`XmlSchemaValidator.validate`.

    Args:
        is_valid: The validation result.
        errors: A list of strings reported from the XML validation engine.

    Attributes:
        is_valid: ``True`` if the validation was successful and ``False``
            otherwise.

    """
    def __init__(self, is_valid, errors=None):
        super(XmlValidationResults, self).__init__(is_valid)
        self.errors = errors

    @property
    def errors(self):
        """"A list of :class:`XmlSchemaError` validation errors."""
        return self._errors

    @errors.setter
    def errors(self, value):
        if not value:
            self._errors = []
        elif hasattr(value, '__getitem__'):
            self._errors = [XmlSchemaError(x) for x in value]
        else:
            self._errors = [XmlSchemaError(value)]

    def as_dict(self):
        """A dictionary representation of the :class:`.XmlValidationResults`
        instance.

        Keys:
            * ``'result'``: The validation results (``True`` or ``False``)
            * ``'errors'``: A list of validation errors.

        Returns:
            A dictionary representation of an instance of this class.

        """
        d = super(XmlValidationResults, self).as_dict()

        if self.errors:
            d['errors'] = [x.as_dict() for x in self.errors]

        return d

class XmlSchemaValidator(object):
    """Validates XML instance documents.

    Note:
        If validating against a single XML schema document, use
        ``lxml.etree.XMLSchema`` instead.

    Args:
        schema_dir: A directory of schema files used to validate XML instance
            documents.

    Attributes:
        OVERRIDE_SCHEMALOC: Overrides the schemalocation for a given namespace
            that may be discovered when walking `schema_dir`. This does not
            alter the schemalocation of namespaces declared by
            ``xsi:schemalLocation`` attributes if validating via
            ``xsi:schemaLocation``.

    """
    OVERRIDE_SCHEMALOC = {}

    def __init__(self, schema_dir=None):
        self._schemalocs = self._map_schemalocs(schema_dir)

    def _get_includes(self, fp, root):
        """Returns a list of ``xs:include`` targets found within `root`.

        The returned list contains paths to the ``xs:include`` targets. The
        file paths are absolute.

        Note:
            This assumes all includes point to local schemas. Remote schema
            locations will not be parsed correctly!

        Args:
            fp: The file path to `root`. This is used to determine the path
                to the included schema if the include path is relative.
            root: An etree._Element representation of the schema.

        Returns:
            A list of file paths to included schemas.

        """
        xs_includes = root.findall(TAG_XS_INCLUDE)
        dir_ = os.path.dirname(fp)

        includes = []
        for include in xs_includes:
            loc = include.attrib['schemaLocation']

            if os.path.isabs(loc):
                locpath = loc
            else:
                locpath = os.path.abspath(os.path.join(dir_, loc))

            includes.append(locpath)

        return includes

    def _build_include_graph(self, schema_paths):
        """Builds a graph of ``xs:include`` directive sources and targets for
        the schemas contained by the `schema_paths` list.

        Args:
            schema_paths: A list of schema file paths

        Returns:
            A graph representing ``xs:include`` statements found within the
            schemas in `schema_paths`.

        """
        graph = collections.defaultdict(list)

        for fp in schema_paths:
            root = utils.get_etree_root(fp)
            includes = self._get_includes(fp, root)
            graph[fp].extend(includes)

        return graph

    def _is_included(self, graph, fp):
        """Returns ``True`` if the schema at `fp` was included by any other
         schemas in `graph`.

         """
        for includes in graph.itervalues():
            if fp in includes:
                return True

        return False

    def _get_include_root(self, ns, list_schemas):
        """Attempts to determine the "root" schema for a targetNamespace.

        This builds a graph of ``xs:include`` directive sources and targets
        and attempts to find a common base for all includes.

        Note:
            If no schemas in `list_schemas` ``xs:include`` another schema,
            then ``list_schemas[0]`` is returned. This occurs when duplicate
            schemas (or different versions of the same schema that define the
            same namespace) were encountered in the initialization schema
            directory.

        Args:
            ns: The target namespace
            list_schemas: A list of schemas which exist or define the
                target namespace.

        Returns:
            A path to the root schema for the input `ns`.

        """
        graph = self._build_include_graph(list_schemas)

        if all(not(x) for x in graph.itervalues()):
            return list_schemas[0]

        for fp in graph:
            has_ancestors = self._is_included(graph, fp)
            has_children  = len(graph[fp]) > 0

            if has_children and not has_ancestors:
                return fp

        raise errors.XMLSchemaIncludeError(
            "Unable to determine base schema for %s" % ns
        )

    def _process_includes(self, imports):
        """Attempts to resolve cases where multiple schemas declare the same
        ``targetNamespace`` value. This is due to the use of the ``xs:include``
        directive, which can be found in OASIS CIQ schemas along with others.

        This is done by building an ``xs:include`` graph, and returning the
        root of that graph.

        Note:
            This method is flawed! This assumes that the ``xs:include`` graph
            is really a tree, and has a root which can be imported and used
            to validate all instance data which belongs to its namespace.

            A better way may be to programatically combine all "split" schemas
            within a single schema document and map the targetNamespace to that
            combined schema document.

        Args:
            imports: A dictionary of namespaces to a list of schema file
                paths. Most often, this list will have only one file path
                in it.

        Returns:
            A dictionary of schema targetNamespaces to a single schema file
            path.

        """
        processed = {}

        for ns, schemas in imports.iteritems():
            if len(schemas) > 1:
                base_schema = self._get_include_root(ns, schemas)
                processed[ns] = base_schema
            else:
                processed[ns] = schemas[0]

        return processed

    def _walk_schemas(self, schema_dir):
        """Walks the `schema_dir` directory and builds a dictionary of
        schema ``targetNamespace`` values to a list of schema file paths.

        Because multiple schemas can declare the same ``targetNamespace``
        value, the ``value`` portion of the returned dictionary is a ``list``.

        Note:
            This method attempts to resolve issues where the same schema
            exists in two or more locations under `schema_dir` by keeping
            a record of visited target namespaces and filenames. If the same
            filename:targetNS (not file path) pair has been visited already,
            the file is not added to the schemalocation dictionary.

        Returns:
            A dictionary of  schema ``targetNamespace`` values to a list of
            schema file paths.

        """
        seen = []
        schemalocs = collections.defaultdict(list)

        for top, _, files in os.walk(schema_dir):
            for fn in files:
                if not fn.endswith('.xsd'):
                    continue

                fp = os.path.abspath(os.path.join(top, fn))
                target_ns = utils.get_target_ns(fp)

                if (target_ns, fn) in seen:
                    continue

                schemalocs[target_ns].append(fp)
                seen.append((target_ns, fn))

        for ns, loc in self.OVERRIDE_SCHEMALOC.iteritems():
            schemalocs[ns] = [loc]

        return schemalocs

    def _map_schemalocs(self, schema_dir):
        """Walks the `schema_dir` directory and builds a dictionary which maps
        schema targetNamespace values to schema file paths.

        If `schema_dir` is ``None``, this function returns immediately.

        Returns:
            A dictionary mapping schema ``targetNamespace`` values to the
            schema file path.

        Raises:
            .XMlSchemaIncludeError: If an error occurs while processing
                ``xs:include`` directives.

        """
        if not schema_dir:
            return

        schemalocs = self._walk_schemas(schema_dir)
        schemalocs = self._process_includes(schemalocs)

        return schemalocs

    def _parse_schemaloc(self, root):
        """Parses the ``xsi:schemaLocation`` attribute found on `root`.

        Returns:
            A dictionary of namespaces to schema locations.

        Raises:
            .XMLSchemaImportError: If `root` did not contain an
                ``xsi:schemaLocation`` attribute.

        """
        try:
            imports = utils.get_schemaloc_pairs(root)
            return dict(imports)
        except KeyError:
            raise errors.XMLSchemaImportError(
                "Cannot validate using xsi:schemaLocation. The "
                "xsi:schemaLocation attribute was not found on the input "
                "document"
            )

    def _get_required_schemas(self, root):
        """Retrieve all the namespaces and schemalocations needed to validate
        `root`.

        Args:
            root: An etree._Element XML document.

        Returns:
            A dictionary mapping namespaces to schemalocations.

        """
        def _get_schemalocs(node):
            schemalocs = {}
            for ns in node.nsmap.itervalues():
                # Ignore ns lookup failures. Should this raise an Exception?
                with utils.ignored(KeyError):
                    schemalocs[ns] = self._schemalocs[ns]
            return schemalocs

        imports = {}
        for elem in root.iter():
            schemalocs = _get_schemalocs(elem)
            imports.update(schemalocs)

        return imports

    def _build_required_imports(self, doc, schemaloc=False):
        root = utils.get_etree_root(doc)

        if schemaloc:
            return self._parse_schemaloc(root)

        return self._get_required_schemas(root)

    def _build_uber_schema(self, doc, schemaloc=False):
        """Builds a schema which is made up of ``xs:import`` directives for
        each schema required to validate `doc`.

        If schemaloc is ``True``, the ``xsi:schemaLocation`` attribute values
        are used to create the ``xs:import`` directives. If ``False``, the
        initialization schema directory is used.

        Returns:
            An ``etree.XMLSchema`` instance used to validate `doc`.

        Raise:
            .XMLSchemaImportError: If an error occurred while building the
                dictionary of namespace to schemalocation mappings used to
                drive the uber schema creation.

        """
        root = utils.get_etree_root(doc)
        imports = self._build_required_imports(root, schemaloc)

        if not imports:
            raise errors.XMLSchemaImportError(
                "Cannot validate document. Error occurred while determining "
                "schemas required for validation."
            )

        xsd = etree.fromstring(
            """
            <xs:schema
                xmlns:xs="http://www.w3.org/2001/XMLSchema"
                targetNamespace="http://stix.mitre.org/tools/validator"
                elementFormDefault="qualified"
                attributeFormDefault="qualified"/>
            """
        )

        for ns, loc in imports.iteritems():
            loc = loc.replace("\\", "/")
            attrib = dict(namespace=ns, schemaLocation=loc)
            import_ = etree.Element(TAG_XS_IMPORT, attrib=attrib)
            xsd.append(import_)

        return etree.XMLSchema(xsd)

    def validate(self, doc, schemaloc=False):
        """Validates an XML instance document.

        Args:
            doc: An XML instance document. This can be a filename, file-like
                object, ``etree._Element``, or ``etree._ElementTree``.
            schemaloc: If ``True``, the document will be validated using the
                ``xsi:schemaLocation`` attribute found on the instance
                document root.

        Returns:
            An instance of
            :class:`.XmlValidationResults`.

        Raises:
            .ValidationError: If the class was not initialized with a
                schema directory and `schemaloc` is ``False`` or if there are
                any issues parsing `doc`.
            .XMLSchemaIncludeError: If an error occurs while processing the
                schemas required for validation.
            .XMLSchemaIncludeError: If an error occurs while processing
                ``xs:include`` directives.

        """
        if not any((schemaloc, self._schemalocs)):
            raise errors.ValidationError(
                "No schemas to validate against! Try instantiating "
                "XmlValidator with use_schemaloc=True or setting the "
                "schema_dir param in __init__"
            )

        root = utils.get_etree_root(doc)
        xsd = self._build_uber_schema(root, schemaloc)
        is_valid = xsd.validate(root)

        return XmlValidationResults(is_valid, xsd.error_log)

__all__ = [
    'XmlSchemaValidator',
    'XmlValidationResults',
    'XmlSchemaError'
]
<commit_msg>More code cleanup in xml schema module<commit_after># Copyright (c) 2014, The MITRE Corporation. All rights reserved.
# See LICENSE.txt for complete terms.

# builtin
import os
import collections

# external
from lxml import etree

# internal
import sdv.errors as errors
import sdv.utils as utils
from sdv.validators.base import (ValidationError, ValidationResults)

NS_XML_SCHEMA_INSTANCE = "http://www.w3.org/2001/XMLSchema-instance"
NS_XML_SCHEMA = "http://www.w3.org/2001/XMLSchema"
TAG_XS_INCLUDE = "{%s}include" % (NS_XML_SCHEMA)
TAG_XS_IMPORT = "{%s}import" % (NS_XML_SCHEMA)

class XmlSchemaError(ValidationError):
    """Represents an XML Schema validation error.

    Args:
        error: An error returned from ``etree`` XML Schema validation error
            log.

    Attributes:
        message: The XML validation error message.

    """
    def __init__(self, error):
        super(XmlSchemaError, self).__init__()
        self.message = str(error)

    @property
    def line(self):
        """Returns the line number associated with the error."""
        if not self.message:
            return None

        try:
            error = self.message
            tokenized = error.split(":")
            return tokenized[1]
        except:
            return "UNKNOWN"

    def as_dict(self):
        """Returns a dictionary representation.

        Keys:
            * ``'message'``: The error message
            * ``'line'``: The line number associated with the error
        """
        return dict(message=self.message, line=self.line)

    def __unicode__(self):
        return unicode(self.message)

    def __str__(self):
        return unicode(self).encode("utf-8")

class XmlValidationResults(ValidationResults):
    """Results of XML schema validation. Returned from
    :meth:`XmlSchemaValidator.validate`.

    Args:
        is_valid: The validation result.
        errors: A list of strings reported from the XML validation engine.

    Attributes:
        is_valid: ``True`` if the validation was successful and ``False``
            otherwise.

    """
    def __init__(self, is_valid, errors=None):
        super(XmlValidationResults, self).__init__(is_valid)
        self.errors = errors

    @property
    def errors(self):
        """"A list of :class:`XmlSchemaError` validation errors."""
        return self._errors

    @errors.setter
    def errors(self, value):
        if not value:
            self._errors = []
        elif hasattr(value, '__getitem__'):
            self._errors = [XmlSchemaError(x) for x in value]
        else:
            self._errors = [XmlSchemaError(value)]

    def as_dict(self):
        """A dictionary representation of the :class:`.XmlValidationResults`
        instance.

        Keys:
            * ``'result'``: The validation results (``True`` or ``False``)
            * ``'errors'``: A list of validation errors.

        Returns:
            A dictionary representation of an instance of this class.

        """
        d = super(XmlValidationResults, self).as_dict()

        if self.errors:
            d['errors'] = [x.as_dict() for x in self.errors]

        return d

class XmlSchemaValidator(object):
    """Validates XML instance documents.

    Note:
        If validating against a single XML schema document, use
        ``lxml.etree.XMLSchema`` instead.

    Args:
        schema_dir: A directory of schema files used to validate XML instance
            documents.

    Attributes:
        OVERRIDE_SCHEMALOC: Overrides the schemalocation for a given namespace
            that may be discovered when walking `schema_dir`. This does not
            alter the schemalocation of namespaces declared by
            ``xsi:schemalLocation`` attributes if validating via
            ``xsi:schemaLocation``.

    """
    OVERRIDE_SCHEMALOC = {}

    def __init__(self, schema_dir=None):
        self._schemalocs = self._map_schemalocs(schema_dir)

    def _get_includes(self, fp, root):
        """Returns a list of ``xs:include`` targets found within `root`.

        The returned list contains paths to the ``xs:include`` targets. The
        file paths are absolute.

        Note:
            This assumes all includes point to local schemas. Remote schema
            locations will not be parsed correctly!

        Args:
            fp: The file path to `root`. This is used to determine the path
                to the included schema if the include path is relative.
            root: An etree._Element representation of the schema.

        Returns:
            A list of file paths to included schemas.

        """
        xs_includes = root.findall(TAG_XS_INCLUDE)
        dir = os.path.dirname(fp)

        includes = []
        for include in xs_includes:
            loc = include.attrib['schemaLocation']

            # If the path is relative, get the absolute path
            if os.path.isabs(loc):
                locpath = loc
            else:
                locpath = os.path.abspath(os.path.join(dir, loc))

            includes.append(locpath)

        return includes

    def _build_include_graph(self, schema_paths):
        """Builds a graph of ``xs:include`` directive sources and targets for
        the schemas contained by the `schema_paths` list.

        Args:
            schema_paths: A list of schema file paths

        Returns:
            A graph representing ``xs:include`` statements found within the
            schemas in `schema_paths`.

        """
        graph = collections.defaultdict(list)

        for fp in schema_paths:
            root = utils.get_etree_root(fp)
            includes = self._get_includes(fp, root)
            graph[fp].extend(includes)

        return graph

    def _is_included(self, graph, fp):
        """Returns ``True`` if the schema at `fp` was included by any other
        schemas in `graph`.

        """
        return any(fp in includes for includes in graph.itervalues())

    def _get_include_root(self, ns, list_schemas):
        """Attempts to determine the "root" schema for a targetNamespace.

        This builds a graph of ``xs:include`` directive sources and targets
        and attempts to find a common base for all includes.

        Note:
            If no schemas in `list_schemas` ``xs:include`` another schema,
            then ``list_schemas[0]`` is returned. This occurs when duplicate
            schemas (or different versions of the same schema that define the
            same namespace) were encountered in the initialization schema
            directory.

        Args:
            ns: The target namespace
            list_schemas: A list of schemas which exist or define the
                target namespace.

        Returns:
            A path to the root schema for the input `ns`.

        """
        graph = self._build_include_graph(list_schemas)

        if all(not(x) for x in graph.itervalues()):
            return list_schemas[0]

        for fp in graph:
            has_ancestors = self._is_included(graph, fp)
            has_children  = len(graph[fp]) > 0

            if has_children and not has_ancestors:
                return fp

        raise errors.XMLSchemaIncludeError(
            "Unable to determine base schema for %s" % ns
        )

    def _process_includes(self, imports):
        """Attempts to resolve cases where multiple schemas declare the same
        ``targetNamespace`` value. This is due to the use of the ``xs:include``
        directive, which can be found in OASIS CIQ schemas along with others.

        This is done by building an ``xs:include`` graph, and returning the
        root of that graph.

        Note:
            This method is flawed! This assumes that the ``xs:include`` graph
            is really a tree, and has a root which can be imported and used
            to validate all instance data which belongs to its namespace.

            A better way may be to programatically combine all "split" schemas
            within a single schema document and map the targetNamespace to that
            combined schema document.

        Args:
            imports: A dictionary of namespaces to a list of schema file
                paths. Most often, this list will have only one file path
                in it.

        Returns:
            A dictionary of schema targetNamespaces to a single schema file
            path.

        """
        processed = {}

        for ns, schemas in imports.iteritems():
            if len(schemas) > 1:
                base_schema = self._get_include_root(ns, schemas)
                processed[ns] = base_schema
            else:
                processed[ns] = schemas[0]

        return processed

    def _walk_schemas(self, schema_dir):
        """Walks the `schema_dir` directory and builds a dictionary of
        schema ``targetNamespace`` values to a list of schema file paths.

        Because multiple schemas can declare the same ``targetNamespace``
        value, the ``value`` portion of the returned dictionary is a ``list``.

        Note:
            This method attempts to resolve issues where the same schema
            exists in two or more locations under `schema_dir` by keeping
            a record of visited target namespaces and filenames. If the same
            filename:targetNS (not file path) pair has been visited already,
            the file is not added to the schemalocation dictionary.

        Returns:
            A dictionary of  schema ``targetNamespace`` values to a list of
            schema file paths.

        """
        seen = []
        schemalocs = collections.defaultdict(list)

        for top, _, files in os.walk(schema_dir):
            for fn in files:
                if not fn.endswith('.xsd'):
                    continue

                fp = os.path.abspath(os.path.join(top, fn))
                target_ns = utils.get_target_ns(fp)

                if (target_ns, fn) in seen:
                    continue

                schemalocs[target_ns].append(fp)
                seen.append((target_ns, fn))

        for ns, loc in self.OVERRIDE_SCHEMALOC.iteritems():
            schemalocs[ns] = [loc]

        return schemalocs

    def _map_schemalocs(self, schema_dir):
        """Walks the `schema_dir` directory and builds a dictionary which maps
        schema targetNamespace values to schema file paths.

        If `schema_dir` is ``None``, this function returns immediately.

        Returns:
            A dictionary mapping schema ``targetNamespace`` values to the
            schema file path.

        Raises:
            .XMlSchemaIncludeError: If an error occurs while processing
                ``xs:include`` directives.

        """
        if not schema_dir:
            return

        schemalocs = self._walk_schemas(schema_dir)
        schemalocs = self._process_includes(schemalocs)

        return schemalocs

    def _parse_schemaloc(self, root):
        """Parses the ``xsi:schemaLocation`` attribute found on `root`.

        Returns:
            A dictionary of namespaces to schema locations.

        Raises:
            .XMLSchemaImportError: If `root` did not contain an
                ``xsi:schemaLocation`` attribute.

        """
        try:
            imports = utils.get_schemaloc_pairs(root)
            return dict(imports)
        except KeyError:
            raise errors.XMLSchemaImportError(
                "Cannot validate using xsi:schemaLocation. The "
                "xsi:schemaLocation attribute was not found on the input "
                "document"
            )

    def _get_required_schemas(self, root):
        """Retrieve all the namespaces and schemalocations needed to validate
        `root`.

        Args:
            root: An etree._Element XML document.

        Returns:
            A dictionary mapping namespaces to schemalocations.

        """
        def _get_schemalocs(node):
            schemalocs = {}
            for ns in node.nsmap.itervalues():
                # Ignore ns lookup failures. Should this raise an Exception?
                with utils.ignored(KeyError):
                    schemalocs[ns] = self._schemalocs[ns]
            return schemalocs

        imports = {}
        for elem in root.iter():
            schemalocs = _get_schemalocs(elem)
            imports.update(schemalocs)

        return imports

    def _build_required_imports(self, doc, schemaloc=False):
        root = utils.get_etree_root(doc)

        if schemaloc:
            return self._parse_schemaloc(root)

        return self._get_required_schemas(root)

    def _build_uber_schema(self, doc, schemaloc=False):
        """Builds a schema which is made up of ``xs:import`` directives for
        each schema required to validate `doc`.

        If schemaloc is ``True``, the ``xsi:schemaLocation`` attribute values
        are used to create the ``xs:import`` directives. If ``False``, the
        initialization schema directory is used.

        Returns:
            An ``etree.XMLSchema`` instance used to validate `doc`.

        Raise:
            .XMLSchemaImportError: If an error occurred while building the
                dictionary of namespace to schemalocation mappings used to
                drive the uber schema creation.

        """
        root = utils.get_etree_root(doc)
        imports = self._build_required_imports(root, schemaloc)

        if not imports:
            raise errors.XMLSchemaImportError(
                "Cannot validate document. Error occurred while determining "
                "schemas required for validation."
            )

        xsd = etree.fromstring(
            """
            <xs:schema
                xmlns:xs="http://www.w3.org/2001/XMLSchema"
                targetNamespace="http://stix.mitre.org/tools/validator"
                elementFormDefault="qualified"
                attributeFormDefault="qualified"/>
            """
        )

        for ns, loc in imports.iteritems():
            loc = loc.replace("\\", "/")
            attrib = dict(namespace=ns, schemaLocation=loc)
            import_ = etree.Element(TAG_XS_IMPORT, attrib=attrib)
            xsd.append(import_)

        return etree.XMLSchema(xsd)

    def validate(self, doc, schemaloc=False):
        """Validates an XML instance document.

        Args:
            doc: An XML instance document. This can be a filename, file-like
                object, ``etree._Element``, or ``etree._ElementTree``.
            schemaloc: If ``True``, the document will be validated using the
                ``xsi:schemaLocation`` attribute found on the instance
                document root.

        Returns:
            An instance of
            :class:`.XmlValidationResults`.

        Raises:
            .ValidationError: If the class was not initialized with a
                schema directory and `schemaloc` is ``False`` or if there are
                any issues parsing `doc`.
            .XMLSchemaIncludeError: If an error occurs while processing the
                schemas required for validation.
            .XMLSchemaIncludeError: If an error occurs while processing
                ``xs:include`` directives.

        """
        if not any((schemaloc, self._schemalocs)):
            raise errors.ValidationError(
                "No schemas to validate against! Try instantiating "
                "XmlValidator with use_schemaloc=True or setting the "
                "schema_dir param in __init__"
            )

        root = utils.get_etree_root(doc)
        xsd = self._build_uber_schema(root, schemaloc)
        is_valid = xsd.validate(root)

        return XmlValidationResults(is_valid, xsd.error_log)

__all__ = [
    'XmlSchemaValidator',
    'XmlValidationResults',
    'XmlSchemaError'
]
<|endoftext|>


<commit_before>"""
Functions for explaining classifiers that use Image data.
"""
import copy
from functools import partial

import numpy as np
import sklearn
import sklearn.preprocessing
from sklearn.utils import check_random_state
from skimage.color import gray2rgb
from tqdm.auto import tqdm

from . import lime_base
from .wrappers.scikit_image import SegmentationAlgorithm

class ImageExplanation(object):
    def __init__(self, image, segments):
        """Init function.

        Args:
            image: 3d numpy array
            segments: 2d numpy array, with the output from skimage.segmentation
        """
        self.image = image
        self.segments = segments
        self.intercept = {}
        self.local_exp = {}
        self.local_pred = None

    def get_image_and_mask(self, label, positive_only=True, negative_only=False, hide_rest=False,
                           num_features=5, min_weight=0.):
        """Init function.

        Args:
            label: label to explain
            positive_only: if True, only take superpixels that positively contribute to
                the prediction of the label.
            negative_only: if True, only take superpixels that negatively contribute to
                the prediction of the label. If false, and so is positive_only, then both
                negativey and positively contributions will be taken.
                Both can't be True at the same time
            hide_rest: if True, make the non-explanation part of the return
                image gray
            num_features: number of superpixels to include in explanation
            min_weight: minimum weight of the superpixels to include in explanation

        Returns:
            (image, mask), where image is a 3d numpy array and mask is a 2d
            numpy array that can be used with
            skimage.segmentation.mark_boundaries
        """
        if label not in self.local_exp:
            raise KeyError('Label not in explanation')
        if positive_only & negative_only:
            raise ValueError("Positive_only and negative_only cannot be true at the same time.")
        segments = self.segments
        image = self.image
        exp = self.local_exp[label]
        mask = np.zeros(segments.shape, segments.dtype)
        if hide_rest:
            temp = np.zeros(self.image.shape)
        else:
            temp = self.image.copy()
        if positive_only:
            fs = [x[0] for x in exp
                  if x[1] > 0 and x[1] > min_weight][:num_features]
        if negative_only:
            fs = [x[0] for x in exp
                  if x[1] < 0 and abs(x[1]) > min_weight][:num_features]
        if positive_only or negative_only:
            for f in fs:
                temp[segments == f] = image[segments == f].copy()
                mask[segments == f] = 1
            return temp, mask
        else:
            for f, w in exp[:num_features]:
                if np.abs(w) < min_weight:
                    continue
                c = 0 if w < 0 else 1
                mask[segments == f] = -1 if w < 0 else 1
                temp[segments == f] = image[segments == f].copy()
                temp[segments == f, c] = np.max(image)
            return temp, mask

class LimeImageExplainer(object):
    """Explains predictions on Image (i.e. matrix) data.
    For numerical features, perturb them by sampling from a Normal(0,1) and
    doing the inverse operation of mean-centering and scaling, according to the
    means and stds in the training data. For categorical features, perturb by
    sampling according to the training distribution, and making a binary
    feature that is 1 when the value is the same as the instance being
    explained."""

    def __init__(self, kernel_width=.25, kernel=None, verbose=False,
                 feature_selection='auto', random_state=None):
        """Init function.

        Args:
            kernel_width: kernel width for the exponential kernel.
            If None, defaults to sqrt(number of columns) * 0.75.
            kernel: similarity kernel that takes euclidean distances and kernel
                width as input and outputs weights in (0,1). If None, defaults to
                an exponential kernel.
            verbose: if true, print local prediction values from linear model
            feature_selection: feature selection method. can be
                'forward_selection', 'lasso_path', 'none' or 'auto'.
                See function 'explain_instance_with_data' in lime_base.py for
                details on what each of the options does.
            random_state: an integer or numpy.RandomState that will be used to
                generate random numbers. If None, the random state will be
                initialized using the internal numpy seed.
        """
        kernel_width = float(kernel_width)

        if kernel is None:
            def kernel(d, kernel_width):
                return np.sqrt(np.exp(-(d ** 2) / kernel_width ** 2))

        kernel_fn = partial(kernel, kernel_width=kernel_width)

        self.random_state = check_random_state(random_state)
        self.feature_selection = feature_selection
        self.base = lime_base.LimeBase(kernel_fn, verbose, random_state=self.random_state)

    def explain_instance(self, image, classifier_fn, labels=(1,),
                         hide_color=None,
                         top_labels=5, num_features=100000, num_samples=1000,
                         batch_size=10,
                         segmentation_fn=None,
                         distance_metric='cosine',
                         model_regressor=None,
                         random_seed=None):
        """Generates explanations for a prediction.

        First, we generate neighborhood data by randomly perturbing features
        from the instance (see __data_inverse). We then learn locally weighted
        linear models on this neighborhood data to explain each of the classes
        in an interpretable way (see lime_base.py).

        Args:
            image: 3 dimension RGB image. If this is only two dimensional,
                we will assume it's a grayscale image and call gray2rgb.
            classifier_fn: classifier prediction probability function, which
                takes a numpy array and outputs prediction probabilities.  For
                ScikitClassifiers , this is classifier.predict_proba.
            labels: iterable with labels to be explained.
            hide_color: TODO
            top_labels: if not None, ignore labels and produce explanations for
                the K labels with highest prediction probabilities, where K is
                this parameter.
            num_features: maximum number of features present in explanation
            num_samples: size of the neighborhood to learn the linear model
            batch_size: TODO
            distance_metric: the distance metric to use for weights.
            model_regressor: sklearn regressor to use in explanation. Defaults
            to Ridge regression in LimeBase. Must have model_regressor.coef_
            and 'sample_weight' as a parameter to model_regressor.fit()
            segmentation_fn: SegmentationAlgorithm, wrapped skimage
            segmentation function
            random_seed: integer used as random seed for the segmentation
                algorithm. If None, a random integer, between 0 and 1000,
                will be generated using the internal random number generator.

        Returns:
            An ImageExplanation object (see lime_image.py) with the corresponding
            explanations.
        """
        if len(image.shape) == 2:
            image = gray2rgb(image)
        if random_seed is None:
            random_seed = self.random_state.randint(0, high=1000)

        if segmentation_fn is None:
            segmentation_fn = SegmentationAlgorithm('quickshift', kernel_size=4,
                                                    max_dist=200, ratio=0.2,
                                                    random_seed=random_seed)
        try:
            segments = segmentation_fn(image)
        except ValueError as e:
            raise e

        fudged_image = image.copy()
        if hide_color is None:
            for x in np.unique(segments):
                fudged_image[segments == x] = (
                    np.mean(image[segments == x][:, 0]),
                    np.mean(image[segments == x][:, 1]),
                    np.mean(image[segments == x][:, 2]))
        else:
            fudged_image[:] = hide_color

        top = labels

        data, labels = self.data_labels(image, fudged_image, segments,
                                        classifier_fn, num_samples,
                                        batch_size=batch_size)

        distances = sklearn.metrics.pairwise_distances(
            data,
            data[0].reshape(1, -1),
            metric=distance_metric
        ).ravel()

        ret_exp = ImageExplanation(image, segments)
        if top_labels:
            top = np.argsort(labels[0])[-top_labels:]
            ret_exp.top_labels = list(top)
            ret_exp.top_labels.reverse()
        for label in top:
            (ret_exp.intercept[label],
             ret_exp.local_exp[label],
             ret_exp.score, ret_exp.local_pred) = self.base.explain_instance_with_data(
                data, labels, distances, label, num_features,
                model_regressor=model_regressor,
                feature_selection=self.feature_selection)
        return ret_exp

    def data_labels(self,
                    image,
                    fudged_image,
                    segments,
                    classifier_fn,
                    num_samples,
                    batch_size=10):
        """Generates images and predictions in the neighborhood of this image.

        Args:
            image: 3d numpy array, the image
            fudged_image: 3d numpy array, image to replace original image when
                superpixel is turned off
            segments: segmentation of the image
            classifier_fn: function that takes a list of images and returns a
                matrix of prediction probabilities
            num_samples: size of the neighborhood to learn the linear model
            batch_size: classifier_fn will be called on batches of this size.

        Returns:
            A tuple (data, labels), where:
                data: dense num_samples * num_superpixels
                labels: prediction probabilities matrix
        """
        n_features = np.unique(segments).shape[0]
        data = self.random_state.randint(0, 2, num_samples * n_features)\
            .reshape((num_samples, n_features))
        labels = []
        data[0, :] = 1
        imgs = []
        for row in tqdm(data):
            temp = copy.deepcopy(image)
            zeros = np.where(row == 0)[0]
            mask = np.zeros(segments.shape).astype(bool)
            for z in zeros:
                mask[segments == z] = True
            temp[mask] = fudged_image[mask]
            imgs.append(temp)
            if len(imgs) == batch_size:
                preds = classifier_fn(np.array(imgs))
                labels.extend(preds)
                imgs = []
        if len(imgs) > 0:
            preds = classifier_fn(np.array(imgs))
            labels.extend(preds)
        return data, np.array(labels)
<commit_msg>made tqdm optional<commit_after>"""
Functions for explaining classifiers that use Image data.
"""
import copy
from functools import partial

import numpy as np
import sklearn
import sklearn.preprocessing
from sklearn.utils import check_random_state
from skimage.color import gray2rgb
from tqdm.auto import tqdm

from . import lime_base
from .wrappers.scikit_image import SegmentationAlgorithm

class ImageExplanation(object):
    def __init__(self, image, segments):
        """Init function.

        Args:
            image: 3d numpy array
            segments: 2d numpy array, with the output from skimage.segmentation
        """
        self.image = image
        self.segments = segments
        self.intercept = {}
        self.local_exp = {}
        self.local_pred = None

    def get_image_and_mask(self, label, positive_only=True, negative_only=False, hide_rest=False,
                           num_features=5, min_weight=0.):
        """Init function.

        Args:
            label: label to explain
            positive_only: if True, only take superpixels that positively contribute to
                the prediction of the label.
            negative_only: if True, only take superpixels that negatively contribute to
                the prediction of the label. If false, and so is positive_only, then both
                negativey and positively contributions will be taken.
                Both can't be True at the same time
            hide_rest: if True, make the non-explanation part of the return
                image gray
            num_features: number of superpixels to include in explanation
            min_weight: minimum weight of the superpixels to include in explanation

        Returns:
            (image, mask), where image is a 3d numpy array and mask is a 2d
            numpy array that can be used with
            skimage.segmentation.mark_boundaries
        """
        if label not in self.local_exp:
            raise KeyError('Label not in explanation')
        if positive_only & negative_only:
            raise ValueError("Positive_only and negative_only cannot be true at the same time.")
        segments = self.segments
        image = self.image
        exp = self.local_exp[label]
        mask = np.zeros(segments.shape, segments.dtype)
        if hide_rest:
            temp = np.zeros(self.image.shape)
        else:
            temp = self.image.copy()
        if positive_only:
            fs = [x[0] for x in exp
                  if x[1] > 0 and x[1] > min_weight][:num_features]
        if negative_only:
            fs = [x[0] for x in exp
                  if x[1] < 0 and abs(x[1]) > min_weight][:num_features]
        if positive_only or negative_only:
            for f in fs:
                temp[segments == f] = image[segments == f].copy()
                mask[segments == f] = 1
            return temp, mask
        else:
            for f, w in exp[:num_features]:
                if np.abs(w) < min_weight:
                    continue
                c = 0 if w < 0 else 1
                mask[segments == f] = -1 if w < 0 else 1
                temp[segments == f] = image[segments == f].copy()
                temp[segments == f, c] = np.max(image)
            return temp, mask

class LimeImageExplainer(object):
    """Explains predictions on Image (i.e. matrix) data.
    For numerical features, perturb them by sampling from a Normal(0,1) and
    doing the inverse operation of mean-centering and scaling, according to the
    means and stds in the training data. For categorical features, perturb by
    sampling according to the training distribution, and making a binary
    feature that is 1 when the value is the same as the instance being
    explained."""

    def __init__(self, kernel_width=.25, kernel=None, verbose=False,
                 feature_selection='auto', random_state=None):
        """Init function.

        Args:
            kernel_width: kernel width for the exponential kernel.
            If None, defaults to sqrt(number of columns) * 0.75.
            kernel: similarity kernel that takes euclidean distances and kernel
                width as input and outputs weights in (0,1). If None, defaults to
                an exponential kernel.
            verbose: if true, print local prediction values from linear model
            feature_selection: feature selection method. can be
                'forward_selection', 'lasso_path', 'none' or 'auto'.
                See function 'explain_instance_with_data' in lime_base.py for
                details on what each of the options does.
            random_state: an integer or numpy.RandomState that will be used to
                generate random numbers. If None, the random state will be
                initialized using the internal numpy seed.
        """
        kernel_width = float(kernel_width)

        if kernel is None:
            def kernel(d, kernel_width):
                return np.sqrt(np.exp(-(d ** 2) / kernel_width ** 2))

        kernel_fn = partial(kernel, kernel_width=kernel_width)

        self.random_state = check_random_state(random_state)
        self.feature_selection = feature_selection
        self.base = lime_base.LimeBase(kernel_fn, verbose, random_state=self.random_state)

    def explain_instance(self, image, classifier_fn, labels=(1,),
                         hide_color=None,
                         top_labels=5, num_features=100000, num_samples=1000,
                         batch_size=10,
                         segmentation_fn=None,
                         distance_metric='cosine',
                         model_regressor=None,
                         random_seed=None,
                         progress_bar=True):
        """Generates explanations for a prediction.

        First, we generate neighborhood data by randomly perturbing features
        from the instance (see __data_inverse). We then learn locally weighted
        linear models on this neighborhood data to explain each of the classes
        in an interpretable way (see lime_base.py).

        Args:
            image: 3 dimension RGB image. If this is only two dimensional,
                we will assume it's a grayscale image and call gray2rgb.
            classifier_fn: classifier prediction probability function, which
                takes a numpy array and outputs prediction probabilities.  For
                ScikitClassifiers , this is classifier.predict_proba.
            labels: iterable with labels to be explained.
            hide_color: TODO
            top_labels: if not None, ignore labels and produce explanations for
                the K labels with highest prediction probabilities, where K is
                this parameter.
            num_features: maximum number of features present in explanation
            num_samples: size of the neighborhood to learn the linear model
            batch_size: TODO
            distance_metric: the distance metric to use for weights.
            model_regressor: sklearn regressor to use in explanation. Defaults
            to Ridge regression in LimeBase. Must have model_regressor.coef_
            and 'sample_weight' as a parameter to model_regressor.fit()
            segmentation_fn: SegmentationAlgorithm, wrapped skimage
            segmentation function
            random_seed: integer used as random seed for the segmentation
                algorithm. If None, a random integer, between 0 and 1000,
                will be generated using the internal random number generator.
            progress_bar: if True, show tqdm progress bar.

        Returns:
            An ImageExplanation object (see lime_image.py) with the corresponding
            explanations.
        """
        if len(image.shape) == 2:
            image = gray2rgb(image)
        if random_seed is None:
            random_seed = self.random_state.randint(0, high=1000)

        if segmentation_fn is None:
            segmentation_fn = SegmentationAlgorithm('quickshift', kernel_size=4,
                                                    max_dist=200, ratio=0.2,
                                                    random_seed=random_seed)
        try:
            segments = segmentation_fn(image)
        except ValueError as e:
            raise e

        fudged_image = image.copy()
        if hide_color is None:
            for x in np.unique(segments):
                fudged_image[segments == x] = (
                    np.mean(image[segments == x][:, 0]),
                    np.mean(image[segments == x][:, 1]),
                    np.mean(image[segments == x][:, 2]))
        else:
            fudged_image[:] = hide_color

        top = labels

        data, labels = self.data_labels(image, fudged_image, segments,
                                        classifier_fn, num_samples,
                                        batch_size=batch_size,
                                        progress_bar=progress_bar)

        distances = sklearn.metrics.pairwise_distances(
            data,
            data[0].reshape(1, -1),
            metric=distance_metric
        ).ravel()

        ret_exp = ImageExplanation(image, segments)
        if top_labels:
            top = np.argsort(labels[0])[-top_labels:]
            ret_exp.top_labels = list(top)
            ret_exp.top_labels.reverse()
        for label in top:
            (ret_exp.intercept[label],
             ret_exp.local_exp[label],
             ret_exp.score, ret_exp.local_pred) = self.base.explain_instance_with_data(
                data, labels, distances, label, num_features,
                model_regressor=model_regressor,
                feature_selection=self.feature_selection)
        return ret_exp

    def data_labels(self,
                    image,
                    fudged_image,
                    segments,
                    classifier_fn,
                    num_samples,
                    batch_size=10,
                    progress_bar=True):
        """Generates images and predictions in the neighborhood of this image.

        Args:
            image: 3d numpy array, the image
            fudged_image: 3d numpy array, image to replace original image when
                superpixel is turned off
            segments: segmentation of the image
            classifier_fn: function that takes a list of images and returns a
                matrix of prediction probabilities
            num_samples: size of the neighborhood to learn the linear model
            batch_size: classifier_fn will be called on batches of this size.
            progress_bar: if True, show tqdm progress bar.

        Returns:
            A tuple (data, labels), where:
                data: dense num_samples * num_superpixels
                labels: prediction probabilities matrix
        """
        n_features = np.unique(segments).shape[0]
        data = self.random_state.randint(0, 2, num_samples * n_features)\
            .reshape((num_samples, n_features))
        labels = []
        data[0, :] = 1
        imgs = []
        if progress_bar:
            data = tqdm(data)
        for row in data:
            temp = copy.deepcopy(image)
            zeros = np.where(row == 0)[0]
            mask = np.zeros(segments.shape).astype(bool)
            for z in zeros:
                mask[segments == z] = True
            temp[mask] = fudged_image[mask]
            imgs.append(temp)
            if len(imgs) == batch_size:
                preds = classifier_fn(np.array(imgs))
                labels.extend(preds)
                imgs = []
        if len(imgs) > 0:
            preds = classifier_fn(np.array(imgs))
            labels.extend(preds)
        return data, np.array(labels)
<|endoftext|>


<commit_before>import os
import time
import numpy as np
from math import sqrt

class DANN(object):

    def __init__(self, learning_rate=0.05, hidden_layer_size=25, lambda_adapt=1., maxiter=200,
                 epsilon_init=None, seed=12342, verbose=False):
        """
        Domain Adversarial Neural Network for classification

        option "learning_rate" is the learning rate of the neural network.
        option "hidden_layer_size" is the hidden layer size.
        option "lambda_adapt" weights the domain adaptation regularization term.
        option "hidden_layer_size" is the hidden layer size.
        Option "maxiter" number of training iterations.
        Option "epsilon_init" is a term used for initialization.
                if "None" the weight matrices are weighted by 6/(sqrt(r+c))
                (where r and c are the dimensions of the weight matrix)
                .
        Option "seed" is the seed of the random number generator.
        """

        self.hidden_layer_size = hidden_layer_size
        self.maxiter = maxiter
        self.lambda_adapt = lambda_adapt
        self.epsilon_init = epsilon_init
        self.learning_rate = learning_rate
        self.seed = seed
        self.verbose = verbose

    def sigmoid(self, z):
        """
        Sigmoid function.

        """
        return 1. / (1. + np.exp(-z))

    def softmax(self, z):
        """
        Softmax function.

        """
        v = np.exp(z)
        return v / np.sum(v,axis=0)

    def random_init(self, l_in, l_out):
        """
        This method is used to initialize the weight matrices of the DA neural network

        """
        if self.epsilon_init is not None:
            epsilon = self.epsilon_init
        else:
            epsilon = sqrt(6.0 / (l_in + l_out))

        return epsilon * (2 * np.random.rand(l_out, l_in) - 1.0)

    def fit(self, X, Y, X_adapt, X_valid=None, Y_valid=None, do_random_init=True):
        """
            Trains the domain adversarial neural network until it reaches a total number of
            iterations of "self.maxiter" since it was initialize.
            inputs:
                  X : Source data matrix
                  Y : Source labels
                  X_adapt : Target data matrix
                  (X_valid, Y_valid) : validation set used for early stopping.
                  do_random_init : A boolean indicating whether to use random initialization or not.
        """

        nb_examples, nb_features = np.shape(X)
        nb_labels = len(set(Y))
        nb_examples_adapt, _ = np.shape(X_adapt)

        if self.verbose:
            print(self.__dict__)

        np.random.seed(self.seed)

        if do_random_init:
            W = self.random_init(nb_features, self.hidden_layer_size)
            V = self.random_init(self.hidden_layer_size, nb_labels)
            b = np.zeros(self.hidden_layer_size)
            c = np.zeros(nb_labels)
            w = np.zeros(self.hidden_layer_size)
            d = 0.
        else:
            W, V, b, c, w, d = self.W, self.V, self.b, self.c, self.w, self.d

        best_valid_risk = 2.0
        continue_until = 30

        for t in range(self.maxiter):
            for i in range(nb_examples):
                x_t, y_t = X[i,:], Y[i]

                hidden_layer = self.sigmoid( np.dot(W, x_t) + b )
                output_layer = self.softmax( np.dot(V, hidden_layer) + c  )

                y_hot = np.zeros(nb_labels)
                y_hot[y_t] = 1.0

                delta_c = output_layer - y_hot
                delta_V = np.dot(delta_c.reshape(-1,1), hidden_layer.reshape(1,-1))
                delta_b = np.dot(V.T, delta_c) * hidden_layer * (1.-hidden_layer)
                delta_W = np.dot(delta_b.reshape(-1,1), x_t.reshape(1,-1))

                if self.lambda_adapt == 0. :
                    delta_w, delta_d = 0., 0.
                else:
                    # add domain adaptation regularizer from current domain
                    gho_x_t = self.sigmoid( np.dot(w.T,hidden_layer) + d)

                    delta_d = self.lambda_adapt * (1. - gho_x_t)
                    delta_w = delta_d * hidden_layer
                    tmp = delta_d * w * hidden_layer * (1.-hidden_layer)

                    delta_b += tmp
                    delta_W += tmp.reshape(-1,1) * x_t.reshape(1,-1)

                    # add domain adaptation regularizer from current domain
                    t_2 = np.random.randint(nb_examples_adapt)
                    i_2 = t_2 % nb_examples_adapt
                    x_t_2 = X_adapt[i_2,:]
                    hidden_layer_2 = self.sigmoid( np.dot(W, x_t_2) + b)
                    gho_x_t_2 = self.sigmoid( np.dot(w.T,hidden_layer_2) + d)

                    delta_d -= self.lambda_adapt * gho_x_t_2
                    delta_w -= self.lambda_adapt * gho_x_t_2 * hidden_layer_2

                    tmp =- self.lambda_adapt * gho_x_t_2 * w * hidden_layer_2 * (1. - hidden_layer_2)

                    delta_b += tmp
                    delta_W += tmp.reshape(-1,1) * x_t_2.reshape(1,-1)

                W -= delta_W * self.learning_rate
                b -= delta_b * self.learning_rate

                V -= delta_V * self.learning_rate
                c -= delta_c * self.learning_rate

                w += delta_w * self.learning_rate
                d += delta_d * self.learning_rate
            # END for i in range(nb_examples)

            self.W, self.V, self.b, self.c, self.w, self.d = W, V, b, c, w, d

            # early stopping
            if X_valid is not None:
                valid_pred = self.predict(X_valid)
                valid_risk = np.mean( valid_pred != Y_valid )
                if valid_risk <= best_valid_risk:
                    if self.verbose:
                        print('[best valid risk so far] %f (iter %d)' % (valid_risk, t))
                    best_valid_risk = valid_risk
                    best_weights = (W.copy(), V.copy(), b.copy(), c.copy())
                    best_t = t
                    continue_until = max(continue_until, int(1.5*t))
                elif t > continue_until:
                    if self.verbose:
                        print('[early stop] iter %d' % t)
                    break
        # END for t in range(self.maxiter)

        if X_valid is not None:
            self.W, self.V, self.b, self.c = best_weights
            self.nb_iter = best_t
            self.valid_risk = best_valid_risk
        else:
            self.nb_iter = self.maxiter
            self.valid_risk = 2.

        if self.verbose:
            print('[final valid risk] %f (iter %d)' % (valid_risk, t))

    def forward(self, X):
        """
         Compute and return the network outputs for X, i.e., a 2D array of size len(X) by len(set(Y)).
         the ith row of the array contains output probabilities for each class for the ith example.

        """
        hidden_layer = self.sigmoid( np.dot(self.W, X.T) + self.b[:,np.newaxis] )
        output_layer = self.softmax( np.dot(self.V, hidden_layer) + self.c[:,np.newaxis]  )
        return output_layer

    def predict(self, X):
        """
         Compute and return the label predictions for X, i.e., a 1D array of size len(X).
         the ith row of the array contains the predicted class for the ith example .
        """
        output_layer = self.forward(X)
        return np.argmax(output_layer, 0)

<commit_msg>Minor fix: Main/other domain in comment<commit_after>import os
import time
import numpy as np
from math import sqrt

class DANN(object):

    def __init__(self, learning_rate=0.05, hidden_layer_size=25, lambda_adapt=1., maxiter=200,
                 epsilon_init=None, seed=12342, verbose=False):
        """
        Domain Adversarial Neural Network for classification

        option "learning_rate" is the learning rate of the neural network.
        option "hidden_layer_size" is the hidden layer size.
        option "lambda_adapt" weights the domain adaptation regularization term.
        option "hidden_layer_size" is the hidden layer size.
        Option "maxiter" number of training iterations.
        Option "epsilon_init" is a term used for initialization.
                if "None" the weight matrices are weighted by 6/(sqrt(r+c))
                (where r and c are the dimensions of the weight matrix)
                .
        Option "seed" is the seed of the random number generator.
        """

        self.hidden_layer_size = hidden_layer_size
        self.maxiter = maxiter
        self.lambda_adapt = lambda_adapt
        self.epsilon_init = epsilon_init
        self.learning_rate = learning_rate
        self.seed = seed
        self.verbose = verbose

    def sigmoid(self, z):
        """
        Sigmoid function.

        """
        return 1. / (1. + np.exp(-z))

    def softmax(self, z):
        """
        Softmax function.

        """
        v = np.exp(z)
        return v / np.sum(v,axis=0)

    def random_init(self, l_in, l_out):
        """
        This method is used to initialize the weight matrices of the DA neural network

        """
        if self.epsilon_init is not None:
            epsilon = self.epsilon_init
        else:
            epsilon = sqrt(6.0 / (l_in + l_out))

        return epsilon * (2 * np.random.rand(l_out, l_in) - 1.0)

    def fit(self, X, Y, X_adapt, X_valid=None, Y_valid=None, do_random_init=True):
        """
            Trains the domain adversarial neural network until it reaches a total number of
            iterations of "self.maxiter" since it was initialize.
            inputs:
                  X : Source data matrix
                  Y : Source labels
                  X_adapt : Target data matrix
                  (X_valid, Y_valid) : validation set used for early stopping.
                  do_random_init : A boolean indicating whether to use random initialization or not.
        """

        nb_examples, nb_features = np.shape(X)
        nb_labels = len(set(Y))
        nb_examples_adapt, _ = np.shape(X_adapt)

        if self.verbose:
            print(self.__dict__)

        np.random.seed(self.seed)

        if do_random_init:
            W = self.random_init(nb_features, self.hidden_layer_size)
            V = self.random_init(self.hidden_layer_size, nb_labels)
            b = np.zeros(self.hidden_layer_size)
            c = np.zeros(nb_labels)
            w = np.zeros(self.hidden_layer_size)
            d = 0.
        else:
            W, V, b, c, w, d = self.W, self.V, self.b, self.c, self.w, self.d

        best_valid_risk = 2.0
        continue_until = 30

        for t in range(self.maxiter):
            for i in range(nb_examples):
                x_t, y_t = X[i,:], Y[i]

                hidden_layer = self.sigmoid( np.dot(W, x_t) + b )
                output_layer = self.softmax( np.dot(V, hidden_layer) + c  )

                y_hot = np.zeros(nb_labels)
                y_hot[y_t] = 1.0

                delta_c = output_layer - y_hot
                delta_V = np.dot(delta_c.reshape(-1,1), hidden_layer.reshape(1,-1))
                delta_b = np.dot(V.T, delta_c) * hidden_layer * (1.-hidden_layer)
                delta_W = np.dot(delta_b.reshape(-1,1), x_t.reshape(1,-1))

                if self.lambda_adapt == 0. :
                    delta_w, delta_d = 0., 0.
                else:
                    # add domain adaptation regularizer from current domain
                    gho_x_t = self.sigmoid( np.dot(w.T,hidden_layer) + d)

                    delta_d = self.lambda_adapt * (1. - gho_x_t)
                    delta_w = delta_d * hidden_layer
                    tmp = delta_d * w * hidden_layer * (1.-hidden_layer)

                    delta_b += tmp
                    delta_W += tmp.reshape(-1,1) * x_t.reshape(1,-1)

                    # add domain adaptation regularizer from other domain
                    t_2 = np.random.randint(nb_examples_adapt)
                    i_2 = t_2 % nb_examples_adapt
                    x_t_2 = X_adapt[i_2,:]
                    hidden_layer_2 = self.sigmoid( np.dot(W, x_t_2) + b)
                    gho_x_t_2 = self.sigmoid( np.dot(w.T,hidden_layer_2) + d)

                    delta_d -= self.lambda_adapt * gho_x_t_2
                    delta_w -= self.lambda_adapt * gho_x_t_2 * hidden_layer_2

                    tmp =- self.lambda_adapt * gho_x_t_2 * w * hidden_layer_2 * (1. - hidden_layer_2)

                    delta_b += tmp
                    delta_W += tmp.reshape(-1,1) * x_t_2.reshape(1,-1)

                W -= delta_W * self.learning_rate
                b -= delta_b * self.learning_rate

                V -= delta_V * self.learning_rate
                c -= delta_c * self.learning_rate

                w += delta_w * self.learning_rate
                d += delta_d * self.learning_rate
            # END for i in range(nb_examples)

            self.W, self.V, self.b, self.c, self.w, self.d = W, V, b, c, w, d

            # early stopping
            if X_valid is not None:
                valid_pred = self.predict(X_valid)
                valid_risk = np.mean( valid_pred != Y_valid )
                if valid_risk <= best_valid_risk:
                    if self.verbose:
                        print('[best valid risk so far] %f (iter %d)' % (valid_risk, t))
                    best_valid_risk = valid_risk
                    best_weights = (W.copy(), V.copy(), b.copy(), c.copy())
                    best_t = t
                    continue_until = max(continue_until, int(1.5*t))
                elif t > continue_until:
                    if self.verbose:
                        print('[early stop] iter %d' % t)
                    break
        # END for t in range(self.maxiter)

        if X_valid is not None:
            self.W, self.V, self.b, self.c = best_weights
            self.nb_iter = best_t
            self.valid_risk = best_valid_risk
        else:
            self.nb_iter = self.maxiter
            self.valid_risk = 2.

        if self.verbose:
            print('[final valid risk] %f (iter %d)' % (valid_risk, t))

    def forward(self, X):
        """
         Compute and return the network outputs for X, i.e., a 2D array of size len(X) by len(set(Y)).
         the ith row of the array contains output probabilities for each class for the ith example.

        """
        hidden_layer = self.sigmoid( np.dot(self.W, X.T) + self.b[:,np.newaxis] )
        output_layer = self.softmax( np.dot(self.V, hidden_layer) + self.c[:,np.newaxis]  )
        return output_layer

    def predict(self, X):
        """
         Compute and return the label predictions for X, i.e., a 1D array of size len(X).
         the ith row of the array contains the predicted class for the ith example .
        """
        output_layer = self.forward(X)
        return np.argmax(output_layer, 0)

<|endoftext|>


<commit_before>from unittest import TestCase

from seaworthy.logs import EqualsMatcher, RegexMatcher, SequentialLinesMatcher

class TestEqualsMatcher(TestCase):
    def test_matching(self):
        """ Matches exactly equal strings and nothing else. """
        matcher = EqualsMatcher('foo')
        self.assertTrue(matcher('foo'))
        self.assertFalse(matcher('foobar'))

    def test_str(self):
        """ The string representation is readable. """
        matcher = EqualsMatcher('bar')
        self.assertEqual(str(matcher), "EqualsMatcher('bar')")

class TestRegexMatcher(TestCase):
    def test_matching(self):
        """
        Matches strings that match the pattern, doesn't match other strings.
        """
        matcher = RegexMatcher(r'^foo')
        self.assertTrue(matcher('foobar'))
        self.assertFalse(matcher('barfoo'))

    def test_str(self):
        """ The string representation is readable. """
        matcher = RegexMatcher(r'^bar')
        self.assertEqual(str(matcher), "RegexMatcher('^bar')")

class TestSequentialLinesMatcher(TestCase):
    def test_matching(self):
        """
        Applies each matcher sequentially and returns True on the final match.
        """
        matcher = SequentialLinesMatcher(
            EqualsMatcher('foo'), RegexMatcher('^bar'))

        self.assertFalse(matcher('barfoo'))
        self.assertFalse(matcher('foo'))
        self.assertFalse(matcher('baz'))
        self.assertTrue(matcher('barfoo'))

    def test_by_equality(self):
        """
        The ``by_equality`` utility method takes a list of strings and produces
        a matcher that matches those strings by equality sequentially.
        """
        matcher = SequentialLinesMatcher.by_equality('foo', 'bar')

        self.assertFalse(matcher('bar'))
        self.assertFalse(matcher('foo'))
        self.assertFalse(matcher('baz'))
        self.assertTrue(matcher('bar'))

    def test_by_regex(self):
        """
        The ``by_regex`` utility method takes a list of patterns and produces a
        matcher that matches by those regex patterns sequentially.
        """
        matcher = SequentialLinesMatcher.by_regex(r'^foo', r'bar$')

        self.assertFalse(matcher('fuzzbar'))
        self.assertFalse(matcher('foobar'))
        self.assertFalse(matcher('baz'))
        self.assertTrue(matcher('foobar'))

    def test_exhaustion(self):
        """
        Once all matchers have been matched, further calls to ``match`` should
        raise an error.
        """
        matcher = SequentialLinesMatcher.by_equality('foo')
        self.assertTrue(matcher('foo'))

        with self.assertRaises(RuntimeError) as cm:
            matcher('bar')
        self.assertEqual(str(cm.exception),
                         'Matcher exhausted, no more matchers to use')

    def test_str(self):
        """
        The string representation is readable and shows which matchers have
        been matched and which are still to be matched.
        """
        matcher = SequentialLinesMatcher(
            EqualsMatcher('foo'), RegexMatcher('^bar'))

        self.assertEqual(
            str(matcher),
            'SequentialLinesMatcher(matched=[], '
            "unmatched=[EqualsMatcher('foo'), RegexMatcher('^bar')])")

        self.assertFalse(matcher('foo'))

        self.assertEqual(
            str(matcher),
            "SequentialLinesMatcher(matched=[EqualsMatcher('foo')], "
            "unmatched=[RegexMatcher('^bar')])")

        self.assertTrue(matcher('barfoo'))

        self.assertEqual(
            str(matcher),
            "SequentialLinesMatcher(matched=[EqualsMatcher('foo'), "
            "RegexMatcher('^bar')], unmatched=[])")
<commit_msg>Import unittest.TestCase like elsewhere<commit_after>import unittest

from seaworthy.logs import EqualsMatcher, RegexMatcher, SequentialLinesMatcher

class TestEqualsMatcher(unittest.TestCase):
    def test_matching(self):
        """ Matches exactly equal strings and nothing else. """
        matcher = EqualsMatcher('foo')
        self.assertTrue(matcher('foo'))
        self.assertFalse(matcher('foobar'))

    def test_str(self):
        """ The string representation is readable. """
        matcher = EqualsMatcher('bar')
        self.assertEqual(str(matcher), "EqualsMatcher('bar')")

class TestRegexMatcher(unittest.TestCase):
    def test_matching(self):
        """
        Matches strings that match the pattern, doesn't match other strings.
        """
        matcher = RegexMatcher(r'^foo')
        self.assertTrue(matcher('foobar'))
        self.assertFalse(matcher('barfoo'))

    def test_str(self):
        """ The string representation is readable. """
        matcher = RegexMatcher(r'^bar')
        self.assertEqual(str(matcher), "RegexMatcher('^bar')")

class TestSequentialLinesMatcher(unittest.TestCase):
    def test_matching(self):
        """
        Applies each matcher sequentially and returns True on the final match.
        """
        matcher = SequentialLinesMatcher(
            EqualsMatcher('foo'), RegexMatcher('^bar'))

        self.assertFalse(matcher('barfoo'))
        self.assertFalse(matcher('foo'))
        self.assertFalse(matcher('baz'))
        self.assertTrue(matcher('barfoo'))

    def test_by_equality(self):
        """
        The ``by_equality`` utility method takes a list of strings and produces
        a matcher that matches those strings by equality sequentially.
        """
        matcher = SequentialLinesMatcher.by_equality('foo', 'bar')

        self.assertFalse(matcher('bar'))
        self.assertFalse(matcher('foo'))
        self.assertFalse(matcher('baz'))
        self.assertTrue(matcher('bar'))

    def test_by_regex(self):
        """
        The ``by_regex`` utility method takes a list of patterns and produces a
        matcher that matches by those regex patterns sequentially.
        """
        matcher = SequentialLinesMatcher.by_regex(r'^foo', r'bar$')

        self.assertFalse(matcher('fuzzbar'))
        self.assertFalse(matcher('foobar'))
        self.assertFalse(matcher('baz'))
        self.assertTrue(matcher('foobar'))

    def test_exhaustion(self):
        """
        Once all matchers have been matched, further calls to ``match`` should
        raise an error.
        """
        matcher = SequentialLinesMatcher.by_equality('foo')
        self.assertTrue(matcher('foo'))

        with self.assertRaises(RuntimeError) as cm:
            matcher('bar')
        self.assertEqual(str(cm.exception),
                         'Matcher exhausted, no more matchers to use')

    def test_str(self):
        """
        The string representation is readable and shows which matchers have
        been matched and which are still to be matched.
        """
        matcher = SequentialLinesMatcher(
            EqualsMatcher('foo'), RegexMatcher('^bar'))

        self.assertEqual(
            str(matcher),
            'SequentialLinesMatcher(matched=[], '
            "unmatched=[EqualsMatcher('foo'), RegexMatcher('^bar')])")

        self.assertFalse(matcher('foo'))

        self.assertEqual(
            str(matcher),
            "SequentialLinesMatcher(matched=[EqualsMatcher('foo')], "
            "unmatched=[RegexMatcher('^bar')])")

        self.assertTrue(matcher('barfoo'))

        self.assertEqual(
            str(matcher),
            "SequentialLinesMatcher(matched=[EqualsMatcher('foo'), "
            "RegexMatcher('^bar')], unmatched=[])")
<|endoftext|>