<commit_before>#!/usr/bin/python
#
# Copyright (c) 2012 Mikkel Schubert <MSchubert@snm.ku.dk>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
import io
import os
import subprocess
import collections
from itertools import \
    izip_longest

import pysam

from pypeline.node import \
    Node, \
    NodeError
from pypeline.common.fileutils import \
    describe_files, \
    make_dirs
from pypeline.common.utilities import \
    chain_sorted

import pypeline.common.formats.fastq as fastq
import pypeline.common.sampling as sampling

class DetectInputDuplicationNode(Node):
    """Attempts to detect reads included multiple times as input based on the
    presence of reads with identical names AND sequences. This is compromise
    between sensitivity, specificity, and running time.

    A possible refinement would be to consider reads with the same name where
    one read is the prefix of the other (due to different amounts of trimming
    or collapsing of reads).
    """

    def __init__(self, input_files, output_file, dependencies=()):
        Node.__init__(self,
                      description="<DetectInputDuplication: %s>"
                      % (describe_files(input_files)),
                      input_files=input_files,
                      output_files=output_file,
                      dependencies=dependencies)

    def run(self, _):
        handles = []
        try:
            sequences = []
            for fpath in self.input_files:
                handle = pysam.Samfile(fpath)
                handles.append(handle)

                sequence = izip_longest(handle, (), fillvalue=fpath)
                sequences.append(sequence)

            position = 0
            records = chain_sorted(*sequences, key=self._key_by_tid_pos)
            observed_reads = collections.defaultdict(list)
            for (record, fpath) in records:
                if record.pos != position:
                    self._process_reads(observed_reads)
                    observed_reads.clear()
                    position = record.pos

                key = (record.qname, record.seq, record.qual)
                observed_reads[key].append(fpath)
            self._process_reads(observed_reads)

            # Everything is ok, touch the output files
            for fpath in self.output_files:
                make_dirs(os.path.dirname(fpath))
                with open(fpath, "w"):
                    pass
        finally:
            for handle in handles:
                handle.close()

    @classmethod
    def _process_reads(cls, observed_reads):
        for ((name, _, _), fpaths) in observed_reads.iteritems():
            if len(fpaths) > 1:
                message = ["Read %r found in multiple files:" % (name,)]
                for fpath in fpaths:
                    message.append("  - %r" % (fpath,))
                message.append("")
                message.append("This indicates that the same data files have "
                               "been included multiple times in the project. "
                               "Please review the input files used in this "
                               "project, to ensure that each set of data is "
                               "included only once.")
                raise NodeError("\n".join(message))

    @classmethod
    def _key_by_tid_pos(cls, record):
        return (record[0].tid, record[0].pos)

class ValidateFASTQFilesNode(Node):
    def __init__(self, input_files, output_file, offset, dependencies=()):
        self._offset = offset
        Node.__init__(self,
                      description="<ValidateFASTQFilesNode: %s>"
                      % (describe_files(input_files)),
                      input_files=input_files,
                      output_files=output_file,
                      dependencies=dependencies)

    def _run(self, _config, _temp):
        check_fastq_files(self.input_files, self._offset, True)
        output_file = tuple(self.output_files)[0]
        make_dirs(os.path.dirname(output_file))
        with open(output_file, "w"):
            pass

def check_fastq_files(filenames, required_offset, allow_empty=False):
    for filename in filenames:
        qualities = _read_sequences(filename)
        offsets = fastq.classify_quality_strings(qualities)
        if offsets == fastq.OFFSET_BOTH:
            raise NodeError("FASTQ file contains quality scores with both "
                            "quality offsets (33 and 64); file may be "
                            "unexpected format or corrupt. Please ensure "
                            "that this file contains valid FASTQ reads from a "
                            "single source.\n    Filename = %r" % (filename,))
        elif offsets == fastq.OFFSET_MISSING:
            if allow_empty and not qualities:
                return

            raise NodeError("FASTQ file did not contain quality scores; file "
                            "may be unexpected format or corrupt. Ensure that "
                            "the file is a FASTQ file.\n    Filename = %r"
                            % (filename,))
        elif offsets not in (fastq.OFFSET_AMBIGIOUS, required_offset):
            raise NodeError("FASTQ file contains quality scores with wrong "
                            "quality score offset (%i); expected reads with "
                            "quality score offset %i. Ensure that the "
                            "'QualityOffset' specified in the makefile "
                            "corresponds to the input.\n    Filename = %s"
                            % (offsets, required_offset, filename))

def _read_sequences(filename):
    unicat = None
    try:
        unicat = subprocess.Popen(['unicat', filename],
                                  bufsize=io.DEFAULT_BUFFER_SIZE,
                                  stderr=subprocess.PIPE,
                                  stdout=subprocess.PIPE)
        qualities = _collect_qualities(unicat.stdout, filename)

        return sampling.reservoir_sampling(qualities, 100000)
    except:
        if unicat:
            unicat.kill()
            unicat.wait()
            unicat = None
        raise
    finally:
        rc_unicat = unicat.wait() if unicat else 0
        if rc_unicat:
            message = "Error running unicat:\n" \
                      "  Unicat return-code = %i\n\n%s" \
                      % (rc_unicat, unicat.stderr.read())
            raise NodeError(message)

def _collect_qualities(handle, filename):
    lines_offset = 0
    header = handle.readline()
    while header:
        sequence = handle.readline()
        seperator = handle.readline()
        qualities = handle.readline()

        if not header.startswith("@"):
            if header.startswith(">"):
                raise NodeError("Input file appears to be in FASTA format "
                                "(header starts with '>', expected '@'), "
                                "but only FASTQ files are supported\n"
                                "Filename = %r" % (filename,))

            raise NodeError("Input file lacks FASTQ header (expected '@', "
                            "found %r), but only FASTQ files are supported\n"
                            "    Filename = %r" % (header[:1], filename))
        elif not qualities:
            raise NodeError("Partial record found; is not 4 lines long:\n"
                            "Filename = %r\n    Record = '%s'"
                            % (filename, header.rstrip()))
        elif not seperator.startswith("+"):
            raise NodeError("Input file lacks FASTQ seperator (expected '+', "
                            "found %r), but only FASTQ files are supported\n"
                            "    Filename = %r" % (seperator[:1], filename))
        elif len(sequence) != len(qualities):
            raise NodeError("Input file contains malformed FASTQ records; "
                            "length of sequence / qualities are not the "
                            "same.\n    Filename = %r\n    Record = '%s'"
                            % (filename, header.rstrip()))

        yield qualities
        header = handle.readline()
<commit_msg>Tweak node names.<commit_after>#!/usr/bin/python
#
# Copyright (c) 2012 Mikkel Schubert <MSchubert@snm.ku.dk>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
import io
import os
import subprocess
import collections
from itertools import \
    izip_longest

import pysam

from pypeline.node import \
    Node, \
    NodeError
from pypeline.common.fileutils import \
    describe_files, \
    make_dirs
from pypeline.common.utilities import \
    chain_sorted

import pypeline.common.formats.fastq as fastq
import pypeline.common.sampling as sampling

class DetectInputDuplicationNode(Node):
    """Attempts to detect reads included multiple times as input based on the
    presence of reads with identical names AND sequences. This is compromise
    between sensitivity, specificity, and running time.

    A possible refinement would be to consider reads with the same name where
    one read is the prefix of the other (due to different amounts of trimming
    or collapsing of reads).
    """

    def __init__(self, input_files, output_file, dependencies=()):
        Node.__init__(self,
                      description="<Detect Input Duplication: %s>"
                      % (describe_files(input_files)),
                      input_files=input_files,
                      output_files=output_file,
                      dependencies=dependencies)

    def run(self, _):
        handles = []
        try:
            sequences = []
            for fpath in self.input_files:
                handle = pysam.Samfile(fpath)
                handles.append(handle)

                sequence = izip_longest(handle, (), fillvalue=fpath)
                sequences.append(sequence)

            position = 0
            records = chain_sorted(*sequences, key=self._key_by_tid_pos)
            observed_reads = collections.defaultdict(list)
            for (record, fpath) in records:
                if record.pos != position:
                    self._process_reads(observed_reads)
                    observed_reads.clear()
                    position = record.pos

                key = (record.qname, record.seq, record.qual)
                observed_reads[key].append(fpath)
            self._process_reads(observed_reads)

            # Everything is ok, touch the output files
            for fpath in self.output_files:
                make_dirs(os.path.dirname(fpath))
                with open(fpath, "w"):
                    pass
        finally:
            for handle in handles:
                handle.close()

    @classmethod
    def _process_reads(cls, observed_reads):
        for ((name, _, _), fpaths) in observed_reads.iteritems():
            if len(fpaths) > 1:
                message = ["Read %r found in multiple files:" % (name,)]
                for fpath in fpaths:
                    message.append("  - %r" % (fpath,))
                message.append("")
                message.append("This indicates that the same data files have "
                               "been included multiple times in the project. "
                               "Please review the input files used in this "
                               "project, to ensure that each set of data is "
                               "included only once.")
                raise NodeError("\n".join(message))

    @classmethod
    def _key_by_tid_pos(cls, record):
        return (record[0].tid, record[0].pos)

class ValidateFASTQFilesNode(Node):
    def __init__(self, input_files, output_file, offset, dependencies=()):
        self._offset = offset
        Node.__init__(self,
                      description="<Validate FASTQ Files: %s>"
                      % (describe_files(input_files)),
                      input_files=input_files,
                      output_files=output_file,
                      dependencies=dependencies)

    def _run(self, _config, _temp):
        check_fastq_files(self.input_files, self._offset, True)
        output_file = tuple(self.output_files)[0]
        make_dirs(os.path.dirname(output_file))
        with open(output_file, "w"):
            pass

def check_fastq_files(filenames, required_offset, allow_empty=False):
    for filename in filenames:
        qualities = _read_sequences(filename)
        offsets = fastq.classify_quality_strings(qualities)
        if offsets == fastq.OFFSET_BOTH:
            raise NodeError("FASTQ file contains quality scores with both "
                            "quality offsets (33 and 64); file may be "
                            "unexpected format or corrupt. Please ensure "
                            "that this file contains valid FASTQ reads from a "
                            "single source.\n    Filename = %r" % (filename,))
        elif offsets == fastq.OFFSET_MISSING:
            if allow_empty and not qualities:
                return

            raise NodeError("FASTQ file did not contain quality scores; file "
                            "may be unexpected format or corrupt. Ensure that "
                            "the file is a FASTQ file.\n    Filename = %r"
                            % (filename,))
        elif offsets not in (fastq.OFFSET_AMBIGIOUS, required_offset):
            raise NodeError("FASTQ file contains quality scores with wrong "
                            "quality score offset (%i); expected reads with "
                            "quality score offset %i. Ensure that the "
                            "'QualityOffset' specified in the makefile "
                            "corresponds to the input.\n    Filename = %s"
                            % (offsets, required_offset, filename))

def _read_sequences(filename):
    unicat = None
    try:
        unicat = subprocess.Popen(['unicat', filename],
                                  bufsize=io.DEFAULT_BUFFER_SIZE,
                                  stderr=subprocess.PIPE,
                                  stdout=subprocess.PIPE)
        qualities = _collect_qualities(unicat.stdout, filename)

        return sampling.reservoir_sampling(qualities, 100000)
    except:
        if unicat:
            unicat.kill()
            unicat.wait()
            unicat = None
        raise
    finally:
        rc_unicat = unicat.wait() if unicat else 0
        if rc_unicat:
            message = "Error running unicat:\n" \
                      "  Unicat return-code = %i\n\n%s" \
                      % (rc_unicat, unicat.stderr.read())
            raise NodeError(message)

def _collect_qualities(handle, filename):
    lines_offset = 0
    header = handle.readline()
    while header:
        sequence = handle.readline()
        seperator = handle.readline()
        qualities = handle.readline()

        if not header.startswith("@"):
            if header.startswith(">"):
                raise NodeError("Input file appears to be in FASTA format "
                                "(header starts with '>', expected '@'), "
                                "but only FASTQ files are supported\n"
                                "Filename = %r" % (filename,))

            raise NodeError("Input file lacks FASTQ header (expected '@', "
                            "found %r), but only FASTQ files are supported\n"
                            "    Filename = %r" % (header[:1], filename))
        elif not qualities:
            raise NodeError("Partial record found; is not 4 lines long:\n"
                            "Filename = %r\n    Record = '%s'"
                            % (filename, header.rstrip()))
        elif not seperator.startswith("+"):
            raise NodeError("Input file lacks FASTQ seperator (expected '+', "
                            "found %r), but only FASTQ files are supported\n"
                            "    Filename = %r" % (seperator[:1], filename))
        elif len(sequence) != len(qualities):
            raise NodeError("Input file contains malformed FASTQ records; "
                            "length of sequence / qualities are not the "
                            "same.\n    Filename = %r\n    Record = '%s'"
                            % (filename, header.rstrip()))

        yield qualities
        header = handle.readline()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
This module contains the checker mode, a base class for code checker modes.
"""
import logging
from pyqode.core.api import TextBlockUserData
from pyqode.core.api.decoration import TextDecoration
from pyqode.core.api.mode import Mode
from pyqode.core.backend import NotRunning
from pyqode.core.api.utils import DelayJobRunner
from pyqode.qt import QtCore, QtGui

class CheckerMessages(object):
    """
    Enumerates the possible checker message types.
    """
    #: Status value for an information message.
    INFO = 0
    #: Status value for a warning message.
    WARNING = 1
    #: Status value for an error message.
    ERROR = 2

class CheckerMessage(object):
    """
    Holds data for a message displayed by the
    :class:`pyqode.core.modes.CheckerMode`.
    """
    #: Default colors foreach message status
    COLORS = {CheckerMessages.INFO: "#4040DD",
              CheckerMessages.WARNING: "#DDDD40",
              CheckerMessages.ERROR: "#DD4040"}

    @classmethod
    def status_to_string(cls, status):
        """
        Converts a message status to a string.

        :param status: Status to convert (p yqode.core.modes.CheckerMessages)
        :return: The status string.
        :rtype: str
        """
        strings = {CheckerMessages.INFO: "Info",
                   CheckerMessages.WARNING: "Warning",
                   CheckerMessages.ERROR: "Error"}
        return strings[status]

    @property
    def status_string(self):
        """
        Returns the message status as a string.

        :return: The status string.
        """
        return self.status_to_string(self.status)

    def __init__(self, description, status, line, col=None,
                 color=None, path=None):
        """
        :param description: The message description (used as a tooltip)
        :param status: The status associated with the message.
        :param line: The message line number
        :param col: The message start column (at the moment the message ends at
                    the end of the line).
        :param color: Text decoration color
        :param path: file path. Optional
        """
        assert 0 <= status <= 2
        #: The description of the message, used as a tooltip.
        self.description = description
        #: The status associated with the message. One of:
        #:    * :const:`pyqode.core.modes.CheckerMessages.INFO`
        #:    * :const:`pyqode.core.modes.CheckerMessages.WARNING`
        #:    * :const:`pyqode.core.modes.CheckerMessages.ERROR`
        self.status = status
        #: The line of the message
        self.line = line
        #: The start column (used for the text decoration). If the col is None,
        #: the whole line is highlighted.
        self.col = col
        #: The color used for the text decoration. If None, the default color
        #: is used (:const:`pyqode.core.CheckerMessage.COLORS`)
        self.color = color
        if self.color is None:
            self.color = self.COLORS[status]
        self.decoration = None
        self.path = path
        #: store a reference to the associated QTextBlock, for quick acces
        self.block = None

    def __str__(self):
        return "{0} l{1}".format(self.description, self.line)

    def __eq__(self, other):
        return (self.block == other.block and
                self.description == other.description)

def _logger(klass):
    return logging.getLogger('%s [%s]' % (__name__, klass.__name__))

class CheckerMode(Mode, QtCore.QObject):
    """
    Performs a user defined code analysis job using the backend and
    display the results on the editor instance.

    The user defined code analysis job is a simple **function** with the
    following signature:

    .. code-block:: python

        def analysisProcess(data)

    where data is the request data:

    .. code-block:: python

        request_data = {
                'code': self.editor.toPlainText(),
                'path': self.editor.file.path,
                'encoding': self.editor.file.encoding
            }

    and the return value is a tuple made up of the following elements:

        (description, status, line, [col], [icon], [color], [path])

    The background process is ran when the text changed and the ide is an idle
    state for a few seconds.

    You can also request an analysis manually using
    :meth:`pyqode.core.modes.CheckerMode.request_analysis`

    Messages are displayed as text decorations on the editor. A checker panel
    will take care of display message icons next to each line.
    """
    @property
    def messages(self):
        return self._messages

    def __init__(self, worker,
                 delay=500,
                 show_tooltip=True):
        """
        :param worker: The process function or class to call remotely.
        :param delay: The delay used before running the analysis process when
                      trigger is set to
                      :class:pyqode.core.modes.CheckerTriggers`
        :param show_tooltip: Specify if a tooltip must be displayed when the
                             mouse is over a checker message decoration.
        """
        Mode.__init__(self)
        QtCore.QObject.__init__(self)
        # max number of messages to keep good performances
        self.limit = 200
        self._job_runner = DelayJobRunner(delay=delay)
        self._messages = []
        self._worker = worker
        self._mutex = QtCore.QMutex()
        self._show_tooltip = show_tooltip
        self._pending_msg = []
        self._finished = True

    def add_messages(self, messages):
        """
        Adds a message or a list of message.

        :param messages: A list of messages or a single message
        """
        # remove old messages
        if len(messages) > self.limit:
            messages = messages[:self.limit]
        _logger(self.__class__).debug('adding %s messages' % len(messages))
        self._finished = False
        self._new_messages = messages
        self._to_check = list(self._messages)
        self._pending_msg = messages
        # start removing messages, new message won't be added until we
        # checked all message that need to be removed
        QtCore.QTimer.singleShot(1, self._remove_batch)

    def _remove_batch(self):
        if self.editor is None:
            return
        for i in range(100):
            if not len(self._to_check):
                # all messages checker, start adding messages now
                QtCore.QTimer.singleShot(1, self._add_batch)
                self.editor.repaint()
                return False
            msg = self._to_check.pop(0)
            if msg.block is None:
                msg.block = self.editor.document().findBlockByNumber(msg.line)
            if msg not in self._new_messages:
                self.remove_message(msg)
        self.editor.repaint()
        QtCore.QTimer.singleShot(1, self._remove_batch)

    def _add_batch(self):
        if self.editor is None:
            return
        for i in range(10):
            if not len(self._pending_msg):
                # all pending message added
                self._finished = True
                _logger(self.__class__).debug('finished')
                self.editor.repaint()
                return False
            message = self._pending_msg.pop(0)
            if message.line:
                try:
                    usd = message.block.userData()
                except AttributeError:
                    message.block = self.editor.document().findBlockByNumber(
                        message.line)
                    usd = message.block.userData()
                if usd is None:
                    usd = TextBlockUserData()
                    message.block.setUserData(usd)
                # check if the same message already exists
                if message in usd.messages:
                    continue
                self._messages.append(message)
                usd.messages.append(message)
                tooltip = None
                if self._show_tooltip:
                    tooltip = message.description
                message.decoration = TextDecoration(
                    self.editor.textCursor(), start_line=message.line,
                    tooltip=tooltip, draw_order=3)
                message.decoration.select_line()
                message.decoration.set_as_error(color=QtGui.QColor(
                    message.color))
                self.editor.decorations.append(message.decoration)
        QtCore.QTimer.singleShot(1, self._add_batch)
        self.editor.repaint()
        return True

    def remove_message(self, message):
        """
        Removes a message.

        :param message: Message to remove
        """
        import time
        _logger(self.__class__).debug('removing message %s' % message)
        t = time.time()
        usd = message.block.userData()
        if usd:
            try:
                usd.messages.remove(message)
            except (AttributeError, ValueError):
                pass
        if message.decoration:
            self.editor.decorations.remove(message.decoration)
        self._messages.remove(message)

    def clear_messages(self):
        """
        Clears all messages.
        """
        while len(self._messages):
            msg = self._messages.pop(0)
            usd = msg.block.userData()
            if usd and hasattr(usd, 'messages'):
                usd.messages[:] = []
            if msg.decoration:
                self.editor.decorations.remove(msg.decoration)

    def on_state_changed(self, state):
        if state:
            self.editor.textChanged.connect(self.request_analysis)
            self.editor.new_text_set.connect(self.clear_messages)
            self.request_analysis()
        else:
            self.editor.textChanged.disconnect(self.request_analysis)
            self.editor.new_text_set.disconnect(self.clear_messages)
            self._job_runner.cancel_requests()
            self.clear_messages()

    def _on_work_finished(self, status, results):
        """
        Display results.

        :param status: Response status
        :param results: Response data, messages.
        """
        if status:
            messages = []
            for msg in results:
                msg = CheckerMessage(*msg)
                block = self.editor.document().findBlockByNumber(msg.line)
                msg.block = block
                messages.append(msg)
            self.add_messages(messages)

    def request_analysis(self):
        """
        Requests an analysis.
        """
        if self._finished:
            _logger(self.__class__).debug('running analysis')
            self._job_runner.request_job(self._request)
        elif self.editor:
            # retry later
            _logger(self.__class__).debug(
                'delaying analysis (previous analysis not finished)')
            QtCore.QTimer.singleShot(500, self.request_analysis)

    def _request(self):
        """ Requests a checking of the editor content. """
        if not self.editor:
            return
        try:
            request_data = {
                'code': self.editor.toPlainText(),
                'path': self.editor.file.path,
                'encoding': self.editor.file.encoding
            }
        except RuntimeError:
            return
        try:
            self.editor.backend.send_request(
                self._worker, request_data, on_receive=self._on_work_finished)
            self._finished = False
        except NotRunning:
            # retry later
            QtCore.QTimer.singleShot(100, self._request)
<commit_msg>Fix backward incompatibility: icon parameter is used in OpenCobolIDE even though it is not used by the implementation, just keep the parameter for the sake of backward compatibility<commit_after># -*- coding: utf-8 -*-
"""
This module contains the checker mode, a base class for code checker modes.
"""
import logging
from pyqode.core.api import TextBlockUserData
from pyqode.core.api.decoration import TextDecoration
from pyqode.core.api.mode import Mode
from pyqode.core.backend import NotRunning
from pyqode.core.api.utils import DelayJobRunner
from pyqode.qt import QtCore, QtGui

class CheckerMessages(object):
    """
    Enumerates the possible checker message types.
    """
    #: Status value for an information message.
    INFO = 0
    #: Status value for a warning message.
    WARNING = 1
    #: Status value for an error message.
    ERROR = 2

class CheckerMessage(object):
    """
    Holds data for a message displayed by the
    :class:`pyqode.core.modes.CheckerMode`.
    """
    #: Default colors foreach message status
    COLORS = {CheckerMessages.INFO: "#4040DD",
              CheckerMessages.WARNING: "#DDDD40",
              CheckerMessages.ERROR: "#DD4040"}

    @classmethod
    def status_to_string(cls, status):
        """
        Converts a message status to a string.

        :param status: Status to convert (p yqode.core.modes.CheckerMessages)
        :return: The status string.
        :rtype: str
        """
        strings = {CheckerMessages.INFO: "Info",
                   CheckerMessages.WARNING: "Warning",
                   CheckerMessages.ERROR: "Error"}
        return strings[status]

    @property
    def status_string(self):
        """
        Returns the message status as a string.

        :return: The status string.
        """
        return self.status_to_string(self.status)

    def __init__(self, description, status, line, col=None, icon=None,
                 color=None, path=None):
        """
        :param description: The message description (used as a tooltip)
        :param status: The status associated with the message.
        :param line: The message line number
        :param col: The message start column (at the moment the message ends at
                    the end of the line).
        :param icon: Unused, we keep it for backward compatiblity.
        :param color: Text decoration color
        :param path: file path. Optional
        """
        assert 0 <= status <= 2
        #: The description of the message, used as a tooltip.
        self.description = description
        #: The status associated with the message. One of:
        #:    * :const:`pyqode.core.modes.CheckerMessages.INFO`
        #:    * :const:`pyqode.core.modes.CheckerMessages.WARNING`
        #:    * :const:`pyqode.core.modes.CheckerMessages.ERROR`
        self.status = status
        #: The line of the message
        self.line = line
        #: The start column (used for the text decoration). If the col is None,
        #: the whole line is highlighted.
        self.col = col
        #: The color used for the text decoration. If None, the default color
        #: is used (:const:`pyqode.core.CheckerMessage.COLORS`)
        self.color = color
        if self.color is None:
            self.color = self.COLORS[status]
        self.decoration = None
        self.path = path
        #: store a reference to the associated QTextBlock, for quick acces
        self.block = None

    def __str__(self):
        return "{0} l{1}".format(self.description, self.line)

    def __eq__(self, other):
        return (self.block == other.block and
                self.description == other.description)

def _logger(klass):
    return logging.getLogger('%s [%s]' % (__name__, klass.__name__))

class CheckerMode(Mode, QtCore.QObject):
    """
    Performs a user defined code analysis job using the backend and
    display the results on the editor instance.

    The user defined code analysis job is a simple **function** with the
    following signature:

    .. code-block:: python

        def analysisProcess(data)

    where data is the request data:

    .. code-block:: python

        request_data = {
                'code': self.editor.toPlainText(),
                'path': self.editor.file.path,
                'encoding': self.editor.file.encoding
            }

    and the return value is a tuple made up of the following elements:

        (description, status, line, [col], [icon], [color], [path])

    The background process is ran when the text changed and the ide is an idle
    state for a few seconds.

    You can also request an analysis manually using
    :meth:`pyqode.core.modes.CheckerMode.request_analysis`

    Messages are displayed as text decorations on the editor. A checker panel
    will take care of display message icons next to each line.
    """
    @property
    def messages(self):
        return self._messages

    def __init__(self, worker,
                 delay=500,
                 show_tooltip=True):
        """
        :param worker: The process function or class to call remotely.
        :param delay: The delay used before running the analysis process when
                      trigger is set to
                      :class:pyqode.core.modes.CheckerTriggers`
        :param show_tooltip: Specify if a tooltip must be displayed when the
                             mouse is over a checker message decoration.
        """
        Mode.__init__(self)
        QtCore.QObject.__init__(self)
        # max number of messages to keep good performances
        self.limit = 200
        self._job_runner = DelayJobRunner(delay=delay)
        self._messages = []
        self._worker = worker
        self._mutex = QtCore.QMutex()
        self._show_tooltip = show_tooltip
        self._pending_msg = []
        self._finished = True

    def add_messages(self, messages):
        """
        Adds a message or a list of message.

        :param messages: A list of messages or a single message
        """
        # remove old messages
        if len(messages) > self.limit:
            messages = messages[:self.limit]
        _logger(self.__class__).debug('adding %s messages' % len(messages))
        self._finished = False
        self._new_messages = messages
        self._to_check = list(self._messages)
        self._pending_msg = messages
        # start removing messages, new message won't be added until we
        # checked all message that need to be removed
        QtCore.QTimer.singleShot(1, self._remove_batch)

    def _remove_batch(self):
        if self.editor is None:
            return
        for i in range(100):
            if not len(self._to_check):
                # all messages checker, start adding messages now
                QtCore.QTimer.singleShot(1, self._add_batch)
                self.editor.repaint()
                return False
            msg = self._to_check.pop(0)
            if msg.block is None:
                msg.block = self.editor.document().findBlockByNumber(msg.line)
            if msg not in self._new_messages:
                self.remove_message(msg)
        self.editor.repaint()
        QtCore.QTimer.singleShot(1, self._remove_batch)

    def _add_batch(self):
        if self.editor is None:
            return
        for i in range(10):
            if not len(self._pending_msg):
                # all pending message added
                self._finished = True
                _logger(self.__class__).debug('finished')
                self.editor.repaint()
                return False
            message = self._pending_msg.pop(0)
            if message.line:
                try:
                    usd = message.block.userData()
                except AttributeError:
                    message.block = self.editor.document().findBlockByNumber(
                        message.line)
                    usd = message.block.userData()
                if usd is None:
                    usd = TextBlockUserData()
                    message.block.setUserData(usd)
                # check if the same message already exists
                if message in usd.messages:
                    continue
                self._messages.append(message)
                usd.messages.append(message)
                tooltip = None
                if self._show_tooltip:
                    tooltip = message.description
                message.decoration = TextDecoration(
                    self.editor.textCursor(), start_line=message.line,
                    tooltip=tooltip, draw_order=3)
                message.decoration.select_line()
                message.decoration.set_as_error(color=QtGui.QColor(
                    message.color))
                self.editor.decorations.append(message.decoration)
        QtCore.QTimer.singleShot(1, self._add_batch)
        self.editor.repaint()
        return True

    def remove_message(self, message):
        """
        Removes a message.

        :param message: Message to remove
        """
        import time
        _logger(self.__class__).debug('removing message %s' % message)
        t = time.time()
        usd = message.block.userData()
        if usd:
            try:
                usd.messages.remove(message)
            except (AttributeError, ValueError):
                pass
        if message.decoration:
            self.editor.decorations.remove(message.decoration)
        self._messages.remove(message)

    def clear_messages(self):
        """
        Clears all messages.
        """
        while len(self._messages):
            msg = self._messages.pop(0)
            usd = msg.block.userData()
            if usd and hasattr(usd, 'messages'):
                usd.messages[:] = []
            if msg.decoration:
                self.editor.decorations.remove(msg.decoration)

    def on_state_changed(self, state):
        if state:
            self.editor.textChanged.connect(self.request_analysis)
            self.editor.new_text_set.connect(self.clear_messages)
            self.request_analysis()
        else:
            self.editor.textChanged.disconnect(self.request_analysis)
            self.editor.new_text_set.disconnect(self.clear_messages)
            self._job_runner.cancel_requests()
            self.clear_messages()

    def _on_work_finished(self, status, results):
        """
        Display results.

        :param status: Response status
        :param results: Response data, messages.
        """
        if status:
            messages = []
            for msg in results:
                msg = CheckerMessage(*msg)
                block = self.editor.document().findBlockByNumber(msg.line)
                msg.block = block
                messages.append(msg)
            self.add_messages(messages)

    def request_analysis(self):
        """
        Requests an analysis.
        """
        if self._finished:
            _logger(self.__class__).debug('running analysis')
            self._job_runner.request_job(self._request)
        elif self.editor:
            # retry later
            _logger(self.__class__).debug(
                'delaying analysis (previous analysis not finished)')
            QtCore.QTimer.singleShot(500, self.request_analysis)

    def _request(self):
        """ Requests a checking of the editor content. """
        if not self.editor:
            return
        try:
            request_data = {
                'code': self.editor.toPlainText(),
                'path': self.editor.file.path,
                'encoding': self.editor.file.encoding
            }
        except RuntimeError:
            return
        try:
            self.editor.backend.send_request(
                self._worker, request_data, on_receive=self._on_work_finished)
            self._finished = False
        except NotRunning:
            # retry later
            QtCore.QTimer.singleShot(100, self._request)
<|endoftext|>


<commit_before>'''
Unittests for shared PyRate objects.
Created on 12/09/2012
@author: bpd900
'''

# FIXME: replace ROIPAC dependencies with PyRate GeoTIFF files

import os
import shutil
import unittest
from os.path import join, basename
from stat import S_IRGRP, S_IWGRP, S_IWOTH, S_IROTH, S_IRUSR, S_IWUSR

from itertools import product
from numpy import array, isnan, where, nan
from numpy.testing import assert_array_equal

from gdal import Open, Dataset, UseExceptions
UseExceptions()

from pyrate.shared import Ifg, DEM, RasterException, Incidence
from common import SYD_TEST_TIF, INCID_TEST_DIR, SYD_TEST_DEM_TIF

# FIXME: refactor the ifg.open() calls
class IfgTests(unittest.TestCase):
	'''Unit tests for the Ifg/interferogram class.'''

	def setUp(self):
		self.ifg = Ifg(join(SYD_TEST_TIF, 'geo_060619-061002.tif'))

	def test_create_ifg(self):
		self.assertTrue(os.path.exists(self.ifg.data_path)) # validate data path

	def test_headers_as_attr(self):
		self.ifg.open()
		for a in ['ncols', 'nrows', 'x_first', 'x_step',
				'y_first', 'y_step', 'wavelength', 'master', 'slave']:
			self.assertTrue(getattr(self.ifg, a) is not None)

	def test_open(self):
		self.assertTrue(self.ifg.dataset is None)
		self.assertTrue(self.ifg.is_open is False)
		self.ifg.open(readonly=True)
		self.assertTrue(self.ifg.dataset is not None)
		self.assertTrue(self.ifg.is_open is True)
		self.assertTrue(isinstance(self.ifg.dataset, Dataset))

		# ensure open cannot be called twice
		self.failUnlessRaises(RasterException, self.ifg.open, True)

	def test_write(self):
		base = "/tmp"
		src = self.ifg.data_path
		dest = join(base, basename(self.ifg.data_path))

		# shutil.copy needs to copy writeable permission from src
		os.chmod(src, S_IRGRP | S_IWGRP | S_IWOTH | S_IROTH | S_IRUSR | S_IWUSR)
		shutil.copy(src, dest)
		os.chmod(src, S_IRGRP | S_IROTH | S_IRUSR) # revert

		i = Ifg(dest)
		i.open()
		i.phase_data[0,1:] = nan
		i.write_phase()
		del i

		# reopen to ensure data/nans can be read back out
		i = Ifg(dest)
		i.open(readonly=True)
		assert_array_equal(True, isnan(i.phase_data[0,1:]) )
		os.remove(dest)

	def test_readonly_permission_failure(self):
		# ensure failure if opening R/O permission file as writeable/GA_Update
		self.assertRaises(IOError, self.ifg.open, False)

	def test_write_fails_on_readonly(self):
		# check readonly status is same before and after open() for readonly file
		self.assertTrue(self.ifg.is_read_only)
		self.ifg.open(readonly=True)
		self.assertTrue(self.ifg.is_read_only)
		self.assertRaises(IOError, self.ifg.write_phase)

	def test_convert_to_nans(self):
		self.ifg.open()
		self.ifg.convert_to_nans(0)
		self.assertTrue(self.ifg.nan_converted)

	def test_xylast(self):
		# ensure the X|Y_LAST header element has been created
		self.ifg.open()
		self.assertAlmostEqual(self.ifg.x_last, 150.9491667)
		self.assertAlmostEqual(self.ifg.y_last, -34.23)

	def test_num_cells(self):
		# test cell size from header elements
		self.ifg.open()
		data = self.ifg.phase_band.ReadAsArray()
		ys, xs = data.shape
		exp_ncells = ys * xs
		self.assertEqual(exp_ncells, self.ifg.num_cells)

	def test_shape(self):
		self.ifg.open()
		self.assertEqual(self.ifg.shape, self.ifg.phase_data.shape)

# 	def test_amp_band(self):
# 		try:
# 			_ = self.ifg.amp_band
# 			self.fail("Should not be able to access band without open dataset")
# 		except RasterException:
# 			pass
#
# 		self.ifg.open()
# 		data = self.ifg.amp_band.ReadAsArray()
# 		self.assertEqual(data.shape, (72, 47) )

	def test_phase_band(self):
		try:
			_ = self.ifg.phase_band
			self.fail("Should not be able to access band without open dataset")
		except RasterException:
			pass

		self.ifg.open()
		data = self.ifg.phase_band.ReadAsArray()
		self.assertEqual(data.shape, (72, 47) )

	def test_nan_count(self):
		self.ifg.open()
		num_nan = 0
		for row in self.ifg.phase_data:
			for v in row:
				if isnan(v):
					num_nan += 1

		self.assertEqual(num_nan, self.ifg.nan_count)

	def test_nan_fraction(self):
		try:
			# NB: self.assertRaises doesn't work here (as it is a property?)
			_ = self.ifg.nan_fraction
			self.fail("Shouldn't be able to call nan_fraction() with unopened Ifg")
		except RasterException:
			pass

		# NB: source data lacks 0 -> NaN conversion
		self.ifg.open()
		data = self.ifg.dataset.GetRasterBand(1).ReadAsArray()
		data = where(data == 0, nan, data) # fake 0 -> nan for the count below

		# manually count # nan cells
		nans = 0
		ys, xs = data.shape
		for y, x in product(xrange(ys), xrange(xs)):
			if isnan(data[y, x]):
				nans += 1
		del data

		num_cells = float(ys * xs)
		self.assertTrue(nans > 0)
		self.assertTrue(nans <= num_cells)
		self.assertEqual(nans / num_cells, self.ifg.nan_fraction)

	def test_phase_data_properties(self):
		# Use raw GDAL to isolate raster reading from Ifg functionality
		ds = Open(self.ifg.data_path)
		data = ds.GetRasterBand(1).ReadAsArray()
		del ds

		self.ifg.open()

		# test full array and row by row access
		assert_array_equal(data, self.ifg.phase_data)
		for y, row in enumerate(self.ifg.phase_rows):
			assert_array_equal(data[y], row)

		# test the data is cached if changed
		crd = (5,4)
		orig = self.ifg.phase_data[crd]
		self.ifg.phase_data[crd] *= 2
		nv = self.ifg.phase_data[crd] # pull new value out again
		self.assertEqual(nv, 2 * orig)

	def test_xy_size(self):
		self.ifg.open()
		self.assertFalse(self.ifg.ncols is None)
		self.assertFalse(self.ifg.nrows is None)

		# test with tolerance from base 90m cell
		self.assertTrue(self.ifg.y_size > 88.0) # within 2% of cells over sydney?
		self.assertTrue(self.ifg.y_size < 92.0, 'Got %s' % self.ifg.y_size)

		syd_width = 76.9 # from nearby pirate coords
		self.assertTrue(self.ifg.x_size > 0.97 * syd_width) # ~3% tolerance
		self.assertTrue(self.ifg.x_size < 1.03 * syd_width)

	def test_centre_latlong(self):
		self.ifg.open()
		lat_exp = self.ifg.y_first + ((self.ifg.nrows / 2) * self.ifg.y_step)
		long_exp = self.ifg.x_first + ((self.ifg.ncols / 2) * self.ifg.x_step)
		self.assertEqual(lat_exp, self.ifg.lat_centre)
		self.assertEqual(long_exp, self.ifg.long_centre)

	def test_centre_cell(self):
		self.ifg.open()
		self.assertEqual(self.ifg.x_centre, 23)
		self.assertEqual(self.ifg.y_centre, 36)

class IncidenceFileTests(unittest.TestCase):
	'''Unit tests for the interface to Incidence files'''

	def setUp(self):
		raise NotImplementedError
		self.inc = Incidence(join(INCID_TEST_DIR, '128x2.tif'))
		self.inc.open()

	def test_incidence_data(self):
		# check incidences rises while traversing the scene
		data = self.inc.incidence_data
		diff = data.ptp()
		self.assertTrue(diff > 0.5, "Got ptp() diff of %s" % diff)

		# ascending pass, values should increase from W->E across scene
		for i in range(2):
			d = data[i]
			self.assertFalse((d == 0).any()) # ensure no NODATA
			self.assertFalse((isnan(d)).any())

			diff = array([d[i+1] - d[i] for i in range(len(d)-1)])
			res = abs(diff[diff < 0])
			self.assertTrue((res < 1e-4).all()) # TODO: check if this is normal

	def test_azimuth_data(self):
		# ensure azimuth is fairly constant

		az = self.inc.azimuth_data
		self.assertFalse((az == 0).all())
		az = az[az != 0] # filter NODATA cells

		# azimuth should be relatively constant
		ptp = az.ptp()
		self.assertTrue(ptp < 0.1, msg="min -> max diff is %s" % ptp)

class DEMTests(unittest.TestCase):
	'''Unit tests for the generic DEM class.'''

	def setUp(self):
		self.ras = DEM(SYD_TEST_DEM_TIF)

	def test_create_raster(self):
		self.assertTrue(os.path.exists(self.ras.data_path)) # validate header path

	def test_headers_as_attr(self):
		self.ras.open()
		attrs = ['ncols', 'nrows', 'x_first', 'x_step', 'y_first', 'y_step' ]

		# TODO: are 'projection' and 'datum' attrs needed?
		for a in attrs:
			self.assertTrue(getattr(self.ras, a) is not None)

	def test_is_dem(self):
		self.ras = DEM(join(SYD_TEST_TIF, 'geo_060619-061002.tif'))
		self.assertFalse(hasattr(self.ras, 'datum'))

	def test_open(self):
		self.assertTrue(self.ras.dataset is None)
		self.ras.open()
		self.assertTrue(self.ras.dataset is not None)
		self.assertTrue(isinstance(self.ras.dataset, Dataset))

		# ensure open cannot be called twice
		self.failUnlessRaises(RasterException, self.ras.open)

	def test_band(self):
		# test accessing bands with open and unopened datasets
		try:
			_ = self.ras.height_band
			self.fail("Should not be able to access band without open dataset")
		except RasterException:
			pass

		self.ras.open()
		data = self.ras.height_band.ReadAsArray()
		self.assertEqual(data.shape, (72, 47) )

if __name__ == "__main__":
	unittest.main()
<commit_msg>Temporarily commented out incidence file tests (for future implementation)<commit_after>'''
Unittests for shared PyRate objects.
Created on 12/09/2012
@author: bpd900
'''

# FIXME: replace ROIPAC dependencies with PyRate GeoTIFF files

import os
import shutil
import unittest
from os.path import join, basename
from stat import S_IRGRP, S_IWGRP, S_IWOTH, S_IROTH, S_IRUSR, S_IWUSR

from itertools import product
from numpy import array, isnan, where, nan
from numpy.testing import assert_array_equal

from gdal import Open, Dataset, UseExceptions
UseExceptions()

from pyrate.shared import Ifg, DEM, RasterException, Incidence
from common import SYD_TEST_TIF, INCID_TEST_DIR, SYD_TEST_DEM_TIF

# FIXME: refactor the ifg.open() calls
class IfgTests(unittest.TestCase):
	'''Unit tests for the Ifg/interferogram class.'''

	def setUp(self):
		self.ifg = Ifg(join(SYD_TEST_TIF, 'geo_060619-061002.tif'))

	def test_create_ifg(self):
		self.assertTrue(os.path.exists(self.ifg.data_path)) # validate data path

	def test_headers_as_attr(self):
		self.ifg.open()
		for a in ['ncols', 'nrows', 'x_first', 'x_step',
				'y_first', 'y_step', 'wavelength', 'master', 'slave']:
			self.assertTrue(getattr(self.ifg, a) is not None)

	def test_open(self):
		self.assertTrue(self.ifg.dataset is None)
		self.assertTrue(self.ifg.is_open is False)
		self.ifg.open(readonly=True)
		self.assertTrue(self.ifg.dataset is not None)
		self.assertTrue(self.ifg.is_open is True)
		self.assertTrue(isinstance(self.ifg.dataset, Dataset))

		# ensure open cannot be called twice
		self.failUnlessRaises(RasterException, self.ifg.open, True)

	def test_write(self):
		base = "/tmp"
		src = self.ifg.data_path
		dest = join(base, basename(self.ifg.data_path))

		# shutil.copy needs to copy writeable permission from src
		os.chmod(src, S_IRGRP | S_IWGRP | S_IWOTH | S_IROTH | S_IRUSR | S_IWUSR)
		shutil.copy(src, dest)
		os.chmod(src, S_IRGRP | S_IROTH | S_IRUSR) # revert

		i = Ifg(dest)
		i.open()
		i.phase_data[0,1:] = nan
		i.write_phase()
		del i

		# reopen to ensure data/nans can be read back out
		i = Ifg(dest)
		i.open(readonly=True)
		assert_array_equal(True, isnan(i.phase_data[0,1:]) )
		os.remove(dest)

	def test_readonly_permission_failure(self):
		# ensure failure if opening R/O permission file as writeable/GA_Update
		self.assertRaises(IOError, self.ifg.open, False)

	def test_write_fails_on_readonly(self):
		# check readonly status is same before and after open() for readonly file
		self.assertTrue(self.ifg.is_read_only)
		self.ifg.open(readonly=True)
		self.assertTrue(self.ifg.is_read_only)
		self.assertRaises(IOError, self.ifg.write_phase)

	def test_convert_to_nans(self):
		self.ifg.open()
		self.ifg.convert_to_nans(0)
		self.assertTrue(self.ifg.nan_converted)

	def test_xylast(self):
		# ensure the X|Y_LAST header element has been created
		self.ifg.open()
		self.assertAlmostEqual(self.ifg.x_last, 150.9491667)
		self.assertAlmostEqual(self.ifg.y_last, -34.23)

	def test_num_cells(self):
		# test cell size from header elements
		self.ifg.open()
		data = self.ifg.phase_band.ReadAsArray()
		ys, xs = data.shape
		exp_ncells = ys * xs
		self.assertEqual(exp_ncells, self.ifg.num_cells)

	def test_shape(self):
		self.ifg.open()
		self.assertEqual(self.ifg.shape, self.ifg.phase_data.shape)

# 	def test_amp_band(self):
# 		try:
# 			_ = self.ifg.amp_band
# 			self.fail("Should not be able to access band without open dataset")
# 		except RasterException:
# 			pass
#
# 		self.ifg.open()
# 		data = self.ifg.amp_band.ReadAsArray()
# 		self.assertEqual(data.shape, (72, 47) )

	def test_phase_band(self):
		try:
			_ = self.ifg.phase_band
			self.fail("Should not be able to access band without open dataset")
		except RasterException:
			pass

		self.ifg.open()
		data = self.ifg.phase_band.ReadAsArray()
		self.assertEqual(data.shape, (72, 47) )

	def test_nan_count(self):
		self.ifg.open()
		num_nan = 0
		for row in self.ifg.phase_data:
			for v in row:
				if isnan(v):
					num_nan += 1

		self.assertEqual(num_nan, self.ifg.nan_count)

	def test_nan_fraction(self):
		try:
			# NB: self.assertRaises doesn't work here (as it is a property?)
			_ = self.ifg.nan_fraction
			self.fail("Shouldn't be able to call nan_fraction() with unopened Ifg")
		except RasterException:
			pass

		# NB: source data lacks 0 -> NaN conversion
		self.ifg.open()
		data = self.ifg.dataset.GetRasterBand(1).ReadAsArray()
		data = where(data == 0, nan, data) # fake 0 -> nan for the count below

		# manually count # nan cells
		nans = 0
		ys, xs = data.shape
		for y, x in product(xrange(ys), xrange(xs)):
			if isnan(data[y, x]):
				nans += 1
		del data

		num_cells = float(ys * xs)
		self.assertTrue(nans > 0)
		self.assertTrue(nans <= num_cells)
		self.assertEqual(nans / num_cells, self.ifg.nan_fraction)

	def test_phase_data_properties(self):
		# Use raw GDAL to isolate raster reading from Ifg functionality
		ds = Open(self.ifg.data_path)
		data = ds.GetRasterBand(1).ReadAsArray()
		del ds

		self.ifg.open()

		# test full array and row by row access
		assert_array_equal(data, self.ifg.phase_data)
		for y, row in enumerate(self.ifg.phase_rows):
			assert_array_equal(data[y], row)

		# test the data is cached if changed
		crd = (5,4)
		orig = self.ifg.phase_data[crd]
		self.ifg.phase_data[crd] *= 2
		nv = self.ifg.phase_data[crd] # pull new value out again
		self.assertEqual(nv, 2 * orig)

	def test_xy_size(self):
		self.ifg.open()
		self.assertFalse(self.ifg.ncols is None)
		self.assertFalse(self.ifg.nrows is None)

		# test with tolerance from base 90m cell
		self.assertTrue(self.ifg.y_size > 88.0) # within 2% of cells over sydney?
		self.assertTrue(self.ifg.y_size < 92.0, 'Got %s' % self.ifg.y_size)

		syd_width = 76.9 # from nearby pirate coords
		self.assertTrue(self.ifg.x_size > 0.97 * syd_width) # ~3% tolerance
		self.assertTrue(self.ifg.x_size < 1.03 * syd_width)

	def test_centre_latlong(self):
		self.ifg.open()
		lat_exp = self.ifg.y_first + ((self.ifg.nrows / 2) * self.ifg.y_step)
		long_exp = self.ifg.x_first + ((self.ifg.ncols / 2) * self.ifg.x_step)
		self.assertEqual(lat_exp, self.ifg.lat_centre)
		self.assertEqual(long_exp, self.ifg.long_centre)

	def test_centre_cell(self):
		self.ifg.open()
		self.assertEqual(self.ifg.x_centre, 23)
		self.assertEqual(self.ifg.y_centre, 36)

# FIXME:
# class IncidenceFileTests(unittest.TestCase):
# 	'Unit tests to verify operations on GeoTIFF format Incidence rasters'
#
# 	def setUp(self):
# 		raise NotImplementedError
# 		self.inc = Incidence(join(INCID_TEST_DIR, '128x2.tif'))
# 		self.inc.open()
#
#
# 	def test_incidence_data(self):
# 		# check incidences rises while traversing the scene
# 		data = self.inc.incidence_data
# 		diff = data.ptp()
# 		self.assertTrue(diff > 0.5, "Got ptp() diff of %s" % diff)
#
# 		# ascending pass, values should increase from W->E across scene
# 		for i in range(2):
# 			d = data[i]
# 			self.assertFalse((d == 0).any()) # ensure no NODATA
# 			self.assertFalse((isnan(d)).any())
#
# 			diff = array([d[i+1] - d[i] for i in range(len(d)-1)])
# 			res = abs(diff[diff < 0])
# 			self.assertTrue((res < 1e-4).all()) # TODO: check if this is normal
#
#
# 	def test_azimuth_data(self):
# 		# ensure azimuth is fairly constant
#
# 		az = self.inc.azimuth_data
# 		self.assertFalse((az == 0).all())
# 		az = az[az != 0] # filter NODATA cells
#
# 		# azimuth should be relatively constant
# 		ptp = az.ptp()
# 		self.assertTrue(ptp < 0.1, msg="min -> max diff is %s" % ptp)

class DEMTests(unittest.TestCase):
	'Unit tests to verify operations on GeoTIFF format DEMs'

	def setUp(self):
		self.ras = DEM(SYD_TEST_DEM_TIF)

	def test_create_raster(self):
		self.assertTrue(os.path.exists(self.ras.data_path)) # validate header path

	def test_headers_as_attr(self):
		self.ras.open()
		attrs = ['ncols', 'nrows', 'x_first', 'x_step', 'y_first', 'y_step' ]

		# TODO: are 'projection' and 'datum' attrs needed?
		for a in attrs:
			self.assertTrue(getattr(self.ras, a) is not None)

	def test_is_dem(self):
		self.ras = DEM(join(SYD_TEST_TIF, 'geo_060619-061002.tif'))
		self.assertFalse(hasattr(self.ras, 'datum'))

	def test_open(self):
		self.assertTrue(self.ras.dataset is None)
		self.ras.open()
		self.assertTrue(self.ras.dataset is not None)
		self.assertTrue(isinstance(self.ras.dataset, Dataset))

		# ensure open cannot be called twice
		self.failUnlessRaises(RasterException, self.ras.open)

	def test_band(self):
		# test accessing bands with open and unopened datasets
		try:
			_ = self.ras.height_band
			self.fail("Should not be able to access band without open dataset")
		except RasterException:
			pass

		self.ras.open()
		data = self.ras.height_band.ReadAsArray()
		self.assertEqual(data.shape, (72, 47) )

if __name__ == "__main__":
	unittest.main()
<|endoftext|>


<commit_before># Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for distributed_file_utils."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
from tensorflow.python.distribute import distributed_file_utils
from tensorflow.python.eager import test

class DistributedFileUtilsTest(test.TestCase):

  class MockedExtended(object):
    pass

  class MockedChiefStrategy(object):

    def __init__(self):
      self.extended = DistributedFileUtilsTest.MockedExtended()
      self.extended._in_multi_worker_mode = lambda: True
      self.extended.should_checkpoint = True

  class MockedWorkerStrategy(object):

    def __init__(self):
      self.extended = DistributedFileUtilsTest.MockedExtended()
      self.extended._in_multi_worker_mode = lambda: True
      self.extended.should_checkpoint = False
      self.extended._task_id = 3

  class MockedSingleWorkerStrategy(object):

    def __init__(self):
      self.extended = DistributedFileUtilsTest.MockedExtended()
      self.extended._in_multi_worker_mode = lambda: False

  def _write_dummy_file(self, file_to_write):
    with open(file_to_write, 'w') as f:
      f.write('foo bar')

  def testChiefWriteDirAndFilePath(self):
    dirpath = self.get_temp_dir()
    filepath = os.path.join(dirpath, 'foo.bar')
    strategy = DistributedFileUtilsTest.MockedChiefStrategy()
    self.assertEqual(
        distributed_file_utils.write_filepath(filepath, strategy), filepath)
    self.assertEqual(
        distributed_file_utils.write_dirpath(dirpath, strategy), dirpath)

  def testWorkerWriteDirAndFilePath(self):
    dirpath = self.get_temp_dir()
    filepath = os.path.join(dirpath, 'foo.bar')
    strategy = DistributedFileUtilsTest.MockedWorkerStrategy()
    self.assertEqual(
        distributed_file_utils.write_filepath(filepath, strategy),
        os.path.join(dirpath, 'workertemp_3/foo.bar'))
    self.assertEqual(
        distributed_file_utils.write_dirpath(dirpath, strategy),
        os.path.join(dirpath, 'workertemp_3'))

  def testChiefDoesNotRemoveDirAndFilePath(self):
    temp_dir = self.get_temp_dir()
    strategy = DistributedFileUtilsTest.MockedChiefStrategy()
    dir_to_write = distributed_file_utils.write_dirpath(temp_dir, strategy)
    file_to_write = os.path.join(dir_to_write, 'tmp')
    self.assertFalse(os.path.exists(file_to_write))
    self._write_dummy_file(file_to_write)
    self.assertTrue(os.path.exists(file_to_write))
    distributed_file_utils.remove_temp_dir_with_filepath(
        file_to_write, strategy)
    self.assertTrue(os.path.exists(file_to_write))

  def testWorkerDoesRemoveFilePath(self):
    temp_dir = self.get_temp_dir()
    strategy = DistributedFileUtilsTest.MockedWorkerStrategy()
    dir_to_write = distributed_file_utils.write_dirpath(temp_dir, strategy)
    file_to_write = os.path.join(dir_to_write, 'tmp')
    self.assertFalse(os.path.exists(file_to_write))
    self._write_dummy_file(file_to_write)
    self.assertTrue(os.path.exists(file_to_write))
    distributed_file_utils.remove_temp_dir_with_filepath(
        file_to_write, strategy)
    self.assertFalse(os.path.exists(file_to_write))

  def testWorkerDoesRemoveDirPath(self):
    temp_dir = self.get_temp_dir()
    strategy = DistributedFileUtilsTest.MockedWorkerStrategy()
    dir_to_write = distributed_file_utils.write_dirpath(temp_dir, strategy)
    file_to_write = os.path.join(dir_to_write, 'tmp')
    self.assertFalse(os.path.exists(file_to_write))
    self._write_dummy_file(file_to_write)
    self.assertTrue(os.path.exists(file_to_write))
    distributed_file_utils.remove_temp_dirpath(temp_dir, strategy)
    self.assertFalse(os.path.exists(file_to_write))

if __name__ == '__main__':
  test.main()
<commit_msg>Use path.join to make sure that the test passes on windows as well.<commit_after># Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for distributed_file_utils."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
from tensorflow.python.distribute import distributed_file_utils
from tensorflow.python.eager import test

class DistributedFileUtilsTest(test.TestCase):

  class MockedExtended(object):
    pass

  class MockedChiefStrategy(object):

    def __init__(self):
      self.extended = DistributedFileUtilsTest.MockedExtended()
      self.extended._in_multi_worker_mode = lambda: True
      self.extended.should_checkpoint = True

  class MockedWorkerStrategy(object):

    def __init__(self):
      self.extended = DistributedFileUtilsTest.MockedExtended()
      self.extended._in_multi_worker_mode = lambda: True
      self.extended.should_checkpoint = False
      self.extended._task_id = 3

  class MockedSingleWorkerStrategy(object):

    def __init__(self):
      self.extended = DistributedFileUtilsTest.MockedExtended()
      self.extended._in_multi_worker_mode = lambda: False

  def _write_dummy_file(self, file_to_write):
    with open(file_to_write, 'w') as f:
      f.write('foo bar')

  def testChiefWriteDirAndFilePath(self):
    dirpath = self.get_temp_dir()
    filepath = os.path.join(dirpath, 'foo.bar')
    strategy = DistributedFileUtilsTest.MockedChiefStrategy()
    self.assertEqual(
        distributed_file_utils.write_filepath(filepath, strategy), filepath)
    self.assertEqual(
        distributed_file_utils.write_dirpath(dirpath, strategy), dirpath)

  def testWorkerWriteDirAndFilePath(self):
    dirpath = self.get_temp_dir()
    filepath = os.path.join(dirpath, 'foo.bar')
    strategy = DistributedFileUtilsTest.MockedWorkerStrategy()
    self.assertEqual(
        distributed_file_utils.write_filepath(filepath, strategy),
        os.path.join(dirpath, 'workertemp_3', 'foo.bar'))
    self.assertEqual(
        distributed_file_utils.write_dirpath(dirpath, strategy),
        os.path.join(dirpath, 'workertemp_3'))

  def testChiefDoesNotRemoveDirAndFilePath(self):
    temp_dir = self.get_temp_dir()
    strategy = DistributedFileUtilsTest.MockedChiefStrategy()
    dir_to_write = distributed_file_utils.write_dirpath(temp_dir, strategy)
    file_to_write = os.path.join(dir_to_write, 'tmp')
    self.assertFalse(os.path.exists(file_to_write))
    self._write_dummy_file(file_to_write)
    self.assertTrue(os.path.exists(file_to_write))
    distributed_file_utils.remove_temp_dir_with_filepath(
        file_to_write, strategy)
    self.assertTrue(os.path.exists(file_to_write))

  def testWorkerDoesRemoveFilePath(self):
    temp_dir = self.get_temp_dir()
    strategy = DistributedFileUtilsTest.MockedWorkerStrategy()
    dir_to_write = distributed_file_utils.write_dirpath(temp_dir, strategy)
    file_to_write = os.path.join(dir_to_write, 'tmp')
    self.assertFalse(os.path.exists(file_to_write))
    self._write_dummy_file(file_to_write)
    self.assertTrue(os.path.exists(file_to_write))
    distributed_file_utils.remove_temp_dir_with_filepath(
        file_to_write, strategy)
    self.assertFalse(os.path.exists(file_to_write))

  def testWorkerDoesRemoveDirPath(self):
    temp_dir = self.get_temp_dir()
    strategy = DistributedFileUtilsTest.MockedWorkerStrategy()
    dir_to_write = distributed_file_utils.write_dirpath(temp_dir, strategy)
    file_to_write = os.path.join(dir_to_write, 'tmp')
    self.assertFalse(os.path.exists(file_to_write))
    self._write_dummy_file(file_to_write)
    self.assertTrue(os.path.exists(file_to_write))
    distributed_file_utils.remove_temp_dirpath(temp_dir, strategy)
    self.assertFalse(os.path.exists(file_to_write))

if __name__ == '__main__':
  test.main()
<|endoftext|>


<commit_before><commit_msg>remove extern<commit_after><|endoftext|>


<commit_before># Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import hashlib
import logging

from django.conf import settings
from django.contrib.auth import models

from keystoneclient import exceptions as keystone_exceptions

from openstack_auth import utils

LOG = logging.getLogger(__name__)

def set_session_from_user(request, user):
    request.session['token'] = user.token
    request.session['user_id'] = user.id
    request.session['region_endpoint'] = user.endpoint
    request.session['services_region'] = user.services_region
    # Update the user object cached in the request
    request._cached_user = user
    request.user = user

def create_user_from_token(request, token, endpoint, services_region=None):
    return User(id=token.user['id'],
                token=token,
                user=token.user['name'],
                user_domain_id=token.user_domain_id,
                project_id=token.project['id'],
                project_name=token.project['name'],
                domain_id=token.domain['id'],
                domain_name=token.domain['name'],
                enabled=True,
                service_catalog=token.serviceCatalog,
                roles=token.roles,
                endpoint=endpoint,
                services_region=services_region)

class Token(object):
    """Token object that encapsulates the auth_ref (AccessInfo)from keystone
       client.

       Added for maintaining backward compatibility with horizon that expects
       Token object in the user object.
    """
    def __init__(self, auth_ref):
        # User-related attributes
        user = {}
        user['id'] = auth_ref.user_id
        user['name'] = auth_ref.username
        self.user = user
        self.user_domain_id = auth_ref.user_domain_id

        # Token-related attributes
        self.id = auth_ref.auth_token
        if len(self.id) > 32:
            self.id = hashlib.md5(self.id).hexdigest()
        self.expires = auth_ref.expires

        # Project-related attributes
        project = {}
        project['id'] = auth_ref.project_id
        project['name'] = auth_ref.project_name
        self.project = project
        self.tenant = self.project

        # Domain-related attributes
        domain = {}
        domain['id'] = auth_ref.domain_id
        domain['name'] = auth_ref.domain_name
        self.domain = domain

        if auth_ref.version == 'v2.0':
            self.roles = auth_ref['user'].get('roles', [])
        else:
            self.roles = auth_ref.get('roles', [])

        if utils.get_keystone_version() < 3:
            self.serviceCatalog = auth_ref.get('serviceCatalog', [])
        else:
            self.serviceCatalog = auth_ref.get('catalog', [])

class User(models.AnonymousUser):
    """A User class with some extra special sauce for Keystone.

    In addition to the standard Django user attributes, this class also has
    the following:

    .. attribute:: token

        The Keystone token object associated with the current user/tenant.

        The token object is deprecated, user auth_ref instead.

    .. attribute:: tenant_id

        The id of the Keystone tenant for the current user/token.

        The tenant_id keyword argument is deprecated, use project_id instead.

    .. attribute:: tenant_name

        The name of the Keystone tenant for the current user/token.

        The tenant_name keyword argument is deprecated, use project_name
        instead.

    .. attribute:: project_id

        The id of the Keystone project for the current user/token.

    .. attribute:: project_name

        The name of the Keystone project for the current user/token.

    .. attribute:: service_catalog

        The ``ServiceCatalog`` data returned by Keystone.

    .. attribute:: roles

        A list of dictionaries containing role names and ids as returned
        by Keystone.

    .. attribute:: services_region

        A list of non-identity service endpoint regions extracted from the
        service catalog.

    .. attribute:: user_domain_id

        The domain id of the current user.

    .. attribute:: domain_id

        The id of the Keystone domain scoped for the current user/token.

    """
    def __init__(self, id=None, token=None, user=None, tenant_id=None,
                 service_catalog=None, tenant_name=None, roles=None,
                 authorized_tenants=None, endpoint=None, enabled=False,
                 services_region=None, user_domain_id=None, domain_id=None,
                 domain_name=None, project_id=None, project_name=None):
        self.id = id
        self.pk = id
        self.token = token
        self.username = user
        self.user_domain_id = user_domain_id
        self.domain_id = domain_id
        self.domain_name = domain_name
        self.project_id = project_id or tenant_id
        self.project_name = project_name or tenant_name
        self.service_catalog = service_catalog
        self._services_region = (services_region or
                                 self.default_services_region())
        self.roles = roles or []
        self.endpoint = endpoint
        self.enabled = enabled
        self._authorized_tenants = authorized_tenants

        # List of variables to be deprecated.
        self.tenant_id = self.project_id
        self.tenant_name = self.project_name

    def __unicode__(self):
        return self.username

    def __repr__(self):
        return "<%s: %s>" % (self.__class__.__name__, self.username)

    def is_token_expired(self):
        """Determine if the token is expired.

        Returns ``True`` if the token is expired, ``False`` if not, and
        ``None`` if there is no token set.
        """
        if self.token is None:
            return None
        return not utils.check_token_expiration(self.token)

    def is_authenticated(self):
        """Checks for a valid token that has not yet expired."""
        return (self.token is not None and
                utils.check_token_expiration(self.token))

    def is_anonymous(self):
        """Return if the user is not authenticated.

        Returns ``True`` if not authenticated,``False`` otherwise.
        """
        return not self.is_authenticated()

    @property
    def is_active(self):
        return self.enabled

    @property
    def is_superuser(self):
        """Evaluates whether this user has admin privileges.

        Returns ``True`` or ``False``.
        """
        return 'admin' in [role['name'].lower() for role in self.roles]

    @property
    def authorized_tenants(self):
        """Returns a memoized list of tenants this user may access."""
        insecure = getattr(settings, 'OPENSTACK_SSL_NO_VERIFY', False)
        ca_cert = getattr(settings, "OPENSTACK_SSL_CACERT", None)

        if self.is_authenticated() and self._authorized_tenants is None:
            endpoint = self.endpoint
            token = self.token
            try:
                self._authorized_tenants = utils.get_project_list(
                    user_id=self.id,
                    auth_url=endpoint,
                    token=token.id,
                    insecure=insecure,
                    cacert=ca_cert,
                    debug=settings.DEBUG)
            except (keystone_exceptions.ClientException,
                    keystone_exceptions.AuthorizationFailure):
                LOG.exception('Unable to retrieve project list.')
        return self._authorized_tenants or []

    @authorized_tenants.setter
    def authorized_tenants(self, tenant_list):
        self._authorized_tenants = tenant_list

    def default_services_region(self):
        """Returns the first endpoint region for first non-identity service.

        Extracted from the service catalog.
        """
        if self.service_catalog:
            for service in self.service_catalog:
                if service['type'] == 'identity':
                    continue
                for endpoint in service['endpoints']:
                    return endpoint['region']
        return None

    @property
    def services_region(self):
        return self._services_region

    @services_region.setter
    def services_region(self, region):
        self._services_region = region

    @property
    def available_services_regions(self):
        """Returns list of unique region name values in service catalog."""
        regions = []
        if self.service_catalog:
            for service in self.service_catalog:
                if service['type'] == 'identity':
                    continue
                for endpoint in service['endpoints']:
                    if endpoint['region'] not in regions:
                        regions.append(endpoint['region'])
        return regions

    def save(*args, **kwargs):
        # Presume we can't write to Keystone.
        pass

    def delete(*args, **kwargs):
        # Presume we can't write to Keystone.
        pass

    # Check for OR'd permission rules, check that user has one of the
    # required permission.
    def has_a_matching_perm(self, perm_list, obj=None):
        """Returns True if the user has one of the specified permissions.

        If object is passed, it checks if the user has any of the required
        perms for this object.
        """
        # If there are no permissions to check, just return true
        if not perm_list:
            return True
        # Check that user has at least one of the required permissions.
        for perm in perm_list:
            if self.has_perm(perm, obj):
                return True
        return False

    # Override the default has_perms method. Allowing for more
    # complex combinations of permissions.  Will check for logical AND of
    # all top level permissions.  Will use logical OR for all first level
    # tuples (check that use has one permissions in the tuple)
    #
    # Examples:
    #   Checks for all required permissions
    #   ('openstack.roles.admin', 'openstack.roles.L3-support')
    #
    #   Checks for admin AND (L2 or L3)
    #   ('openstack.roles.admin', ('openstack.roles.L3-support',
    #                              'openstack.roles.L2-support'),)
    def has_perms(self, perm_list, obj=None):
        """Returns True if the user has all of the specified permissions.

        Tuples in the list will possess the required permissions if
        the user has a permissions matching one of the elements of
        that tuple
        """
        # If there are no permissions to check, just return true
        if not perm_list:
            return True
        for perm in perm_list:
            if isinstance(perm, basestring):
                # check that the permission matches
                if not self.has_perm(perm, obj):
                    return False
            else:
                # check that a permission in the tuple matches
                if not self.has_a_matching_perm(perm, obj):
                    return False
        return True
<commit_msg>Configurable token hashing algorithm<commit_after># Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import hashlib
import logging

from django.conf import settings
from django.contrib.auth import models

from keystoneclient import exceptions as keystone_exceptions

from openstack_auth import utils

LOG = logging.getLogger(__name__)

def set_session_from_user(request, user):
    request.session['token'] = user.token
    request.session['user_id'] = user.id
    request.session['region_endpoint'] = user.endpoint
    request.session['services_region'] = user.services_region
    # Update the user object cached in the request
    request._cached_user = user
    request.user = user

def create_user_from_token(request, token, endpoint, services_region=None):
    return User(id=token.user['id'],
                token=token,
                user=token.user['name'],
                user_domain_id=token.user_domain_id,
                project_id=token.project['id'],
                project_name=token.project['name'],
                domain_id=token.domain['id'],
                domain_name=token.domain['name'],
                enabled=True,
                service_catalog=token.serviceCatalog,
                roles=token.roles,
                endpoint=endpoint,
                services_region=services_region)

class Token(object):
    """Token object that encapsulates the auth_ref (AccessInfo)from keystone
       client.

       Added for maintaining backward compatibility with horizon that expects
       Token object in the user object.
    """
    def __init__(self, auth_ref):
        # User-related attributes
        user = {}
        user['id'] = auth_ref.user_id
        user['name'] = auth_ref.username
        self.user = user
        self.user_domain_id = auth_ref.user_domain_id

        # Token-related attributes
        self.id = auth_ref.auth_token
        if len(self.id) > 64:
            algorithm = getattr(settings, 'OPENSTACK_TOKEN_HASH_ALGORITHM',
                                'md5')
            hasher = hashlib.new(algorithm)
            hasher.update(self.id)
            self.id = hasher.hexdigest()
        self.expires = auth_ref.expires

        # Project-related attributes
        project = {}
        project['id'] = auth_ref.project_id
        project['name'] = auth_ref.project_name
        self.project = project
        self.tenant = self.project

        # Domain-related attributes
        domain = {}
        domain['id'] = auth_ref.domain_id
        domain['name'] = auth_ref.domain_name
        self.domain = domain

        if auth_ref.version == 'v2.0':
            self.roles = auth_ref['user'].get('roles', [])
        else:
            self.roles = auth_ref.get('roles', [])

        if utils.get_keystone_version() < 3:
            self.serviceCatalog = auth_ref.get('serviceCatalog', [])
        else:
            self.serviceCatalog = auth_ref.get('catalog', [])

class User(models.AnonymousUser):
    """A User class with some extra special sauce for Keystone.

    In addition to the standard Django user attributes, this class also has
    the following:

    .. attribute:: token

        The Keystone token object associated with the current user/tenant.

        The token object is deprecated, user auth_ref instead.

    .. attribute:: tenant_id

        The id of the Keystone tenant for the current user/token.

        The tenant_id keyword argument is deprecated, use project_id instead.

    .. attribute:: tenant_name

        The name of the Keystone tenant for the current user/token.

        The tenant_name keyword argument is deprecated, use project_name
        instead.

    .. attribute:: project_id

        The id of the Keystone project for the current user/token.

    .. attribute:: project_name

        The name of the Keystone project for the current user/token.

    .. attribute:: service_catalog

        The ``ServiceCatalog`` data returned by Keystone.

    .. attribute:: roles

        A list of dictionaries containing role names and ids as returned
        by Keystone.

    .. attribute:: services_region

        A list of non-identity service endpoint regions extracted from the
        service catalog.

    .. attribute:: user_domain_id

        The domain id of the current user.

    .. attribute:: domain_id

        The id of the Keystone domain scoped for the current user/token.

    """
    def __init__(self, id=None, token=None, user=None, tenant_id=None,
                 service_catalog=None, tenant_name=None, roles=None,
                 authorized_tenants=None, endpoint=None, enabled=False,
                 services_region=None, user_domain_id=None, domain_id=None,
                 domain_name=None, project_id=None, project_name=None):
        self.id = id
        self.pk = id
        self.token = token
        self.username = user
        self.user_domain_id = user_domain_id
        self.domain_id = domain_id
        self.domain_name = domain_name
        self.project_id = project_id or tenant_id
        self.project_name = project_name or tenant_name
        self.service_catalog = service_catalog
        self._services_region = (services_region or
                                 self.default_services_region())
        self.roles = roles or []
        self.endpoint = endpoint
        self.enabled = enabled
        self._authorized_tenants = authorized_tenants

        # List of variables to be deprecated.
        self.tenant_id = self.project_id
        self.tenant_name = self.project_name

    def __unicode__(self):
        return self.username

    def __repr__(self):
        return "<%s: %s>" % (self.__class__.__name__, self.username)

    def is_token_expired(self):
        """Determine if the token is expired.

        Returns ``True`` if the token is expired, ``False`` if not, and
        ``None`` if there is no token set.
        """
        if self.token is None:
            return None
        return not utils.check_token_expiration(self.token)

    def is_authenticated(self):
        """Checks for a valid token that has not yet expired."""
        return (self.token is not None and
                utils.check_token_expiration(self.token))

    def is_anonymous(self):
        """Return if the user is not authenticated.

        Returns ``True`` if not authenticated,``False`` otherwise.
        """
        return not self.is_authenticated()

    @property
    def is_active(self):
        return self.enabled

    @property
    def is_superuser(self):
        """Evaluates whether this user has admin privileges.

        Returns ``True`` or ``False``.
        """
        return 'admin' in [role['name'].lower() for role in self.roles]

    @property
    def authorized_tenants(self):
        """Returns a memoized list of tenants this user may access."""
        insecure = getattr(settings, 'OPENSTACK_SSL_NO_VERIFY', False)
        ca_cert = getattr(settings, "OPENSTACK_SSL_CACERT", None)

        if self.is_authenticated() and self._authorized_tenants is None:
            endpoint = self.endpoint
            token = self.token
            try:
                self._authorized_tenants = utils.get_project_list(
                    user_id=self.id,
                    auth_url=endpoint,
                    token=token.id,
                    insecure=insecure,
                    cacert=ca_cert,
                    debug=settings.DEBUG)
            except (keystone_exceptions.ClientException,
                    keystone_exceptions.AuthorizationFailure):
                LOG.exception('Unable to retrieve project list.')
        return self._authorized_tenants or []

    @authorized_tenants.setter
    def authorized_tenants(self, tenant_list):
        self._authorized_tenants = tenant_list

    def default_services_region(self):
        """Returns the first endpoint region for first non-identity service.

        Extracted from the service catalog.
        """
        if self.service_catalog:
            for service in self.service_catalog:
                if service['type'] == 'identity':
                    continue
                for endpoint in service['endpoints']:
                    return endpoint['region']
        return None

    @property
    def services_region(self):
        return self._services_region

    @services_region.setter
    def services_region(self, region):
        self._services_region = region

    @property
    def available_services_regions(self):
        """Returns list of unique region name values in service catalog."""
        regions = []
        if self.service_catalog:
            for service in self.service_catalog:
                if service['type'] == 'identity':
                    continue
                for endpoint in service['endpoints']:
                    if endpoint['region'] not in regions:
                        regions.append(endpoint['region'])
        return regions

    def save(*args, **kwargs):
        # Presume we can't write to Keystone.
        pass

    def delete(*args, **kwargs):
        # Presume we can't write to Keystone.
        pass

    # Check for OR'd permission rules, check that user has one of the
    # required permission.
    def has_a_matching_perm(self, perm_list, obj=None):
        """Returns True if the user has one of the specified permissions.

        If object is passed, it checks if the user has any of the required
        perms for this object.
        """
        # If there are no permissions to check, just return true
        if not perm_list:
            return True
        # Check that user has at least one of the required permissions.
        for perm in perm_list:
            if self.has_perm(perm, obj):
                return True
        return False

    # Override the default has_perms method. Allowing for more
    # complex combinations of permissions.  Will check for logical AND of
    # all top level permissions.  Will use logical OR for all first level
    # tuples (check that use has one permissions in the tuple)
    #
    # Examples:
    #   Checks for all required permissions
    #   ('openstack.roles.admin', 'openstack.roles.L3-support')
    #
    #   Checks for admin AND (L2 or L3)
    #   ('openstack.roles.admin', ('openstack.roles.L3-support',
    #                              'openstack.roles.L2-support'),)
    def has_perms(self, perm_list, obj=None):
        """Returns True if the user has all of the specified permissions.

        Tuples in the list will possess the required permissions if
        the user has a permissions matching one of the elements of
        that tuple
        """
        # If there are no permissions to check, just return true
        if not perm_list:
            return True
        for perm in perm_list:
            if isinstance(perm, basestring):
                # check that the permission matches
                if not self.has_perm(perm, obj):
                    return False
            else:
                # check that a permission in the tuple matches
                if not self.has_a_matching_perm(perm, obj):
                    return False
        return True
<|endoftext|>


<commit_before># Copyright 2016 Internap.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from hamcrest import assert_that, contains_inanyorder

from netman.core.objects.exceptions import UnknownVlan
from tests.adapters.configured_test_case import ConfiguredTestCase

class GetVlanInterfacesTest(ConfiguredTestCase):
    _dev_sample = "cisco"

    def setUp(self):
        super(GetVlanInterfacesTest, self).setUp()

    def test_returns_vlan_interfaces(self):
        self.client.add_vlan(1000, name="vlan1000")

        self.try_to.set_access_mode(self.test_ports[0].name)
        self.try_to.set_access_vlan(self.test_ports[0].name, 1000)

        self.try_to.set_trunk_mode(self.test_ports[1].name)
        self.try_to.add_trunk_vlan(self.test_ports[1].name, 1000)

        # At this time, interface_native_vlan is not supported on get_vlan_interfaces

        assert_that(self.client.get_vlan_interfaces(1000),
                    contains_inanyorder([self.test_ports[0].name, self.test_ports[1].name]))

    def test_fails_when_the_vlan_does_not_exist(self):
        with self.assertRaises(UnknownVlan):
            self.client.get_vlan_interfaces(2000)

    def tearDown(self):
        self.janitor.unset_interface_access_vlan(self.test_ports[0].name)
        self.janitor.unset_interface_native_vlan(self.test_ports[0].name)

        self.janitor.remove_trunk_vlan(self.test_ports[1].name, 1000)
        self.janitor.set_access_mode(self.test_ports[1].name)

        self.janitor.remove_vlan(1000)
        super(GetVlanInterfacesTest, self).tearDown()
<commit_msg>Change contains_inanyorder to proper usage.<commit_after># Copyright 2016 Internap.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from hamcrest import assert_that, contains_inanyorder

from netman.core.objects.exceptions import UnknownVlan
from tests.adapters.configured_test_case import ConfiguredTestCase

class GetVlanInterfacesTest(ConfiguredTestCase):
    _dev_sample = "cisco"

    def setUp(self):
        super(GetVlanInterfacesTest, self).setUp()

    def test_returns_vlan_interfaces(self):
        self.client.add_vlan(1000, name="vlan1000")

        self.try_to.set_access_mode(self.test_ports[0].name)
        self.try_to.set_access_vlan(self.test_ports[0].name, 1000)

        self.try_to.set_trunk_mode(self.test_ports[1].name)
        self.try_to.add_trunk_vlan(self.test_ports[1].name, 1000)

        # At this time, interface_native_vlan is not supported on get_vlan_interfaces

        assert_that(self.client.get_vlan_interfaces(1000),
                    contains_inanyorder(self.test_ports[0].name, self.test_ports[1].name))

    def test_fails_when_the_vlan_does_not_exist(self):
        with self.assertRaises(UnknownVlan):
            self.client.get_vlan_interfaces(2000)

    def tearDown(self):
        self.janitor.unset_interface_access_vlan(self.test_ports[0].name)
        self.janitor.unset_interface_native_vlan(self.test_ports[0].name)

        self.janitor.remove_trunk_vlan(self.test_ports[1].name, 1000)
        self.janitor.set_access_mode(self.test_ports[1].name)

        self.janitor.remove_vlan(1000)
        super(GetVlanInterfacesTest, self).tearDown()
<|endoftext|>


<commit_before>import zeit.cms.testing
import zeit.content.article.testing

class SocialFormTest(zeit.cms.testing.BrowserTestCase):

    layer = zeit.content.article.testing.LAYER

    def setUp(self):
        super(SocialFormTest, self).setUp()
        self.browser.open(
            'http://localhost/++skin++vivi/repository/'
            'online/2007/01/Somalia/@@checkout')

    def get_article(self):
        with zeit.cms.testing.site(self.getRootFolder()):
            with zeit.cms.testing.interaction():
                return zeit.cms.interfaces.ICMSWCContent(
                    'http://xml.zeit.de/online/2007/01/Somalia')

    def open_form(self):
        # XXX A simple browser.reload() does not work, why?
        self.browser.open(
            'http://localhost/++skin++vivi/workingcopy/zope.user/'
            'Somalia/@@edit.form.social?show_form=1')

    def test_checkbox_is_only_editable_while_checked_out(self):
        self.open_form()
        b = self.browser
        # Assert nothing raised
        b.getControl('Push after next publish?').selected = True

        b.open('@@checkin')
        b.open('@@edit.form.social?show_form=1')
        with self.assertRaises(AttributeError) as err:
            b.getControl('Push after next publish?').selected = True
        self.assertEqual('item is disabled', err.exception.message)

        b.open('@@checkout')
        b.open('@@edit.form.social?show_form=1')
        # Assert nothing raised
        b.getControl('Push after next publish?').selected = True

    def test_smoke_form_submit_stores_values(self):
        self.open_form()
        b = self.browser
        b.getControl('Push after next publish?').selected = True
        b.getControl('Apply').click()
        article = self.get_article()
        push = zeit.push.interfaces.IPushMessages(article)
        self.assertEqual(True, push.enabled)
<commit_msg>re VIV-516: Fix test by making sure that field value was actually changed (previously just overwrote the default with the same value).<commit_after>import zeit.cms.testing
import zeit.content.article.testing

class SocialFormTest(zeit.cms.testing.BrowserTestCase):

    layer = zeit.content.article.testing.LAYER

    def setUp(self):
        super(SocialFormTest, self).setUp()
        self.browser.open(
            'http://localhost/++skin++vivi/repository/'
            'online/2007/01/Somalia/@@checkout')

    def get_article(self):
        with zeit.cms.testing.site(self.getRootFolder()):
            with zeit.cms.testing.interaction():
                return zeit.cms.interfaces.ICMSWCContent(
                    'http://xml.zeit.de/online/2007/01/Somalia')

    def open_form(self):
        # XXX A simple browser.reload() does not work, why?
        self.browser.open(
            'http://localhost/++skin++vivi/workingcopy/zope.user/'
            'Somalia/@@edit.form.social?show_form=1')

    def test_checkbox_is_only_editable_while_checked_out(self):
        self.open_form()
        b = self.browser
        # Assert nothing raised
        b.getControl('Push after next publish?').selected = True

        b.open('@@checkin')
        b.open('@@edit.form.social?show_form=1')
        with self.assertRaises(AttributeError) as err:
            b.getControl('Push after next publish?').selected = True
        self.assertEqual('item is disabled', err.exception.message)

        b.open('@@checkout')
        b.open('@@edit.form.social?show_form=1')
        # Assert nothing raised
        b.getControl('Push after next publish?').selected = True

    def test_smoke_form_submit_stores_values(self):
        self.open_form()
        b = self.browser
        self.assertFalse(b.getControl('Enable Twitter').selected)
        b.getControl('Enable Twitter').selected = True
        b.getControl('Apply').click()
        article = self.get_article()
        push = zeit.push.interfaces.IPushMessages(article)
        self.assertIn(
            {'type': 'twitter', 'enabled': True, 'account': 'twitter-test'},
            push.message_config)
<|endoftext|>