<commit_before># GUI Application automation and testing library
# Copyright (C) 2006-2018 Mark Mc Mahon and Contributors
# https://github.com/pywinauto/pywinauto/graphs/contributors
# http://pywinauto.readthedocs.io/en/latest/credits.html
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice, this
#   list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
#
# * Neither the name of pywinauto nor the names of its
#   contributors may be used to endorse or promote products derived from
#   this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""Cross-platform module to emulate mouse events like a real user"""

import sys
import time
from math import ceil
if sys.platform == 'win32':
    import pywintypes
    import win32api
    import win32gui
    from .windows import win32defines, win32functions
    from .timings import Timings
    from . import keyboard
else:
    from Xlib.display import Display
    from Xlib import X
    from Xlib.ext.xtest import fake_input

BUTTON_MAPPING = {'left': 0, 'middle': 1, 'right': 2, 'up_scroll': 3,
                  'down_scroll': 4, 'left_scroll': 5, 'right_scroll': 6}

if sys.platform == 'win32':

    def _get_cursor_pos():
        return win32api.GetCursorPos()

    def _set_cursor_pos(coords):
        """Wrapped SetCursorPos that handles non-active desktop case (coords is a tuple)"""
        try:
            win32api.SetCursorPos(coords)
        except pywintypes.error as exc:
            if str(exc) == "(0, 'SetCursorPos', 'No error message is available')":
                raise RuntimeError("There is no active desktop required for moving mouse cursor!\n")
            else:
                raise exc

    def _perform_click_input(
        button="left",
        coords=(None, None),
        double=False,
        button_down=True,
        button_up=True,
        wheel_dist=0,
        pressed="",
        key_down=True,
        key_up=True,
    ):
        """Perform a click action using SendInput

        All the *click_input() and *mouse_input() methods use this function.

        Thanks to a bug report from Tomas Walch (twalch) on sourceforge and code
        seen at http://msdn.microsoft.com/en-us/magazine/cc164126.aspx this
        function now always works the same way whether the mouse buttons are
        swapped or not.

        For example if you send a right click to Notepad.Edit - it will always
        bring up a popup menu rather than 'clicking' it.
        """

        # Handle if the mouse buttons are swapped
        if win32functions.GetSystemMetrics(win32defines.SM_SWAPBUTTON):
            if button.lower() == 'left':
                button = 'right'
            elif button.lower() == 'right':
                button = 'left'

        events = []
        if button.lower() == 'left':
            events.append(win32defines.MOUSEEVENTF_MOVE)
            if button_down:
                events.append(win32defines.MOUSEEVENTF_LEFTDOWN)
            if button_up:
                events.append(win32defines.MOUSEEVENTF_LEFTUP)
        elif button.lower() == 'right':
            if button_down:
                events.append(win32defines.MOUSEEVENTF_RIGHTDOWN)
            if button_up:
                events.append(win32defines.MOUSEEVENTF_RIGHTUP)
        elif button.lower() == 'middle':
            if button_down:
                events.append(win32defines.MOUSEEVENTF_MIDDLEDOWN)
            if button_up:
                events.append(win32defines.MOUSEEVENTF_MIDDLEUP)
        elif button.lower() == 'move':
            events.append(win32defines.MOUSEEVENTF_MOVE)
            events.append(win32defines.MOUSEEVENTF_ABSOLUTE)
        elif button.lower() == 'x':
            if button_down:
                events.append(win32defines.MOUSEEVENTF_XDOWN)
            if button_up:
                events.append(win32defines.MOUSEEVENTF_XUP)

        if button.lower() == 'wheel':
            events.append(win32defines.MOUSEEVENTF_WHEEL)

        # if we were asked to double click (and we are doing a full click
        # not just up or down.
        if double and button_down and button_up:
            events *= 2

        if button_down and (button.lower() not in ['move', 'wheel']):
            # wait while previous click is not affecting our current click
            while 0 < win32api.GetTickCount() - win32api.GetLastInputInfo() < win32gui.GetDoubleClickTime():
                time.sleep(Timings.after_clickinput_wait)

        # set the cursor position
        _set_cursor_pos((coords[0], coords[1]))
        time.sleep(Timings.after_setcursorpos_wait)
        if win32api.GetCursorPos() != (coords[0], coords[1]):
            _set_cursor_pos((coords[0], coords[1]))
            time.sleep(Timings.after_setcursorpos_wait)

        keyboard_keys = pressed.lower().split()
        if ('control' in keyboard_keys) and key_down:
            keyboard.VirtualKeyAction(keyboard.VK_CONTROL, up=False).run()
        if ('shift' in keyboard_keys) and key_down:
            keyboard.VirtualKeyAction(keyboard.VK_SHIFT, up=False).run()
        if ('alt' in keyboard_keys) and key_down:
            keyboard.VirtualKeyAction(keyboard.VK_MENU, up=False).run()

        dw_flags = 0
        for event in events:
            dw_flags |= event

        dw_data = 0
        if button.lower() == 'wheel':
            wheel_dist = wheel_dist * 120
            dw_data = wheel_dist

        if button.lower() == 'move':
            x_res = win32functions.GetSystemMetrics(win32defines.SM_CXSCREEN)
            y_res = win32functions.GetSystemMetrics(win32defines.SM_CYSCREEN)
            x_coord = int(ceil(coords[0] * 65535 / (x_res - 1.)))  # in Python 2.7 return float val
            y_coord = int(ceil(coords[1] * 65535 / (y_res - 1.)))  # in Python 2.7 return float val
            win32api.mouse_event(dw_flags, x_coord, y_coord, dw_data)
        else:
            for event in events:
                if event == win32defines.MOUSEEVENTF_MOVE:
                    x_res = win32functions.GetSystemMetrics(win32defines.SM_CXSCREEN)
                    y_res = win32functions.GetSystemMetrics(win32defines.SM_CYSCREEN)
                    x_coord = int(ceil(coords[0] * 65535 / (x_res - 1.)))  # in Python 2.7 return float val
                    y_coord = int(ceil(coords[1] * 65535 / (y_res - 1.)))  # in Python 2.7 return float val
                    win32api.mouse_event(
                        win32defines.MOUSEEVENTF_MOVE | win32defines.MOUSEEVENTF_ABSOLUTE,
                        x_coord, y_coord, dw_data)
                else:
                    win32api.mouse_event(
                        event | win32defines.MOUSEEVENTF_ABSOLUTE,
                        coords[0], coords[1], dw_data)

        time.sleep(Timings.after_clickinput_wait)

        if ('control' in keyboard_keys) and key_up:
            keyboard.VirtualKeyAction(keyboard.VK_CONTROL, down=False).run()
        if ('shift' in keyboard_keys) and key_up:
            keyboard.VirtualKeyAction(keyboard.VK_SHIFT, down=False).run()
        if ('alt' in keyboard_keys) and key_up:
            keyboard.VirtualKeyAction(keyboard.VK_MENU, down=False).run()

else:
    _display = Display()

    def _get_cursor_pos(coords):
        data = _display.screen().root.query_pointer()._data
        return data["root_x"], data["root_y"]

    def _perform_click_input(button='left', coords=(0, 0),
                             button_down=True, button_up=True, double=False,
                             wheel_dist=0, pressed="", key_down=True, key_up=True):
        """Perform a click action using Python-xlib"""
        #Move mouse
        x = int(coords[0])
        y = int(coords[1])
        fake_input(_display, X.MotionNotify, x=x, y=y)
        _display.sync()

        if button == 'wheel':
            if wheel_dist == 0:
                return
            if wheel_dist > 0:
                button = 'up_scroll'
            if wheel_dist < 0:
                button = 'down_scroll'
            for _ in range(abs(wheel_dist)):
                _perform_click_input(button, coords)
        else:
            pointer_map = _display.get_pointer_mapping()
            button = pointer_map[BUTTON_MAPPING[button]]
            repeat = 1
            if double:
                repeat = 2
            for _ in range(repeat):
                if button_down:
                    fake_input(_display, X.ButtonPress, button)
                    _display.sync()
                if button_up:
                    fake_input(_display, X.ButtonRelease, button)
                    _display.sync()

def click(button='left', coords=(0, 0)):
    """Click at the specified coordinates"""
    _perform_click_input(button=button, coords=coords)

def double_click(button='left', coords=(0, 0)):
    """Double click at the specified coordinates"""
    _perform_click_input(button=button, coords=coords, double=True)

def right_click(coords=(0, 0)):
    """Right click at the specified coords"""
    _perform_click_input(button='right', coords=coords)

def move(coords=(0, 0), duration=0.0):
    """Move the mouse"""
    if not isinstance(duration, float):
        raise TypeError("duration must be float (in seconds)")
    minimum_duration = 0.05
    if duration >= minimum_duration:
        x_start, y_start = _get_cursor_pos()
        delta_x = coords[0] - x_start
        delta_y = coords[1] - y_start
        max_delta = max(abs(delta_x), abs(delta_y))
        num_steps = max_delta
        sleep_amount = duration / max(num_steps, 1)
        if sleep_amount < minimum_duration:
            num_steps = int(num_steps * sleep_amount / minimum_duration)
            sleep_amount = minimum_duration
        delta_x /= max(num_steps, 1)
        delta_y /= max(num_steps, 1)
        for step in range(num_steps):
            _set_cursor_pos((x_start + int(delta_x*step), y_start + int(delta_y*step)))
            time.sleep(sleep_amount)
    _perform_click_input(button='move',coords=coords,button_down=False,button_up=False)

def press(button='left', coords=(0, 0)):
    """Press the mouse button"""
    _perform_click_input(button=button, coords=coords, button_down=True, button_up=False)

def release(button='left', coords=(0, 0)):
    """Release the mouse button"""
    _perform_click_input(button=button, coords=coords, button_down=False, button_up=True)

def scroll(coords=(0, 0), wheel_dist=1):
    """Do mouse wheel"""
    if wheel_dist:
        _perform_click_input(button='wheel', wheel_dist=wheel_dist, coords=coords)

def wheel_click(coords=(0, 0)):
    """Middle mouse button click at the specified coords"""
    _perform_click_input(button='middle', coords=coords)
<commit_msg>added fast move<commit_after># GUI Application automation and testing library
# Copyright (C) 2006-2018 Mark Mc Mahon and Contributors
# https://github.com/pywinauto/pywinauto/graphs/contributors
# http://pywinauto.readthedocs.io/en/latest/credits.html
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice, this
#   list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
#
# * Neither the name of pywinauto nor the names of its
#   contributors may be used to endorse or promote products derived from
#   this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""Cross-platform module to emulate mouse events like a real user"""

import sys
import time
from math import ceil
if sys.platform == 'win32':
    import pywintypes
    import win32api
    import win32gui
    from .windows import win32defines, win32functions
    from .timings import Timings
    from . import keyboard
else:
    from Xlib.display import Display
    from Xlib import X
    from Xlib.ext.xtest import fake_input

BUTTON_MAPPING = {'left': 0, 'middle': 1, 'right': 2, 'up_scroll': 3,
                  'down_scroll': 4, 'left_scroll': 5, 'right_scroll': 6}

if sys.platform == 'win32':

    def _get_cursor_pos():
        return win32api.GetCursorPos()

    def _set_cursor_pos(coords):
        """Wrapped SetCursorPos that handles non-active desktop case (coords is a tuple)"""
        try:
            win32api.SetCursorPos(coords)
        except pywintypes.error as exc:
            if str(exc) == "(0, 'SetCursorPos', 'No error message is available')":
                raise RuntimeError("There is no active desktop required for moving mouse cursor!\n")
            else:
                raise exc

    def _perform_click_input(
        button="left",
        coords=(None, None),
        double=False,
        button_down=True,
        button_up=True,
        wheel_dist=0,
        pressed="",
        key_down=True,
        key_up=True,
        fast_move=False,
    ):
        """Perform a click action using SendInput

        All the *click_input() and *mouse_input() methods use this function.

        Thanks to a bug report from Tomas Walch (twalch) on sourceforge and code
        seen at http://msdn.microsoft.com/en-us/magazine/cc164126.aspx this
        function now always works the same way whether the mouse buttons are
        swapped or not.

        For example if you send a right click to Notepad.Edit - it will always
        bring up a popup menu rather than 'clicking' it.
        """

        # Handle if the mouse buttons are swapped
        if win32functions.GetSystemMetrics(win32defines.SM_SWAPBUTTON):
            if button.lower() == 'left':
                button = 'right'
            elif button.lower() == 'right':
                button = 'left'

        events = []
        if button.lower() == 'left':
            events.append(win32defines.MOUSEEVENTF_MOVE)
            if button_down:
                events.append(win32defines.MOUSEEVENTF_LEFTDOWN)
            if button_up:
                events.append(win32defines.MOUSEEVENTF_LEFTUP)
        elif button.lower() == 'right':
            if button_down:
                events.append(win32defines.MOUSEEVENTF_RIGHTDOWN)
            if button_up:
                events.append(win32defines.MOUSEEVENTF_RIGHTUP)
        elif button.lower() == 'middle':
            if button_down:
                events.append(win32defines.MOUSEEVENTF_MIDDLEDOWN)
            if button_up:
                events.append(win32defines.MOUSEEVENTF_MIDDLEUP)
        elif button.lower() == 'move':
            events.append(win32defines.MOUSEEVENTF_MOVE)
            events.append(win32defines.MOUSEEVENTF_ABSOLUTE)
        elif button.lower() == 'x':
            if button_down:
                events.append(win32defines.MOUSEEVENTF_XDOWN)
            if button_up:
                events.append(win32defines.MOUSEEVENTF_XUP)

        if button.lower() == 'wheel':
            events.append(win32defines.MOUSEEVENTF_WHEEL)

        # if we were asked to double click (and we are doing a full click
        # not just up or down.
        if double and button_down and button_up:
            events *= 2

        if button_down and (button.lower() not in ['move', 'wheel']):
            # wait while previous click is not affecting our current click
            while 0 < win32api.GetTickCount() - win32api.GetLastInputInfo() < win32gui.GetDoubleClickTime():
                time.sleep(Timings.after_clickinput_wait)

        # set the cursor position
        _set_cursor_pos((coords[0], coords[1]))
        if not fast_move:
            time.sleep(Timings.after_setcursorpos_wait)
        if win32api.GetCursorPos() != (coords[0], coords[1]):
            _set_cursor_pos((coords[0], coords[1]))
            time.sleep(Timings.after_setcursorpos_wait)

        keyboard_keys = pressed.lower().split()
        if ('control' in keyboard_keys) and key_down:
            keyboard.VirtualKeyAction(keyboard.VK_CONTROL, up=False).run()
        if ('shift' in keyboard_keys) and key_down:
            keyboard.VirtualKeyAction(keyboard.VK_SHIFT, up=False).run()
        if ('alt' in keyboard_keys) and key_down:
            keyboard.VirtualKeyAction(keyboard.VK_MENU, up=False).run()

        dw_flags = 0
        for event in events:
            dw_flags |= event

        dw_data = 0
        if button.lower() == 'wheel':
            wheel_dist = wheel_dist * 120
            dw_data = wheel_dist

        if button.lower() == 'move':
            x_res = win32functions.GetSystemMetrics(win32defines.SM_CXSCREEN)
            y_res = win32functions.GetSystemMetrics(win32defines.SM_CYSCREEN)
            x_coord = int(ceil(coords[0] * 65535 / (x_res - 1.)))  # in Python 2.7 return float val
            y_coord = int(ceil(coords[1] * 65535 / (y_res - 1.)))  # in Python 2.7 return float val
            win32api.mouse_event(dw_flags, x_coord, y_coord, dw_data)
        else:
            for event in events:
                if event == win32defines.MOUSEEVENTF_MOVE:
                    x_res = win32functions.GetSystemMetrics(win32defines.SM_CXSCREEN)
                    y_res = win32functions.GetSystemMetrics(win32defines.SM_CYSCREEN)
                    x_coord = int(ceil(coords[0] * 65535 / (x_res - 1.)))  # in Python 2.7 return float val
                    y_coord = int(ceil(coords[1] * 65535 / (y_res - 1.)))  # in Python 2.7 return float val
                    win32api.mouse_event(
                        win32defines.MOUSEEVENTF_MOVE | win32defines.MOUSEEVENTF_ABSOLUTE,
                        x_coord, y_coord, dw_data)
                else:
                    win32api.mouse_event(
                        event | win32defines.MOUSEEVENTF_ABSOLUTE,
                        coords[0], coords[1], dw_data)

        if not fast_move:
            time.sleep(Timings.after_clickinput_wait)

        if ('control' in keyboard_keys) and key_up:
            keyboard.VirtualKeyAction(keyboard.VK_CONTROL, down=False).run()
        if ('shift' in keyboard_keys) and key_up:
            keyboard.VirtualKeyAction(keyboard.VK_SHIFT, down=False).run()
        if ('alt' in keyboard_keys) and key_up:
            keyboard.VirtualKeyAction(keyboard.VK_MENU, down=False).run()

else:
    _display = Display()

    def _get_cursor_pos(coords):
        data = _display.screen().root.query_pointer()._data
        return data["root_x"], data["root_y"]

    def _perform_click_input(button='left', coords=(0, 0),
                             button_down=True, button_up=True, double=False,
                             wheel_dist=0, pressed="", key_down=True, key_up=True,
                             fast_move=False):
        """Perform a click action using Python-xlib"""
        #Move mouse
        x = int(coords[0])
        y = int(coords[1])
        fake_input(_display, X.MotionNotify, x=x, y=y)
        if not fast_move:
            _display.sync()

        if button == 'wheel':
            if wheel_dist == 0:
                return
            if wheel_dist > 0:
                button = 'up_scroll'
            if wheel_dist < 0:
                button = 'down_scroll'
            for _ in range(abs(wheel_dist)):
                _perform_click_input(button, coords)
        else:
            pointer_map = _display.get_pointer_mapping()
            button = pointer_map[BUTTON_MAPPING[button]]
            repeat = 1
            if double:
                repeat = 2
            for _ in range(repeat):
                if button_down:
                    fake_input(_display, X.ButtonPress, button)
                    _display.sync()
                if button_up:
                    fake_input(_display, X.ButtonRelease, button)
                    _display.sync()

def click(button='left', coords=(0, 0)):
    """Click at the specified coordinates"""
    _perform_click_input(button=button, coords=coords)

def double_click(button='left', coords=(0, 0)):
    """Double click at the specified coordinates"""
    _perform_click_input(button=button, coords=coords, double=True)

def right_click(coords=(0, 0)):
    """Right click at the specified coords"""
    _perform_click_input(button='right', coords=coords)

def move(coords=(0, 0), duration=0.0):
    """Move the mouse"""
    if not isinstance(duration, float):
        raise TypeError("duration must be float (in seconds)")
    minimum_duration = 0.05
    if duration >= minimum_duration:
        x_start, y_start = _get_cursor_pos()
        delta_x = coords[0] - x_start
        delta_y = coords[1] - y_start
        max_delta = max(abs(delta_x), abs(delta_y))
        num_steps = max_delta
        sleep_amount = duration / max(num_steps, 1)
        if sleep_amount < minimum_duration:
            num_steps = int(num_steps * sleep_amount / minimum_duration)
            sleep_amount = minimum_duration
        delta_x /= max(num_steps, 1)
        delta_y /= max(num_steps, 1)
        for step in range(num_steps):
             _perform_click_input(button='move', coords=(x_start + int(delta_x*step), y_start + int(delta_y*step)),
                                  button_down=False, button_up=False, fast_move=True)
            time.sleep(sleep_amount)
    _perform_click_input(button='move',coords=coords,button_down=False,button_up=False)

def press(button='left', coords=(0, 0)):
    """Press the mouse button"""
    _perform_click_input(button=button, coords=coords, button_down=True, button_up=False)

def release(button='left', coords=(0, 0)):
    """Release the mouse button"""
    _perform_click_input(button=button, coords=coords, button_down=False, button_up=True)

def scroll(coords=(0, 0), wheel_dist=1):
    """Do mouse wheel"""
    if wheel_dist:
        _perform_click_input(button='wheel', wheel_dist=wheel_dist, coords=coords)

def wheel_click(coords=(0, 0)):
    """Middle mouse button click at the specified coords"""
    _perform_click_input(button='middle', coords=coords)
<|endoftext|>


<commit_before>import os
import warnings
from . import defaultconfig as config
from .cpptypeconv import (is_type_with_automatic_conversion,
                          create_type_converter)
from .utils import lines, indent_block, from_camel_case

class CythonDeclarationExporter:
    """Export AST to Cython declaration file (.pxd).

    This class implements the visitor pattern.
    """
    def __init__(self):
        self.output = ""
        self.ctors = []
        self.methods = []
        self.arguments = []
        self.includes = None

    def visit_ast(self, ast):
        pass

    def visit_includes(self, includes):
        self.includes = includes

    def visit_class(self, clazz):
        class_decl_parts = [config.class_def % clazz.__dict__,
                            os.linesep.join(self.ctors),
                            os.linesep.join(self.methods)]
        class_decl_parts = [p for p in class_decl_parts if p != ""]
        self.output += (os.linesep * 2 + os.linesep.join(class_decl_parts) +
                        os.linesep * 2)

        self.ctors = []
        self.methods = []

    def visit_constructor(self, ctor):
        const_dict = {"args": ", ".join(self.arguments)}
        const_dict.update(ctor.__dict__)
        const_str = config.constructor_def % const_dict
        self.ctors.append(const_str)
        self.arguments = []

    def visit_method(self, method):
        method_dict = {"args": ", ".join(self.arguments)}
        method_dict.update(method.__dict__)
        method_str = config.method_def % method_dict
        self.methods.append(method_str)
        self.arguments = []

    def visit_function(self, function):
        function_dict = {"args": ", ".join(self.arguments)}
        function_dict.update(function.__dict__)
        function_str = config.function_def % function_dict
        self.output += (os.linesep * 2 + function_str + os.linesep)
        self.arguments = []

    def visit_param(self, param):
        self.arguments.append(config.arg_def % param.__dict__)

    def export(self):
        return self.includes.declarations_import() + self.output

class CythonImplementationExporter:
    """Export AST to Cython implementation file (.pyx).

    This class implements the visitor pattern.
    """
    def __init__(self, classes):
        self.classes = classes
        self.output = ""
        self.ctors = []
        self.methods = []
        self.arguments = []
        self.includes = None

    def visit_ast(self, ast):
        pass

    def visit_includes(self, includes):
        self.includes = includes

    def visit_class(self, clazz):
        if len(self.ctors) > 1:
            msg = ("Class '%s' has more than one constructor. This is not "
                   "compatible to Python. The last constructor will overwrite "
                   "all others." % clazz.name)
            warnings.warn(msg)

        class_def_parts = [config.py_class_def % clazz.__dict__,
                           os.linesep.join(self.ctors),
                           os.linesep.join(self.methods)]
        class_def_parts = [p for p in class_def_parts if p != ""]
        self.output += (os.linesep * 2 + os.linesep.join(class_def_parts) +
                        os.linesep * 2)

        self.ctors = []
        self.methods = []

    def visit_constructor(self, ctor):
        function_def = ConstructorDefinition(
            ctor.class_name, ctor.name, ctor.arguments, self.includes,
            initial_args=["self"], classes=self.classes).make()
        self.ctors.append(indent_block(function_def, 1))
        self.arguments = []

    def visit_method(self, method):
        function_def = MethodDefinition(
            method.name, method.arguments, self.includes, ["self"],
            method.result_type, classes=self.classes).make()
        self.methods.append(indent_block(function_def, 1))
        self.arguments = []

    def visit_function(self, function):
        try:
            function_def = FunctionDefinition(
                function.name, function.arguments, self.includes, [],
                function.result_type, classes=self.classes).make()
            self.output += (os.linesep * 2 + function_def + os.linesep)
        except NotImplementedError as e:
            warnings.warn(e.message + " Ignoring function '%s'" % function.name)
        finally:
            self.arguments = []

    def visit_param(self, param):
        self.arguments.append(config.py_arg_def % param.__dict__)

    def export(self):
        return self.includes.implementations_import() + self.output

class FunctionDefinition(object):
    def __init__(self, name, arguments, includes, initial_args, result_type,
                 classes):
        self.name = name
        self.arguments = arguments
        self.includes = includes
        self.initial_args = initial_args
        self.result_type = result_type
        self.classes = classes
        self.type_converters = []

    def make(self):
        self._create_type_converters()
        body, call_args = self._input_type_conversions(self.includes)
        body += self._call_cpp_function(call_args)
        body += self.output_type_converter.return_output()
        return self._signature() + os.linesep + indent_block(body, 1)

    def _create_type_converters(self):
        skip = 0
        for arg in self.arguments:
            if skip > 0:
                skip -= 1
                continue
            type_converter = create_type_converter(
                arg.tipe, arg.name, self.classes)
            type_converter.add_includes(self.includes)
            self.type_converters.append(type_converter)
            skip = type_converter.n_cpp_args() - 1
        self.output_type_converter = create_type_converter(
            self.result_type, None, self.classes)
        self.output_type_converter.add_includes(self.includes)

    def _call_cpp_function(self, call_args):
        call = "cpp.{fname}({args})".format(
            fname=self.name, args=", ".join(call_args))
        if self.result_type != "void":
            call = "cdef {result_type} result = {call}".format(
                result_type=self.result_type, call=call)
        return call + os.linesep

    def _signature(self):
        args = self._cython_signature_args()
        return "def %s(%s):" % (from_camel_case(self.name), ", ".join(args))

    def _cython_signature_args(self):
        cython_signature_args = []
        cython_signature_args.extend(self.initial_args)
        for type_converter in self.type_converters:
            arg = type_converter.cython_signature()
            cython_signature_args.append(arg)
        return cython_signature_args

    def _input_type_conversions(self, includes):
        body = ""
        call_args = []
        for type_converter in self.type_converters:
            body += type_converter.python_to_cpp() + os.linesep
            call_args.extend(type_converter.cpp_call_args())
        return body, call_args

class ConstructorDefinition(FunctionDefinition):
    def __init__(self, class_name, name, arguments, includes, initial_args,
                 classes):
        super(ConstructorDefinition, self).__init__(
            name, arguments, includes, initial_args, result_type=None,
            classes=classes)
        self.class_name = class_name

    def _call_cpp_function(self, call_args):
        return "self.thisptr = new cpp.%s(%s)%s" % (
            self.class_name, ", ".join(call_args), os.linesep)

    def _signature(self):
        args = self._cython_signature_args()
        return "def __init__(%s):" % ", ".join(args)

class MethodDefinition(FunctionDefinition):
    def _call_cpp_function(self, call_args):
        call = "self.thisptr.{fname}({args})".format(
            fname=self.name, args=", ".join(call_args))
        if self.result_type != "void":
            call = "cdef {result_type} result = {call}".format(
                result_type=self.output_type_converter.cpp_type_decl(),
                call=call)
        return call + os.linesep

    def _signature(self):
        args = self._cython_signature_args()
        return "cpdef %s(%s):" % (from_camel_case(self.name), ", ".join(args))
<commit_msg>Catch NotImplementedError and print warning<commit_after>import os
import warnings
from . import defaultconfig as config
from .cpptypeconv import (is_type_with_automatic_conversion,
                          create_type_converter)
from .utils import lines, indent_block, from_camel_case

class CythonDeclarationExporter:
    """Export AST to Cython declaration file (.pxd).

    This class implements the visitor pattern.
    """
    def __init__(self):
        self.output = ""
        self.ctors = []
        self.methods = []
        self.arguments = []
        self.includes = None

    def visit_ast(self, ast):
        pass

    def visit_includes(self, includes):
        self.includes = includes

    def visit_class(self, clazz):
        class_decl_parts = [config.class_def % clazz.__dict__,
                            os.linesep.join(self.ctors),
                            os.linesep.join(self.methods)]
        class_decl_parts = [p for p in class_decl_parts if p != ""]
        self.output += (os.linesep * 2 + os.linesep.join(class_decl_parts) +
                        os.linesep * 2)

        self.ctors = []
        self.methods = []

    def visit_constructor(self, ctor):
        const_dict = {"args": ", ".join(self.arguments)}
        const_dict.update(ctor.__dict__)
        const_str = config.constructor_def % const_dict
        self.ctors.append(const_str)
        self.arguments = []

    def visit_method(self, method):
        method_dict = {"args": ", ".join(self.arguments)}
        method_dict.update(method.__dict__)
        method_str = config.method_def % method_dict
        self.methods.append(method_str)
        self.arguments = []

    def visit_function(self, function):
        function_dict = {"args": ", ".join(self.arguments)}
        function_dict.update(function.__dict__)
        function_str = config.function_def % function_dict
        self.output += (os.linesep * 2 + function_str + os.linesep)
        self.arguments = []

    def visit_param(self, param):
        self.arguments.append(config.arg_def % param.__dict__)

    def export(self):
        return self.includes.declarations_import() + self.output

class CythonImplementationExporter:
    """Export AST to Cython implementation file (.pyx).

    This class implements the visitor pattern.
    """
    def __init__(self, classes):
        self.classes = classes
        self.output = ""
        self.ctors = []
        self.methods = []
        self.arguments = []
        self.includes = None

    def visit_ast(self, ast):
        pass

    def visit_includes(self, includes):
        self.includes = includes

    def visit_class(self, clazz):
        if len(self.ctors) > 1:
            msg = ("Class '%s' has more than one constructor. This is not "
                   "compatible to Python. The last constructor will overwrite "
                   "all others." % clazz.name)
            warnings.warn(msg)

        class_def_parts = [config.py_class_def % clazz.__dict__,
                           os.linesep.join(self.ctors),
                           os.linesep.join(self.methods)]
        class_def_parts = [p for p in class_def_parts if p != ""]
        self.output += (os.linesep * 2 + os.linesep.join(class_def_parts) +
                        os.linesep * 2)

        self.ctors = []
        self.methods = []

    def visit_constructor(self, ctor):
        function_def = ConstructorDefinition(
            ctor.class_name, ctor.name, ctor.arguments, self.includes,
            initial_args=["self"], classes=self.classes).make()
        self.ctors.append(indent_block(function_def, 1))
        self.arguments = []

    def visit_method(self, method):
        try:
            function_def = MethodDefinition(
                method.name, method.arguments, self.includes, ["self"],
                method.result_type, classes=self.classes).make()
            self.methods.append(indent_block(function_def, 1))
        except NotImplementedError as e:
            warnings.warn(e.message + " Ignoring method '%s'" % method.name)
        finally:
            self.arguments = []

    def visit_function(self, function):
        try:
            function_def = FunctionDefinition(
                function.name, function.arguments, self.includes, [],
                function.result_type, classes=self.classes).make()
            self.output += (os.linesep * 2 + function_def + os.linesep)
        except NotImplementedError as e:
            warnings.warn(e.message + " Ignoring function '%s'" % function.name)
        finally:
            self.arguments = []

    def visit_param(self, param):
        self.arguments.append(config.py_arg_def % param.__dict__)

    def export(self):
        return self.includes.implementations_import() + self.output

class FunctionDefinition(object):
    def __init__(self, name, arguments, includes, initial_args, result_type,
                 classes):
        self.name = name
        self.arguments = arguments
        self.includes = includes
        self.initial_args = initial_args
        self.result_type = result_type
        self.classes = classes
        self.type_converters = []

    def make(self):
        self._create_type_converters()
        body, call_args = self._input_type_conversions(self.includes)
        body += self._call_cpp_function(call_args)
        body += self.output_type_converter.return_output()
        return self._signature() + os.linesep + indent_block(body, 1)

    def _create_type_converters(self):
        skip = 0
        for arg in self.arguments:
            if skip > 0:
                skip -= 1
                continue
            type_converter = create_type_converter(
                arg.tipe, arg.name, self.classes)
            type_converter.add_includes(self.includes)
            self.type_converters.append(type_converter)
            skip = type_converter.n_cpp_args() - 1
        self.output_type_converter = create_type_converter(
            self.result_type, None, self.classes)
        self.output_type_converter.add_includes(self.includes)

    def _call_cpp_function(self, call_args):
        call = "cpp.{fname}({args})".format(
            fname=self.name, args=", ".join(call_args))
        if self.result_type != "void":
            call = "cdef {result_type} result = {call}".format(
                result_type=self.result_type, call=call)
        return call + os.linesep

    def _signature(self):
        args = self._cython_signature_args()
        return "def %s(%s):" % (from_camel_case(self.name), ", ".join(args))

    def _cython_signature_args(self):
        cython_signature_args = []
        cython_signature_args.extend(self.initial_args)
        for type_converter in self.type_converters:
            arg = type_converter.cython_signature()
            cython_signature_args.append(arg)
        return cython_signature_args

    def _input_type_conversions(self, includes):
        body = ""
        call_args = []
        for type_converter in self.type_converters:
            body += type_converter.python_to_cpp() + os.linesep
            call_args.extend(type_converter.cpp_call_args())
        return body, call_args

class ConstructorDefinition(FunctionDefinition):
    def __init__(self, class_name, name, arguments, includes, initial_args,
                 classes):
        super(ConstructorDefinition, self).__init__(
            name, arguments, includes, initial_args, result_type=None,
            classes=classes)
        self.class_name = class_name

    def _call_cpp_function(self, call_args):
        return "self.thisptr = new cpp.%s(%s)%s" % (
            self.class_name, ", ".join(call_args), os.linesep)

    def _signature(self):
        args = self._cython_signature_args()
        return "def __init__(%s):" % ", ".join(args)

class MethodDefinition(FunctionDefinition):
    def _call_cpp_function(self, call_args):
        call = "self.thisptr.{fname}({args})".format(
            fname=self.name, args=", ".join(call_args))
        if self.result_type != "void":
            call = "cdef {result_type} result = {call}".format(
                result_type=self.output_type_converter.cpp_type_decl(),
                call=call)
        return call + os.linesep

    def _signature(self):
        args = self._cython_signature_args()
        return "cpdef %s(%s):" % (from_camel_case(self.name), ", ".join(args))
<|endoftext|>


<commit_before>import json
import tempfile
import numpy as np
import os
import requests
import subprocess
import sys
import urllib

from .clickpaths import (DownloadBEDFileFromModal,
                         DownloadBEDFileFromTable,
                         DownloadDocuments,
                         DownloadDocumentsFromAntibodyPage,
                         DownloadFileFromButton,
                         DownloadFileFromFilePage,
                         DownloadFileFromModal,
                         DownloadFileFromTable,
                         DownloadGraphFromExperimentPage,
                         DownloadMetaDataFromSearchPage,
                         DownloadTSVFromReportPage,
                         OpenUCSCGenomeBrowser,
                         OpenUCSCGenomeBrowserCE10,
                         OpenUCSCGenomeBrowserCE11,
                         OpenUCSCGenomeBrowserDM3,
                         OpenUCSCGenomeBrowserDM6,
                         OpenUCSCGenomeBrowserGRCh38,
                         OpenUCSCGenomeBrowserHG19,
                         OpenUCSCGenomeBrowserMM10,
                         OpenUCSCGenomeBrowserMM10Minimal,
                         OpenUCSCGenomeBrowserMM9)
from .comparisons import (BrowserComparison,
                          CompareFacetNumbersBetweenBrowsers,
                          CompareFacetNumbersBetweenURLS,
                          CompareScreenShots,
                          URLComparison)
from .defaults import (USERS,
                       BROWSERS,
                       bcolors,
                       ActionTuples)
from .tasks import (DownloadFiles,
                    Driver,
                    GetFacetNumbers,
                    GetScreenShot,
                    SeleniumTask,
                    SignIn)
from .worker import DataWorker, DataManager

################################################################
# QANCODE object gets data, compares data using defined tasks. #
################################################################

class QANCODE(ActionTuples):
    """
    Object to keep track of Task/Comparison combinations and run QA
    process with one method call.
    """

    def __init__(self, rc_url, prod_url='https://www.encodeproject.org'):
        self.rc_url = rc_url
        self.prod_url = prod_url
        self.browsers = [b for b in BROWSERS]
        self.users = [u for u in USERS]
        # Default action attributes inherited from ActionTuples.
        super().__init__()

    def __repr__(self):
        return 'QANCODE(prod_url={}, rc_url={})'.format(self.prod_url,
                                                        self.rc_url)

    def list_methods(self):
        """
        List all possible tests.
        """
        print(*[f for f in sorted(dir(QANCODE)) if callable(getattr(QANCODE, f))
                and not f.startswith('__') and not f.startswith('_')], sep='\n')

    def compare_facets(self,
                       browsers='all',
                       users='all',
                       item_types='all',
                       task=GetFacetNumbers,
                       url_comparison=True,
                       browser_comparison=True):
        """
        Gets RC URL facet numbers and compares them to production URL facet
        numbers for given item_type, browser, user.
        """
        # Define item type pages (search and matrix views) to check.
        all_item_types = self.compare_facets_default_actions
        if browsers == 'all':
            browsers = self.browsers
        if users == 'all':
            users = self.users
        if item_types == 'all':
            item_types = [t for t in all_item_types]
        urls = [self.prod_url, self.rc_url]
        click_paths = [None for c in item_types]
        dm = DataManager(browsers=browsers,
                         urls=urls,
                         users=users,
                         item_types=item_types,
                         click_paths=click_paths,
                         task=task)
        dm.run_tasks()
        if url_comparison:
            for browser in browsers:
                for user in users:
                    for item_type in item_types:
                        cfn_url = CompareFacetNumbersBetweenURLS(browser=browser,
                                                                 user=user,
                                                                 prod_url=self.prod_url,
                                                                 rc_url=self.rc_url,
                                                                 item_type=item_type,
                                                                 click_path=None,
                                                                 all_data=dm.all_data)
                        cfn_url.compare_data()
        if browser_comparison:
            for url in urls:
                for user in users:
                    for item_type in item_types:
                        cfn_browser = CompareFacetNumbersBetweenBrowsers(user=user,
                                                                         url=url,
                                                                         item_type=item_type,
                                                                         browsers=browsers,
                                                                         all_data=dm.all_data)
                        cfn_browser.compare_data()

    @staticmethod
    def _expand_action_list(actions):
        """
        Returns item_types and click_paths given list of tuples.
        """
        item_types = [t[0] for t in actions]
        click_paths = [p[1] for p in actions]
        return item_types, click_paths

    def _parse_arguments(self,
                         browsers,
                         users,
                         item_types,
                         click_paths,
                         action_tuples,
                         actions,
                         admin_only_actions,
                         public_only_actions):
        if browsers == 'all':
            browsers = self.browsers
        if users == 'all':
            users = self.users
        if action_tuples is not None:
            item_types, click_paths = self._expand_action_list(action_tuples)
        else:
            if item_types == 'all':
                item_types, click_paths = self._expand_action_list(actions)
            elif item_types == 'random':
                # Choose random item_types from action list to check.
                # Note np.random.choice won't work for tuples.
                random_actions = [actions[i]
                                  for i in np.random.randint(len(actions) - 1, size=3)]
                item_types, click_paths = self._expand_action_list(
                    random_actions)
            elif item_types == 'admin':
                item_types, click_paths = self._expand_action_list(
                    admin_only_actions)
            elif item_types == 'public':
                item_types, click_paths = self._expand_action_list(
                    public_only_actions)
            else:
                if len(item_types) == len(click_paths):
                    pass
                elif len(click_paths) == 1:
                    # Broadcast single click_path (e.g. None) to all user-defined item_types.
                    click_paths = [click_paths[0] for t in item_types]
                else:
                    raise ValueError(
                        'item_types and click_paths must have same length')
        return browsers, users, item_types, click_paths

    def find_differences(self,
                         browsers='all',
                         users='all',
                         action_tuples=None,
                         item_types='all',
                         click_paths=[None],
                         task=GetScreenShot):
        """
        Does image diff for given item_types. If click_path defined will
        perform that action before taking screenshot.
        """
        # Tuple of (item_type, click_path)
        actions = self.find_differences_default_actions
        admin_only_actions = [('/biosamples/ENCBS681LAC/', None),
                              ('/search/?searchTerm=ENCBS681LAC&type=Biosample', None)]
        public_only_actions = [('/experiments/?status=deleted', None)]
        browsers, users, item_types, click_paths = self._parse_arguments(browsers=browsers,
                                                                         users=users,
                                                                         item_types=item_types,
                                                                         click_paths=click_paths,
                                                                         action_tuples=action_tuples,
                                                                         actions=actions,
                                                                         admin_only_actions=admin_only_actions,
                                                                         public_only_actions=public_only_actions)
        urls = [self.prod_url, self.rc_url]
        results = []
        with tempfile.TemporaryDirectory() as td:
            dm = DataManager(browsers=browsers,
                             urls=urls,
                             users=users,
                             item_types=item_types,
                             click_paths=click_paths,
                             task=task,
                             temp_dir=td)
            dm.run_tasks()
            for browser in browsers:
                for user in users:
                    for item_type, click_path in zip(item_types, click_paths):
                        css = CompareScreenShots(browser=browser,
                                                 user=user,
                                                 prod_url=self.prod_url,
                                                 rc_url=self.rc_url,
                                                 item_type=item_type,
                                                 click_path=click_path,
                                                 all_data=dm.all_data)
                        result = css.compare_data()
                        results.append(result)
        self.find_differences_output = results
        return self.find_differences_output

    def show_differences(self):
        """
        This is a Jupyter notebook helper function that loads image diffs
        into the notebook and displays them. Won't work in terminal.
        """
        try:
            # In Jupyter notebook?
            get_ipython().config
        except NameError:
            print('Must use in Jupyter notebook')
            return
        from IPython.core.display import display, HTML, Image
        try:
            for out in self.find_differences_output:
                if out[0]:
                    print('{}:'.format(out[1]))
                    image_diff_path = os.path.expanduser(
                        '~/Desktop/image_diff/')
                    display(Image(filename='{}{}'.format(image_diff_path,
                                                         out[1])))
                    print('\n')
        except AttributeError:
            print('No image diffs found')

    def check_trackhubs(self, browsers=['Safari'], users=['Public'], action_tuples=None):
        """
        Runs find_differences() image diff for selected list of trackhub
        actions.
        """
        print('Running check trackhubs')
        trackhub_actions = self.check_trackhubs_default_actions
        if action_tuples is None:
            action_tuples = trackhub_actions
        self.find_differences(users=users, browsers=browsers,
                              action_tuples=action_tuples)

    def check_permissions(self, browsers=['Safari'], users=['Public']):
        """
        Runs find_differences() image diff on permission check pages.
        """
        print('Running check permissions')
        permission_actions = self.check_permissions_default_actions
        self.find_differences(users=users, browsers=browsers,
                              action_tuples=permission_actions)

    def check_downloads(self,
                        browsers='all',
                        users='all',
                        action_tuples=None,
                        item_types='all',
                        click_paths=[None],
                        task=DownloadFiles,
                        urls='all',
                        delete=True):
        """
        Clicks download button and checks download folder for file.
        """
        print('Running check downloads')
        actions = self.check_downloads_default_actions
        admin_only_actions = []
        public_only_actions = []
        browsers, users, item_types, click_paths = self._parse_arguments(browsers=browsers,
                                                                         users=users,
                                                                         item_types=item_types,
                                                                         click_paths=click_paths,
                                                                         action_tuples=action_tuples,
                                                                         actions=actions,
                                                                         admin_only_actions=admin_only_actions,
                                                                         public_only_actions=public_only_actions)
        if urls == 'all':
            urls = [self.prod_url, self.rc_url]
        dm = DataManager(browsers=browsers,
                         urls=urls,
                         users=users,
                         item_types=item_types,
                         click_paths=click_paths,
                         task=task,
                         delete=delete)
        dm.run_tasks()

    @staticmethod
    def _create_authentication_header(headers, authid, authpw):
        headers['Authorization'] = requests.auth._basic_auth_str(
            authid, authpw)
        return headers

    @staticmethod
    def _block_production_edit(url):
        # Don't test production.
        if 'encodeproject.org' in url:
            raise SystemError('No editing production.')
        return None

    def _get_auth_credentials(self, user):
        if user == 'Public':
            return None, None
        cred_file = os.path.expanduser('~/qa_credentials.json')
        with open(cred_file) as f:
            creds = [(k['authid'], k['authpw']) for k
                     in json.load(f) if k['username'] == user]
        if not creds:
            raise ValueError('User not found.')
        return creds[0]

    def check_requests(self):
        """
        Runs post, patch, and get requests as different users.
        """
        url = self.rc_url
        self._block_production_edit(url)
        action_dict = self.check_requests_default_actions
        for k, v in action_dict.items():
            for item in v:
                request_url = urllib.parse.urljoin(url, item[0])
                for user, status_code in item[2]:
                    headers = {'content-type': 'application/json',
                               'accept': 'application/json'}
                    (authid, authpw) = self._get_auth_credentials(user)
                    if authid is not None:
                        headers = self._create_authentication_header(
                            headers, authid, authpw)
                    for payload in item[1]:
                        print('Trying to {} {} with {} as user {}'.format(
                            k, request_url, payload, user))
                        print('Expected: {}'.format(status_code))
                        r = requests.request(
                            k, url=request_url, headers=headers, data=json.dumps(payload))
                        try:
                            print('Actual: {}'.format(r.status_code))
                            assert r.status_code == status_code
                            print('{}{} SUCCESSFUL{}'.format(
                                bcolors.OKBLUE, k.upper(), bcolors.ENDC))
                        except AssertionError:
                            print('{}{} FAILURE\n{}{}'.format(
                                bcolors.FAIL, k.upper(), r.text, bcolors.ENDC))
                        finally:
                            print()

    def _add_rc_to_keypairs(self, url):
        """
        Gets production keypair and copies it to rc_url server.
        """
        keypairs = os.path.expanduser('~/keypairs.json')
        try:
            with open(keypairs, 'r') as f:
                data = json.load(f)
                prod_data = data['prod']
                authid, authpw = prod_data['key'], prod_data['secret']
                data['current_rc'] = {'server': url,
                                      'key': authid,
                                      'secret': authpw}
            # Overwrite.
            with open(keypairs, 'w') as f:
                json.dump(data, f)
        except Exception as e:
            print(e)
            raise

    def check_tools(self):
        """
        Runs pyencoded-tools against RC.
        """
        url = self.rc_url
        self._block_production_edit(url)
        self._add_rc_to_keypairs(url)
        key = '--key current_rc'
        # Expected outut a very weak check.
        tools = self.check_tools_default_actions
        # Get relative Python executable.
        python_executor = sys.executable
        for tool in tools:
            print('Running {}'.format(tool['name']))
            command = tool['command'].format(
                python_executor, tool['name'], key)
            print(command)
            output = subprocess.check_output(
                command, shell=True).decode('utf-8').strip()
            try:
                assert ((output == tool['expected_output'].strip())
                        or (tool['expected_output'] in output))
                print('{}{} SUCCESSFUL{}'.format(
                    bcolors.OKBLUE, tool['name'].upper(), bcolors.ENDC))
            except AssertionError:
                print('{}{} FAILURE{}'.format(
                    bcolors.FAIL, tool['name'].upper(), bcolors.ENDC))
                print('Expected: ')
                print(tool['expected_output'])
                print('Actual: ')
                print(output)
<commit_msg>Add check_response_time function<commit_after>import json
import tempfile
import numpy as np
import os
import requests
import subprocess
import sys
import urllib

from .clickpaths import (DownloadBEDFileFromModal,
                         DownloadBEDFileFromTable,
                         DownloadDocuments,
                         DownloadDocumentsFromAntibodyPage,
                         DownloadFileFromButton,
                         DownloadFileFromFilePage,
                         DownloadFileFromModal,
                         DownloadFileFromTable,
                         DownloadGraphFromExperimentPage,
                         DownloadMetaDataFromSearchPage,
                         DownloadTSVFromReportPage,
                         OpenUCSCGenomeBrowser,
                         OpenUCSCGenomeBrowserCE10,
                         OpenUCSCGenomeBrowserCE11,
                         OpenUCSCGenomeBrowserDM3,
                         OpenUCSCGenomeBrowserDM6,
                         OpenUCSCGenomeBrowserGRCh38,
                         OpenUCSCGenomeBrowserHG19,
                         OpenUCSCGenomeBrowserMM10,
                         OpenUCSCGenomeBrowserMM10Minimal,
                         OpenUCSCGenomeBrowserMM9)
from .comparisons import (BrowserComparison,
                          CompareFacetNumbersBetweenBrowsers,
                          CompareFacetNumbersBetweenURLS,
                          CompareScreenShots,
                          URLComparison)
from .defaults import (USERS,
                       BROWSERS,
                       bcolors,
                       ActionTuples)
from .tasks import (DownloadFiles,
                    Driver,
                    GetFacetNumbers,
                    GetScreenShot,
                    SeleniumTask,
                    SignIn)
from .worker import DataWorker, DataManager

################################################################
# QANCODE object gets data, compares data using defined tasks. #
################################################################

class QANCODE(ActionTuples):
    """
    Object to keep track of Task/Comparison combinations and run QA
    process with one method call.
    """

    def __init__(self, rc_url, prod_url='https://www.encodeproject.org'):
        self.rc_url = rc_url
        self.prod_url = prod_url
        self.browsers = [b for b in BROWSERS]
        self.users = [u for u in USERS]
        # Default action attributes inherited from ActionTuples.
        super().__init__()

    def __repr__(self):
        return 'QANCODE(prod_url={}, rc_url={})'.format(self.prod_url,
                                                        self.rc_url)

    def list_methods(self):
        """
        List all possible tests.
        """
        print(*[f for f in sorted(dir(QANCODE)) if callable(getattr(QANCODE, f))
                and not f.startswith('__') and not f.startswith('_')], sep='\n')

    def compare_facets(self,
                       browsers='all',
                       users='all',
                       item_types='all',
                       task=GetFacetNumbers,
                       url_comparison=True,
                       browser_comparison=True):
        """
        Gets RC URL facet numbers and compares them to production URL facet
        numbers for given item_type, browser, user.
        """
        # Define item type pages (search and matrix views) to check.
        all_item_types = self.compare_facets_default_actions
        if browsers == 'all':
            browsers = self.browsers
        if users == 'all':
            users = self.users
        if item_types == 'all':
            item_types = [t for t in all_item_types]
        urls = [self.prod_url, self.rc_url]
        click_paths = [None for c in item_types]
        dm = DataManager(browsers=browsers,
                         urls=urls,
                         users=users,
                         item_types=item_types,
                         click_paths=click_paths,
                         task=task)
        dm.run_tasks()
        if url_comparison:
            for browser in browsers:
                for user in users:
                    for item_type in item_types:
                        cfn_url = CompareFacetNumbersBetweenURLS(browser=browser,
                                                                 user=user,
                                                                 prod_url=self.prod_url,
                                                                 rc_url=self.rc_url,
                                                                 item_type=item_type,
                                                                 click_path=None,
                                                                 all_data=dm.all_data)
                        cfn_url.compare_data()
        if browser_comparison:
            for url in urls:
                for user in users:
                    for item_type in item_types:
                        cfn_browser = CompareFacetNumbersBetweenBrowsers(user=user,
                                                                         url=url,
                                                                         item_type=item_type,
                                                                         browsers=browsers,
                                                                         all_data=dm.all_data)
                        cfn_browser.compare_data()

    @staticmethod
    def _expand_action_list(actions):
        """
        Returns item_types and click_paths given list of tuples.
        """
        item_types = [t[0] for t in actions]
        click_paths = [p[1] for p in actions]
        return item_types, click_paths

    def _parse_arguments(self,
                         browsers,
                         users,
                         item_types,
                         click_paths,
                         action_tuples,
                         actions,
                         admin_only_actions,
                         public_only_actions):
        if browsers == 'all':
            browsers = self.browsers
        if users == 'all':
            users = self.users
        if action_tuples is not None:
            item_types, click_paths = self._expand_action_list(action_tuples)
        else:
            if item_types == 'all':
                item_types, click_paths = self._expand_action_list(actions)
            elif item_types == 'random':
                # Choose random item_types from action list to check.
                # Note np.random.choice won't work for tuples.
                random_actions = [actions[i]
                                  for i in np.random.randint(len(actions) - 1, size=3)]
                item_types, click_paths = self._expand_action_list(
                    random_actions)
            elif item_types == 'admin':
                item_types, click_paths = self._expand_action_list(
                    admin_only_actions)
            elif item_types == 'public':
                item_types, click_paths = self._expand_action_list(
                    public_only_actions)
            else:
                if len(item_types) == len(click_paths):
                    pass
                elif len(click_paths) == 1:
                    # Broadcast single click_path (e.g. None) to all user-defined item_types.
                    click_paths = [click_paths[0] for t in item_types]
                else:
                    raise ValueError(
                        'item_types and click_paths must have same length')
        return browsers, users, item_types, click_paths

    def find_differences(self,
                         browsers='all',
                         users='all',
                         action_tuples=None,
                         item_types='all',
                         click_paths=[None],
                         task=GetScreenShot):
        """
        Does image diff for given item_types. If click_path defined will
        perform that action before taking screenshot.
        """
        # Tuple of (item_type, click_path)
        actions = self.find_differences_default_actions
        admin_only_actions = [('/biosamples/ENCBS681LAC/', None),
                              ('/search/?searchTerm=ENCBS681LAC&type=Biosample', None)]
        public_only_actions = [('/experiments/?status=deleted', None)]
        browsers, users, item_types, click_paths = self._parse_arguments(browsers=browsers,
                                                                         users=users,
                                                                         item_types=item_types,
                                                                         click_paths=click_paths,
                                                                         action_tuples=action_tuples,
                                                                         actions=actions,
                                                                         admin_only_actions=admin_only_actions,
                                                                         public_only_actions=public_only_actions)
        urls = [self.prod_url, self.rc_url]
        results = []
        with tempfile.TemporaryDirectory() as td:
            dm = DataManager(browsers=browsers,
                             urls=urls,
                             users=users,
                             item_types=item_types,
                             click_paths=click_paths,
                             task=task,
                             temp_dir=td)
            dm.run_tasks()
            for browser in browsers:
                for user in users:
                    for item_type, click_path in zip(item_types, click_paths):
                        css = CompareScreenShots(browser=browser,
                                                 user=user,
                                                 prod_url=self.prod_url,
                                                 rc_url=self.rc_url,
                                                 item_type=item_type,
                                                 click_path=click_path,
                                                 all_data=dm.all_data)
                        result = css.compare_data()
                        results.append(result)
        self.find_differences_output = results
        return self.find_differences_output

    def show_differences(self):
        """
        This is a Jupyter notebook helper function that loads image diffs
        into the notebook and displays them. Won't work in terminal.
        """
        try:
            # In Jupyter notebook?
            get_ipython().config
        except NameError:
            print('Must use in Jupyter notebook')
            return
        from IPython.core.display import display, HTML, Image
        try:
            for out in self.find_differences_output:
                if out[0]:
                    print('{}:'.format(out[1]))
                    image_diff_path = os.path.expanduser(
                        '~/Desktop/image_diff/')
                    display(Image(filename='{}{}'.format(image_diff_path,
                                                         out[1])))
                    print('\n')
        except AttributeError:
            print('No image diffs found')

    def check_trackhubs(self, browsers=['Safari'], users=['Public'], action_tuples=None):
        """
        Runs find_differences() image diff for selected list of trackhub
        actions.
        """
        print('Running check trackhubs')
        trackhub_actions = self.check_trackhubs_default_actions
        if action_tuples is None:
            action_tuples = trackhub_actions
        self.find_differences(users=users, browsers=browsers,
                              action_tuples=action_tuples)

    def check_permissions(self, browsers=['Safari'], users=['Public']):
        """
        Runs find_differences() image diff on permission check pages.
        """
        print('Running check permissions')
        permission_actions = self.check_permissions_default_actions
        self.find_differences(users=users, browsers=browsers,
                              action_tuples=permission_actions)

    def check_downloads(self,
                        browsers='all',
                        users='all',
                        action_tuples=None,
                        item_types='all',
                        click_paths=[None],
                        task=DownloadFiles,
                        urls='all',
                        delete=True):
        """
        Clicks download button and checks download folder for file.
        """
        print('Running check downloads')
        actions = self.check_downloads_default_actions
        admin_only_actions = []
        public_only_actions = []
        browsers, users, item_types, click_paths = self._parse_arguments(browsers=browsers,
                                                                         users=users,
                                                                         item_types=item_types,
                                                                         click_paths=click_paths,
                                                                         action_tuples=action_tuples,
                                                                         actions=actions,
                                                                         admin_only_actions=admin_only_actions,
                                                                         public_only_actions=public_only_actions)
        if urls == 'all':
            urls = [self.prod_url, self.rc_url]
        dm = DataManager(browsers=browsers,
                         urls=urls,
                         users=users,
                         item_types=item_types,
                         click_paths=click_paths,
                         task=task,
                         delete=delete)
        dm.run_tasks()

    @staticmethod
    def _create_authentication_header(headers, authid, authpw):
        headers['Authorization'] = requests.auth._basic_auth_str(
            authid, authpw)
        return headers

    @staticmethod
    def _block_production_edit(url):
        # Don't test production.
        if 'encodeproject.org' in url:
            raise SystemError('No editing production.')
        return None

    def _get_auth_credentials(self, user):
        if user == 'Public':
            return None, None
        cred_file = os.path.expanduser('~/qa_credentials.json')
        with open(cred_file) as f:
            creds = [(k['authid'], k['authpw']) for k
                     in json.load(f) if k['username'] == user]
        if not creds:
            raise ValueError('User not found.')
        return creds[0]

    def check_requests(self):
        """
        Runs post, patch, and get requests as different users.
        """
        url = self.rc_url
        self._block_production_edit(url)
        action_dict = self.check_requests_default_actions
        for k, v in action_dict.items():
            for item in v:
                request_url = urllib.parse.urljoin(url, item[0])
                for user, status_code in item[2]:
                    headers = {'content-type': 'application/json',
                               'accept': 'application/json'}
                    (authid, authpw) = self._get_auth_credentials(user)
                    if authid is not None:
                        headers = self._create_authentication_header(
                            headers, authid, authpw)
                    for payload in item[1]:
                        print('Trying to {} {} with {} as user {}'.format(
                            k, request_url, payload, user))
                        print('Expected: {}'.format(status_code))
                        r = requests.request(
                            k, url=request_url, headers=headers, data=json.dumps(payload))
                        try:
                            print('Actual: {}'.format(r.status_code))
                            assert r.status_code == status_code
                            print('{}{} SUCCESSFUL{}'.format(
                                bcolors.OKBLUE, k.upper(), bcolors.ENDC))
                        except AssertionError:
                            print('{}{} FAILURE\n{}{}'.format(
                                bcolors.FAIL, k.upper(), r.text, bcolors.ENDC))
                        finally:
                            print()

    def _add_rc_to_keypairs(self, url):
        """
        Gets production keypair and copies it to rc_url server.
        """
        keypairs = os.path.expanduser('~/keypairs.json')
        try:
            with open(keypairs, 'r') as f:
                data = json.load(f)
                prod_data = data['prod']
                authid, authpw = prod_data['key'], prod_data['secret']
                data['current_rc'] = {'server': url,
                                      'key': authid,
                                      'secret': authpw}
            # Overwrite.
            with open(keypairs, 'w') as f:
                json.dump(data, f)
        except Exception as e:
            print(e)
            raise

    def check_tools(self):
        """
        Runs pyencoded-tools against RC.
        """
        url = self.rc_url
        self._block_production_edit(url)
        self._add_rc_to_keypairs(url)
        key = '--key current_rc'
        # Expected outut a very weak check.
        tools = self.check_tools_default_actions
        # Get relative Python executable.
        python_executor = sys.executable
        for tool in tools:
            print('Running {}'.format(tool['name']))
            command = tool['command'].format(
                python_executor, tool['name'], key)
            print(command)
            output = subprocess.check_output(
                command, shell=True).decode('utf-8').strip()
            try:
                assert ((output == tool['expected_output'].strip())
                        or (tool['expected_output'] in output))
                print('{}{} SUCCESSFUL{}'.format(
                    bcolors.OKBLUE, tool['name'].upper(), bcolors.ENDC))
            except AssertionError:
                print('{}{} FAILURE{}'.format(
                    bcolors.FAIL, tool['name'].upper(), bcolors.ENDC))
                print('Expected: ')
                print(tool['expected_output'])
                print('Actual: ')
                print(output)

    @staticmethod
    def _color_by_value(value):
        # Return color based on value.
        if value < 150.0:
            return bcolors.OKBLUE
        elif value >= 400.0:
            return bcolors.FAIL
        return bcolors.WARNING

    @staticmethod
    def _get_time_headers(url, n):
        time_headers = []
        for i in range(n):
            r = requests.get(url)
            assert r.status_code == 200
            time_headers.append(r.headers)
        return time_headers

    @staticmethod
    def _parse_header(headers):
        headers_split = [(h.split('=')) for h in headers['X-Stats'].split('&')]
        headers_dict = {h[0]: float(h[1])
                        for h in headers_split if 'time' in h[0]}
        return headers_dict

    @staticmethod
    def _summary_for_category(values):
        return round(np.mean(values) / 1000, 3), round(np.std(values) / 1000, 3), len(values)

    @staticmethod
    def _calculate_total_times(values):
        return [sum(item.values()) for item in values]

    @staticmethod
    def _print_header(url):
        break_size = 80 if len(url) < 70 else len(url) + 20
        print(' {} '.format(url).center(break_size, '-'))

    def _print_results(self, title, mean, std, count):
        print('{}Average {}: {} ± {} ms (n={}){}'.format(self._color_by_value(mean),
                                                         title,
                                                         mean,
                                                         std,
                                                         count,
                                                         bcolors.ENDC))

    def _average_time_for_get(self, url, n):
        self._print_header(url)
        time_headers = self._get_time_headers(url, n)
        parsed_headers = [self._parse_header(h) for h in time_headers]
        for key in sorted(parsed_headers[0].keys()):
            group_values = [v[key] for v in parsed_headers]
            group_mean, group_std, group_count = self._summary_for_category(
                group_values)
            self._print_results(key, group_mean, group_std, group_count)
        total_mean, total_std, total_count = self._summary_for_category(
            self._calculate_total_times(parsed_headers))
        self._print_results('total time', total_mean, total_std, total_count)

    def check_response_time(self, urls=None, item_types=[None], n=10):
        if urls is None:
            urls = [self.prod_url, self.rc_url]
        print('Checking response time')
        for item in item_types:
            print('\n*** item_type: {}'.format(item))
            for url in urls:
                if item is not None:
                    url = url + item
                self._average_time_for_get(url, n)
            print()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
import os.path as op
import pandas as pd
import matplotlib
matplotlib.use('Agg')

import matplotlib.pyplot as plt
from matplotlib.backends.backend_pdf import PdfPages

from .plotting import (plot_measures, plot_mosaic, plot_all,
                       plot_fd, plot_dist)

# matplotlib.rc('figure', figsize=(11.69, 8.27))  # for DINA4 size

def get_documentation(doc_type, out_file):
    from xhtml2pdf import pisa
    # open output file for writing (truncated binary)
    result = open(out_file, "w+b")

    html_dir = op.abspath(
        op.join(op.dirname(__file__), 'html', '%s.html' % doc_type))

    with open(html_dir, 'r') as f:
        html = f.read()

    # convert HTML to PDF
    status = pisa.CreatePDF(html, dest=result)
    result.close()

    # return True on success and False on errors
    return status.err

def _read_csv(in_csv):
    df = pd.read_csv(in_csv).fillna(0).sort(['subject', 'session'])
    # df['subject'] = df['subject'].astype(str)
    return df

def concat_pdf(in_files, out_file='concatenated.pdf'):
    """
    Concatenate PDF list (http://stackoverflow.com/a/3444735)
    """
    from PyPDF2 import PdfFileWriter, PdfFileReader
    outpdf = PdfFileWriter()

    for in_file in in_files:
        inpdf = PdfFileReader(file(in_file, 'rb'))
        for p in range(inpdf.numPages):
            outpdf.addPage(inpdf.getPage(p))
    outpdf.write(file(out_file, 'wb'))
    return out_file

def _write_report(df, groups, sub_id=None, sc_split=False, condensed=True,
                  out_file='report.pdf'):
    headers = [v for gnames in groups for v in gnames]
    sessions = sorted(pd.unique(df.session.ravel()))

    columns = df.columns.ravel()

    for h in headers:
        if h not in columns:
            headers.remove(h)
            for g in groups:
                if h in g:
                    g.remove(h)

    report = PdfPages(out_file)
    for ss in sessions:
        sesdf = df.copy().loc[df['session'] == ss]
        scans = pd.unique(sesdf.scan.ravel())
        if sc_split:
            for sc in scans:
                subset = sesdf.loc[sesdf['scan'] == sc]
                if len(subset.index) > 1:
                    if sub_id is None:
                        subtitle = '(%s_%s)' % (ss, sc)
                    else:
                        subtitle = '(subject %s_%s_%s)' % (sub_id, ss, sc)
                    if condensed:
                        fig = plot_all(sesdf, groups, subject=sub_id,
                                       title='QC measures ' + subtitle)
                    else:
                        fig = plot_measures(
                            sesdf, headers, subject=sub_id,
                            title='QC measures ' + subtitle)
                    report.savefig(fig, dpi=300)
                    fig.clf()
        else:
            if len(sesdf.index) > 1:
                if sub_id is None:
                    subtitle = '(%s)' % (ss)
                else:
                    subtitle = '(subject %s_%s)' % (sub_id, ss)
                if condensed:
                    fig = plot_all(sesdf, groups, subject=sub_id,
                                   title='QC measures ' + subtitle)
                else:
                    fig = plot_measures(
                        sesdf, headers, subject=sub_id,
                        title='QC measures ' + subtitle)
                report.savefig(fig, dpi=300)
                fig.clf()

    report.close()
    plt.close()
    # print 'Written report file %s' % out_file
    return out_file

def _write_all_reports(df, groups, sc_split=False, condensed=True,
                       out_file='report.pdf'):

    outlist = []
    _write_report(
        df, groups, sc_split=sc_split, condensed=condensed, out_file=out_file)

    subject_list = sorted(pd.unique(df.subject.ravel()))
    for sub_id in subject_list:
        tpl, _ = op.splitext(op.basename(out_file))
        tpl = op.join(op.dirname(out_file), tpl) + '_%s.pdf'
        outlist.append(_write_report(
            df, groups, sub_id=sub_id, sc_split=sc_split, condensed=condensed,
            out_file=tpl % sub_id))
    return out_file, outlist

def all_anatomical(in_csv, sc_split=False, condensed=True,
                   out_file='anatomical.pdf'):
    groups = [['bg_size', 'fg_size'],
              ['bg_mean', 'fg_mean'],
              ['bg_std', 'fg_std'],
              ['csf_size', 'gm_size', 'wm_size'],
              ['csf_mean', 'gm_mean', 'wm_mean'],
              ['csf_std', 'gm_std', 'wm_std'],
              ['cnr'],
              ['efc'],
              ['fber'],
              ['fwhm', 'fwhm_x', 'fwhm_y', 'fwhm_z'],
              ['qi1'],
              ['snr']]
    return _write_all_reports(
        _read_csv(in_csv), groups, sc_split=sc_split,
        condensed=condensed, out_file=out_file)

def all_func_temporal(in_csv, sc_split=False, condensed=True,
                      out_file='func_temporal.pdf'):
    groups = [['dvars'], ['gcor'], ['m_tsnr'], ['mean_fd'],
              ['num_fd'], ['outlier'], ['perc_fd'], ['quality']]
    return _write_all_reports(
        _read_csv(in_csv), groups, sc_split=sc_split,
        condensed=condensed, out_file=out_file)

def all_func_spatial(in_csv, sc_split=False, condensed=False,
                     out_file='func_spatial.pdf'):
    groups = [['bg_size', 'fg_size'],
              ['bg_mean', 'fg_mean'],
              ['bg_std', 'fg_std'],
              ['efc'],
              ['fber'],
              ['fwhm', 'fwhm_x', 'fwhm_y', 'fwhm_z'],
              ['ghost_%s' % a for a in ['x', 'y', 'z']],
              ['snr']]
    return _write_all_reports(
        _read_csv(in_csv), groups, sc_split=sc_split,
        condensed=condensed, out_file=out_file)

def report_anatomical(in_csv, subject=None, sc_split=False, condensed=True,
                      out_file='anatomical.pdf'):
    groups = [['bg_size', 'fg_size'],
              ['bg_mean', 'fg_mean'],
              ['bg_std', 'fg_std'],
              ['csf_size', 'gm_size', 'wm_size'],
              ['csf_mean', 'gm_mean', 'wm_mean'],
              ['csf_std', 'gm_std', 'wm_std'],
              ['cnr'],
              ['efc'],
              ['fber'],
              ['fwhm', 'fwhm_x', 'fwhm_y', 'fwhm_z'],
              ['qi1'],
              ['snr']]
    return _write_report(
        _read_csv(in_csv), groups, sub_id=subject,
        sc_split=sc_split, condensed=condensed, out_file=out_file)

def report_func_temporal(in_csv, subject=None, sc_split=False, condensed=True,
                         out_file='func_temporal.pdf'):
    groups = [['dvars'], ['gcor'], ['m_tsnr'], ['mean_fd'],
              ['num_fd'], ['outlier'], ['perc_fd'], ['quality']]
    return _write_report(
        _read_csv(in_csv), groups, sub_id=subject,
        sc_split=sc_split, condensed=condensed, out_file=out_file)

def report_func_spatial(in_csv, subject=None, sc_split=False, condensed=False,
                        out_file='func_spatial.pdf'):
    groups = [['bg_size', 'fg_size'],
              ['bg_mean', 'fg_mean'],
              ['bg_std', 'fg_std'],
              ['efc'],
              ['fber'],
              ['fwhm', 'fwhm_x', 'fwhm_y', 'fwhm_z'],
              ['ghost_%s' % a for a in ['x', 'y', 'z']],
              ['snr']]
    return _write_report(
        _read_csv(in_csv), groups, sub_id=subject,
        sc_split=sc_split, condensed=condensed, out_file=out_file)

<commit_msg>fix default parameter on functional_spatial<commit_after>#!/usr/bin/env python
# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
import os.path as op
import pandas as pd
import matplotlib
matplotlib.use('Agg')

import matplotlib.pyplot as plt
from matplotlib.backends.backend_pdf import PdfPages

from .plotting import (plot_measures, plot_mosaic, plot_all,
                       plot_fd, plot_dist)

# matplotlib.rc('figure', figsize=(11.69, 8.27))  # for DINA4 size

def get_documentation(doc_type, out_file):
    from xhtml2pdf import pisa
    # open output file for writing (truncated binary)
    result = open(out_file, "w+b")

    html_dir = op.abspath(
        op.join(op.dirname(__file__), 'html', '%s.html' % doc_type))

    with open(html_dir, 'r') as f:
        html = f.read()

    # convert HTML to PDF
    status = pisa.CreatePDF(html, dest=result)
    result.close()

    # return True on success and False on errors
    return status.err

def _read_csv(in_csv):
    df = pd.read_csv(in_csv).fillna(0).sort(['subject', 'session'])
    # df['subject'] = df['subject'].astype(str)
    return df

def concat_pdf(in_files, out_file='concatenated.pdf'):
    """
    Concatenate PDF list (http://stackoverflow.com/a/3444735)
    """
    from PyPDF2 import PdfFileWriter, PdfFileReader
    outpdf = PdfFileWriter()

    for in_file in in_files:
        inpdf = PdfFileReader(file(in_file, 'rb'))
        for p in range(inpdf.numPages):
            outpdf.addPage(inpdf.getPage(p))
    outpdf.write(file(out_file, 'wb'))
    return out_file

def _write_report(df, groups, sub_id=None, sc_split=False, condensed=True,
                  out_file='report.pdf'):
    headers = [v for gnames in groups for v in gnames]
    sessions = sorted(pd.unique(df.session.ravel()))

    columns = df.columns.ravel()

    for h in headers:
        if h not in columns:
            headers.remove(h)
            for g in groups:
                if h in g:
                    g.remove(h)

    report = PdfPages(out_file)
    for ss in sessions:
        sesdf = df.copy().loc[df['session'] == ss]
        scans = pd.unique(sesdf.scan.ravel())
        if sc_split:
            for sc in scans:
                subset = sesdf.loc[sesdf['scan'] == sc]
                if len(subset.index) > 1:
                    if sub_id is None:
                        subtitle = '(%s_%s)' % (ss, sc)
                    else:
                        subtitle = '(subject %s_%s_%s)' % (sub_id, ss, sc)
                    if condensed:
                        fig = plot_all(sesdf, groups, subject=sub_id,
                                       title='QC measures ' + subtitle)
                    else:
                        fig = plot_measures(
                            sesdf, headers, subject=sub_id,
                            title='QC measures ' + subtitle)
                    report.savefig(fig, dpi=300)
                    fig.clf()
        else:
            if len(sesdf.index) > 1:
                if sub_id is None:
                    subtitle = '(%s)' % (ss)
                else:
                    subtitle = '(subject %s_%s)' % (sub_id, ss)
                if condensed:
                    fig = plot_all(sesdf, groups, subject=sub_id,
                                   title='QC measures ' + subtitle)
                else:
                    fig = plot_measures(
                        sesdf, headers, subject=sub_id,
                        title='QC measures ' + subtitle)
                report.savefig(fig, dpi=300)
                fig.clf()

    report.close()
    plt.close()
    # print 'Written report file %s' % out_file
    return out_file

def _write_all_reports(df, groups, sc_split=False, condensed=True,
                       out_file='report.pdf'):

    outlist = []
    _write_report(
        df, groups, sc_split=sc_split, condensed=condensed, out_file=out_file)

    subject_list = sorted(pd.unique(df.subject.ravel()))
    for sub_id in subject_list:
        tpl, _ = op.splitext(op.basename(out_file))
        tpl = op.join(op.dirname(out_file), tpl) + '_%s.pdf'
        outlist.append(_write_report(
            df, groups, sub_id=sub_id, sc_split=sc_split, condensed=condensed,
            out_file=tpl % sub_id))
    return out_file, outlist

def all_anatomical(in_csv, sc_split=False, condensed=True,
                   out_file='anatomical.pdf'):
    groups = [['bg_size', 'fg_size'],
              ['bg_mean', 'fg_mean'],
              ['bg_std', 'fg_std'],
              ['csf_size', 'gm_size', 'wm_size'],
              ['csf_mean', 'gm_mean', 'wm_mean'],
              ['csf_std', 'gm_std', 'wm_std'],
              ['cnr'],
              ['efc'],
              ['fber'],
              ['fwhm', 'fwhm_x', 'fwhm_y', 'fwhm_z'],
              ['qi1'],
              ['snr']]
    return _write_all_reports(
        _read_csv(in_csv), groups, sc_split=sc_split,
        condensed=condensed, out_file=out_file)

def all_func_temporal(in_csv, sc_split=False, condensed=True,
                      out_file='func_temporal.pdf'):
    groups = [['dvars'], ['gcor'], ['m_tsnr'], ['mean_fd'],
              ['num_fd'], ['outlier'], ['perc_fd'], ['quality']]
    return _write_all_reports(
        _read_csv(in_csv), groups, sc_split=sc_split,
        condensed=condensed, out_file=out_file)

def all_func_spatial(in_csv, sc_split=False, condensed=False,
                     out_file='func_spatial.pdf'):
    groups = [['bg_size', 'fg_size'],
              ['bg_mean', 'fg_mean'],
              ['bg_std', 'fg_std'],
              ['efc'],
              ['fber'],
              ['fwhm', 'fwhm_x', 'fwhm_y', 'fwhm_z'],
              ['ghost_%s' % a for a in ['x', 'y', 'z']],
              ['snr']]
    return _write_all_reports(
        _read_csv(in_csv), groups, sc_split=sc_split,
        condensed=condensed, out_file=out_file)

def report_anatomical(in_csv, subject=None, sc_split=False, condensed=True,
                      out_file='anatomical.pdf'):
    groups = [['bg_size', 'fg_size'],
              ['bg_mean', 'fg_mean'],
              ['bg_std', 'fg_std'],
              ['csf_size', 'gm_size', 'wm_size'],
              ['csf_mean', 'gm_mean', 'wm_mean'],
              ['csf_std', 'gm_std', 'wm_std'],
              ['cnr'],
              ['efc'],
              ['fber'],
              ['fwhm', 'fwhm_x', 'fwhm_y', 'fwhm_z'],
              ['qi1'],
              ['snr']]
    return _write_report(
        _read_csv(in_csv), groups, sub_id=subject,
        sc_split=sc_split, condensed=condensed, out_file=out_file)

def report_func_temporal(in_csv, subject=None, sc_split=False, condensed=True,
                         out_file='func_temporal.pdf'):
    groups = [['dvars'], ['gcor'], ['m_tsnr'], ['mean_fd'],
              ['num_fd'], ['outlier'], ['perc_fd'], ['quality']]
    return _write_report(
        _read_csv(in_csv), groups, sub_id=subject,
        sc_split=sc_split, condensed=condensed, out_file=out_file)

def report_func_spatial(in_csv, subject=None, sc_split=False, condensed=True,
                        out_file='func_spatial.pdf'):
    groups = [['bg_size', 'fg_size'],
              ['bg_mean', 'fg_mean'],
              ['bg_std', 'fg_std'],
              ['efc'],
              ['fber'],
              ['fwhm', 'fwhm_x', 'fwhm_y', 'fwhm_z'],
              ['ghost_%s' % a for a in ['x', 'y', 'z']],
              ['snr']]
    return _write_report(
        _read_csv(in_csv), groups, sub_id=subject,
        sc_split=sc_split, condensed=condensed, out_file=out_file)

<|endoftext|>