<commit_before># GUI Application automation and testing library
# Copyright (C) 2006 Mark Mc Mahon
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public License
# as published by the Free Software Foundation; either version 2.1
# of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
#    Free Software Foundation, Inc.,
#    59 Temple Place,
#    Suite 330,
#    Boston, MA 02111-1307 USA

"""Repeated Hotkeys Test

**What is checked**
This test checks all the controls in a dialog to see if there are controls that
use the same hotkey character.

**How is it checked**
A list of all the hotkeys (converted to uppercase) used in the dialog is
created. Then this list is examined to see if any hotkeys are used more than
once. If any are used more than once a list of all the controls that use this
hotkey are compiled to be used in the bug report.

**When is a bug reported**
If more than one control has the same hotkey then a bug is reported.

**Bug Extra Information**
The bug contains the following extra information
Name	Description
RepeatedHotkey	This is the hotkey that is repeated between the 2 controls
converted to uppercase, String
CharsUsedInDialog	This is a list of all the hotkeys used in the dialog,
String
AllCharsInDialog	This is a list of all the characters in the dialog for
controls that have a hotkeys, String
AvailableInControlS	A list of the available characters for each control.
Any of the characters in this list could be used as the new hotkey without
conflicting with any existing hotkey.

**Is Reference dialog needed**
The reference dialog does not need to be available. If it is available then
for each bug discovered it is checked to see if it is a problem in the
reference dialog.
NOTE: Checking the reference dialog is not so exact here! Only when the
equivalent controls in the reference dialog all have the hotkeys will it be
reported as being in the reference also. I.e. if there are 3 controls with the
same hotkey in the Localised software  then those same controls in the
reference dialog must have the same hotkey for it to be reported as existing
in the reference also.

**False positive bug reports**
There should be very few false positives from this test. Sometimes a control
only has one or 2 characters eg "X:" and it is impossible to avoid a hotkey
clash. Also for Asian languages hotkeys should be the same as the US software
so probably this test should be run on those languages.

**Test Identifier**
The identifier for this test/bug is "RepeatedHotkey"
"""

testname = "RepeatedHotkey"
__revision__ = "$Revision$"

from pywinauto.win32defines import SS_NOPREFIX

#-----------------------------------------------------------------------------
def RepeatedHotkeyTest(windows):
    "Return the repeated hotkey errors"

    hotkeyControls, allChars, hotkeys = _CollectDialogInfo(windows)

    # get the available characters in the dialog
    dlgAvailable = allChars.difference(hotkeys)

    # remove some characters that aren't good choices for hotkeys
    dlgAvailable.difference_update(set("-& _"))

    bugs = []
    # for each hotkey
    for char, controls in hotkeyControls.items():

        # if there is more than one control associated then it is a bug
        if len(controls) > 1:

            ctrlsAvailableChars = ""

            # build up the available characters for each control
            for ctrl in controls:
                controlChars = ""
                controlChars = set(ctrl.WindowText().lower())

                controlAvailableChars = controlChars.intersection(dlgAvailable)
                controlAvailableChars = \
                    "<%s>" % _SetAsString(controlAvailableChars)

                ctrlsAvailableChars += controlAvailableChars

            refCtrls = [ctrl.ref for ctrl in controls if ctrl.ref]
            refHotkeyControls, refAllChars, refHotkeys = \
                _CollectDialogInfo(refCtrls)

            isInRef = -1
            if len(refHotkeys) > 1:
                isInRef = 1
            else:
                isInRef = 0

            bugs.append((
                controls,
                {
                    "RepeatedHotkey" : 		char,
                    "CharsUsedInDialog" :   _SetAsString(hotkeys),
                    "AllCharsInDialog" :    _SetAsString(allChars),
                    "AvailableInControls" : ctrlsAvailableChars,
                },
                testname,
                isInRef)
            )

#	# What is the algorithm to try and do all that is necessary to find
#	# if it is possible to get a fix character if none of the current
#	# characters are free
#	for bug in bugs:
#		for c, chars in bug.bugData:
#			# control has no possibilities
#			if not chars:
#				# check if there are any other hotkey controls
#				# in the dialog that could be used
#				others = set(c.Title.lower()).intersection(unUsedChars)

    return  bugs

#-----------------------------------------------------------------------------
def _CollectDialogInfo(windows):
    "Collect information on the hotkeys in the dialog"
    hotkeyControls = {}
    allChars = ''

    for win in windows:
        # skip it if it doesn't implement hotkey functionality
        if not ImplementsHotkey(win):
            continue

        # get the hotkey
        pos, char = GetHotkey(win.WindowText())

        # if no hotkey for this control
        # then continue
        if not char:
            continue

        # store hotkey with list of used hotkeys
        # map this hotkey to the list of controls that have it
        hotkeyControls.setdefault(char.lower(), []).append(win)

        # Add the title of this control to the list of available
        # characters for the dialog
        allChars += win.WindowText().lower()

    allChars = set(allChars)
    hotkeys = set(hotkeyControls.keys())

    return hotkeyControls, allChars, hotkeys

#-----------------------------------------------------------------------------
# get hokey position
def GetHotkey(text):
    "Return the position and character of the hotkey"

    # find the last & character that is not followed
    # by & or by the end of the string

    curEnd = len(text) + 1
    text = text.replace("&&", "__")
    while True:
        pos = text.rfind("&", 0, curEnd)

        # One found was at the end of the text or
        # no (more) & were found
        # so return the None value
        if pos == -1 or pos == len(text):
            return (-1, '')

        # One found but was prededed by non valid hotkey character
        # so continue, as that can't be a shortcut
        if text[pos - 1] == '&':
            curEnd = pos - 2
            continue

        # 2 ampersands in a row - so skip
        # the 1st one and continue
        return (pos, text[pos+1])

#-----------------------------------------------------------------------------
def _SetAsString(settojoin):
    "Convert the set to a ordered string"
    return "".join(sorted(settojoin))

#-----------------------------------------------------------------------------
def ImplementsHotkey(win):
    "checks whether a control interprets & character to be a hotkey"

    # buttons always implement hotkey
    if win.Class() == "Button":
        return True

    # Statics do if they don't have SS_NOPREFIX style
    elif win.Class() == "Static" and not win.HasStyle(SS_NOPREFIX):
        return True

    if win.Class() == "MenuItem" and win.State() != "2048":
        return True

    # Most controls don't - so just return false if
    # neither of the above condition hold
    return False

RepeatedHotkeyTest.TestsMenus = True

<commit_msg>Changes for 2.3<commit_after># GUI Application automation and testing library
# Copyright (C) 2006 Mark Mc Mahon
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public License
# as published by the Free Software Foundation; either version 2.1
# of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
#    Free Software Foundation, Inc.,
#    59 Temple Place,
#    Suite 330,
#    Boston, MA 02111-1307 USA

"""Repeated Hotkeys Test

**What is checked**
This test checks all the controls in a dialog to see if there are controls that
use the same hotkey character.

**How is it checked**
A list of all the hotkeys (converted to uppercase) used in the dialog is
created. Then this list is examined to see if any hotkeys are used more than
once. If any are used more than once a list of all the controls that use this
hotkey are compiled to be used in the bug report.

**When is a bug reported**
If more than one control has the same hotkey then a bug is reported.

**Bug Extra Information**
The bug contains the following extra information
Name	Description
RepeatedHotkey	This is the hotkey that is repeated between the 2 controls
converted to uppercase, String
CharsUsedInDialog	This is a list of all the hotkeys used in the dialog,
String
AllCharsInDialog	This is a list of all the characters in the dialog for
controls that have a hotkeys, String
AvailableInControlS	A list of the available characters for each control.
Any of the characters in this list could be used as the new hotkey without
conflicting with any existing hotkey.

**Is Reference dialog needed**
The reference dialog does not need to be available. If it is available then
for each bug discovered it is checked to see if it is a problem in the
reference dialog.
NOTE: Checking the reference dialog is not so exact here! Only when the
equivalent controls in the reference dialog all have the hotkeys will it be
reported as being in the reference also. I.e. if there are 3 controls with the
same hotkey in the Localised software  then those same controls in the
reference dialog must have the same hotkey for it to be reported as existing
in the reference also.

**False positive bug reports**
There should be very few false positives from this test. Sometimes a control
only has one or 2 characters eg "X:" and it is impossible to avoid a hotkey
clash. Also for Asian languages hotkeys should be the same as the US software
so probably this test should be run on those languages.

**Test Identifier**
The identifier for this test/bug is "RepeatedHotkey"
"""

testname = "RepeatedHotkey"
__revision__ = "$Revision$"

# to be able to run on Python 2.3 we need to use sets module
# rather then built-in set object

import sets

from pywinauto.win32defines import SS_NOPREFIX

#-----------------------------------------------------------------------------
def RepeatedHotkeyTest(windows):
    "Return the repeated hotkey errors"

    hotkeyControls, allChars, hotkeys = _CollectDialogInfo(windows)

    # get the available characters in the dialog
    dlgAvailable = allChars.difference(hotkeys)

    # remove some characters that aren't good choices for hotkeys
    dlgAvailable.difference_update(sets.Set("-& _"))

    bugs = []
    # for each hotkey
    for char, controls in hotkeyControls.items():

        # if there is more than one control associated then it is a bug
        if len(controls) > 1:

            ctrlsAvailableChars = ""

            # build up the available characters for each control
            for ctrl in controls:
                controlChars = ""
                controlChars = sets.Set(ctrl.WindowText().lower())

                controlAvailableChars = controlChars.intersection(dlgAvailable)
                controlAvailableChars = \
                    "<%s>" % _SetAsString(controlAvailableChars)

                ctrlsAvailableChars += controlAvailableChars

            refCtrls = [ctrl.ref for ctrl in controls if ctrl.ref]
            refHotkeyControls, refAllChars, refHotkeys = \
                _CollectDialogInfo(refCtrls)

            isInRef = -1
            if len(refHotkeys) > 1:
                isInRef = 1
            else:
                isInRef = 0

            bugs.append((
                controls,
                {
                    "RepeatedHotkey" : 		char,
                    "CharsUsedInDialog" :   _SetAsString(hotkeys),
                    "AllCharsInDialog" :    _SetAsString(allChars),
                    "AvailableInControls" : ctrlsAvailableChars,
                },
                testname,
                isInRef)
            )

#	# What is the algorithm to try and do all that is necessary to find
#	# if it is possible to get a fix character if none of the current
#	# characters are free
#	for bug in bugs:
#		for c, chars in bug.bugData:
#			# control has no possibilities
#			if not chars:
#				# check if there are any other hotkey controls
#				# in the dialog that could be used
#				others = set(c.Title.lower()).intersection(unUsedChars)

    return  bugs

#-----------------------------------------------------------------------------
def _CollectDialogInfo(windows):
    "Collect information on the hotkeys in the dialog"
    hotkeyControls = {}
    allChars = ''

    for win in windows:
        # skip it if it doesn't implement hotkey functionality
        if not ImplementsHotkey(win):
            continue

        # get the hotkey
        pos, char = GetHotkey(win.WindowText())

        # if no hotkey for this control
        # then continue
        if not char:
            continue

        # store hotkey with list of used hotkeys
        # map this hotkey to the list of controls that have it
        hotkeyControls.setdefault(char.lower(), []).append(win)

        # Add the title of this control to the list of available
        # characters for the dialog
        allChars += win.WindowText().lower()

    allChars = sets.Set(allChars)
    hotkeys = sets.Set(hotkeyControls.keys())

    return hotkeyControls, allChars, hotkeys

#-----------------------------------------------------------------------------
# get hokey position
def GetHotkey(text):
    "Return the position and character of the hotkey"

    # find the last & character that is not followed
    # by & or by the end of the string

    curEnd = len(text) + 1
    text = text.replace("&&", "__")
    while True:
        pos = text.rfind("&", 0, curEnd)

        # One found was at the end of the text or
        # no (more) & were found
        # so return the None value
        if pos == -1 or pos == len(text):
            return (-1, '')

        # One found but was prededed by non valid hotkey character
        # so continue, as that can't be a shortcut
        if text[pos - 1] == '&':
            curEnd = pos - 2
            continue

        # 2 ampersands in a row - so skip
        # the 1st one and continue
        return (pos, text[pos+1])

#-----------------------------------------------------------------------------
def _SetAsString(settojoin):
    "Convert the set to a ordered string"
    return "".join(sorted(settojoin))

#-----------------------------------------------------------------------------
def ImplementsHotkey(win):
    "checks whether a control interprets & character to be a hotkey"

    # buttons always implement hotkey
    if win.Class() == "Button":
        return True

    # Statics do if they don't have SS_NOPREFIX style
    elif win.Class() == "Static" and not win.HasStyle(SS_NOPREFIX):
        return True

    if win.Class() == "MenuItem" and win.State() != "2048":
        return True

    # Most controls don't - so just return false if
    # neither of the above condition hold
    return False

RepeatedHotkeyTest.TestsMenus = True

<|endoftext|>


<commit_before>"""Tests for mouse.py"""

import time
import copy
import sys
import os
import unittest
if sys.platform == 'win32':
    import win32api
    import win32clipboard
    sys.path.append(".")
    from pywinauto.application import Application
    from pywinauto.keyboard import send_keys
    from pywinauto import mouse
    from pywinauto.timings import Timings
else:
    import subprocess
    from Xlib.display import Display
    parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    sys.path.append(parent_dir)
    import mouse
    send_keys_dir = os.path.join(parent_dir, r"linux")
    sys.path.insert(0, send_keys_dir)
    from keyboard import send_keys
    import clipboard

def _test_app():
    test_folder = os.path.join(os.path.dirname
                               (os.path.dirname
                                (os.path.dirname
                                 (os.path.abspath(__file__)))),
                               r"apps/MouseTester")
    if sys.platform == 'win32':
        return os.path.join(test_folder, "mousebuttons.exe")
    else:
        return os.path.join(test_folder, "mousebuttons")

class MouseTests(unittest.TestCase):

    def setUp(self):
        """Set some data and ensure the application is in the state we want"""
        if sys.platform == 'win32':
            Timings.defaults()
            self.app = Application()
            self.app.start(_test_app())
            self.dlg = self.app.mousebuttons
        else:
            self.display = Display()
            self.app = subprocess.Popen("exec " + _test_app(), shell=True)
            time.sleep(1)

    def tearDown(self):
        self.app.kill()

    def __get_pos(self, shift):
        if sys.platform == 'win32':
            rect = self.dlg.rectangle()
            center = rect.mid_point()
            return center.x + shift, center.y + shift
        else:
            root = self.display.screen().root
            left_pos = root.get_geometry().width / 2
            top_pos = root.get_geometry().height / 2
            return left_pos - shift, top_pos - shift

    def __get_text(self):
        data = ''
        time.sleep(1)
        send_keys('^a^c', pause=0.2)
        if sys.platform == 'win32':
            win32clipboard.OpenClipboard()
            data = win32clipboard.GetClipboardData()
            win32clipboard.CloseClipboard()
        else:
            data = clipboard.get_data()
        return data

    def test_left_click(self):
        left, top = self.__get_pos(50)
        mouse.click(coords=(left, top))
        print(left, top)
        data = self.__get_text()
        print(data)
        self.assertTrue(str(int(top)) in data)
        self.assertTrue(str(int(left)) in data)
        self.assertTrue("LeftButton" in data)
        self.assertTrue("Mouse Press" in data)
        self.assertTrue("Mouse Release" in data)

    def test_double_click(self):
        mouse.double_click(coords=(self.__get_pos(50)))
        data = self.__get_text()
        self.assertTrue("Mouse DoubleClick" in data)

    def test_press_release(self):
        left, top = self.__get_pos(50)
        left1, top1 = self.__get_pos(20)
        mouse.press(coords=(left, top))
        mouse.release(coords=(left1, top1))
        data = self.__get_text()
        self.assertEqual(str(top) in data, str(top1) in data)
        self.assertEqual(str(left) in data, str(left1) in data)

    def test_right_click(self):
        mouse.right_click((self.__get_pos(50)))
        data = self.__get_text()
        self.assertTrue("Mouse Press" in data)
        self.assertTrue("Mouse Release" in data)
        self.assertTrue("RightButton" in data)

    def test_vertical_scroll_up(self):
        mouse.click(coords=(self.__get_pos(50)))
        mouse.scroll(self.__get_pos(50), 1)
        data = self.__get_text()
        self.assertTrue("UP" in data)

    def test_vertical_scroll_down(self):
        mouse.click(coords=(self.__get_pos(50)))
        mouse.scroll(self.__get_pos(50), -1)
        data = self.__get_text()
        self.assertTrue("DOWN" in data)

    def test_wheel_click(self):
        mouse.wheel_click((self.__get_pos(50)))
        data = self.__get_text()
        self.assertTrue("Mouse Press" in data)
        self.assertTrue("Mouse Release" in data)
        self.assertTrue("MiddleButton" in data)

    if sys.platform == "win32":
        def test_mouse_can_move_cursor1(self):
            coord = (20, 10)
            mouse.move(coord)
            self.assertEqual(coord, win32api.GetCursorPos())

        def test_mouse_can_move_cursor2(self):
            coord = (0, 1)
            mouse.move(coord)
            self.assertEqual(coord, win32api.GetCursorPos())

        def test_mouse_can_move_cursor2(self):
            coord = (0, 1)
            mouse.move(coord)
            self.assertEqual(coord, win32api.GetCursorPos())

        def test_mouse_tween1(self):
            coord = (0, 0)
            mouse.move(coord, duration=0.5)
            self.assertEqual(coord, win32api.GetCursorPos())

        def test_mouse_tween2(self):
            coord = (0, 0)
            mouse.move(coord)
            mouse.move(coord, duration=0.5)
            self.assertEqual(coord, win32api.GetCursorPos())

    if sys.platform != 'win32':
        def test_swapped_buttons(self):
            current_map = self.display.get_pointer_mapping()
            swapped_map = copy.copy(current_map)
            swapped_map[0], swapped_map[2] = swapped_map[2], swapped_map[0]
            self.display.set_pointer_mapping(swapped_map)
            try:
                mouse.right_click((self.__get_pos(50)))
                data = self.__get_text()
                self.assertTrue("RightButton" in data)
            finally:
                self.display.set_pointer_mapping(current_map)

if __name__ == "__main__":
    unittest.main()
<commit_msg>fix test_mouse_can_move_cursor3<commit_after>"""Tests for mouse.py"""

import time
import copy
import sys
import os
import unittest
if sys.platform == 'win32':
    import win32api
    import win32clipboard
    sys.path.append(".")
    from pywinauto.application import Application
    from pywinauto.keyboard import send_keys
    from pywinauto import mouse
    from pywinauto.timings import Timings
else:
    import subprocess
    from Xlib.display import Display
    parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    sys.path.append(parent_dir)
    import mouse
    send_keys_dir = os.path.join(parent_dir, r"linux")
    sys.path.insert(0, send_keys_dir)
    from keyboard import send_keys
    import clipboard

def _test_app():
    test_folder = os.path.join(os.path.dirname
                               (os.path.dirname
                                (os.path.dirname
                                 (os.path.abspath(__file__)))),
                               r"apps/MouseTester")
    if sys.platform == 'win32':
        return os.path.join(test_folder, "mousebuttons.exe")
    else:
        return os.path.join(test_folder, "mousebuttons")

class MouseTests(unittest.TestCase):

    def setUp(self):
        """Set some data and ensure the application is in the state we want"""
        if sys.platform == 'win32':
            Timings.defaults()
            self.app = Application()
            self.app.start(_test_app())
            self.dlg = self.app.mousebuttons
        else:
            self.display = Display()
            self.app = subprocess.Popen("exec " + _test_app(), shell=True)
            time.sleep(1)

    def tearDown(self):
        self.app.kill()

    def __get_pos(self, shift):
        if sys.platform == 'win32':
            rect = self.dlg.rectangle()
            center = rect.mid_point()
            return center.x + shift, center.y + shift
        else:
            root = self.display.screen().root
            left_pos = root.get_geometry().width / 2
            top_pos = root.get_geometry().height / 2
            return left_pos - shift, top_pos - shift

    def __get_text(self):
        data = ''
        time.sleep(1)
        send_keys('^a^c', pause=0.2)
        if sys.platform == 'win32':
            win32clipboard.OpenClipboard()
            data = win32clipboard.GetClipboardData()
            win32clipboard.CloseClipboard()
        else:
            data = clipboard.get_data()
        return data

    def test_left_click(self):
        left, top = self.__get_pos(50)
        mouse.click(coords=(left, top))
        print(left, top)
        data = self.__get_text()
        print(data)
        self.assertTrue(str(int(top)) in data)
        self.assertTrue(str(int(left)) in data)
        self.assertTrue("LeftButton" in data)
        self.assertTrue("Mouse Press" in data)
        self.assertTrue("Mouse Release" in data)

    def test_double_click(self):
        mouse.double_click(coords=(self.__get_pos(50)))
        data = self.__get_text()
        self.assertTrue("Mouse DoubleClick" in data)

    def test_press_release(self):
        left, top = self.__get_pos(50)
        left1, top1 = self.__get_pos(20)
        mouse.press(coords=(left, top))
        mouse.release(coords=(left1, top1))
        data = self.__get_text()
        self.assertEqual(str(top) in data, str(top1) in data)
        self.assertEqual(str(left) in data, str(left1) in data)

    def test_right_click(self):
        mouse.right_click((self.__get_pos(50)))
        data = self.__get_text()
        self.assertTrue("Mouse Press" in data)
        self.assertTrue("Mouse Release" in data)
        self.assertTrue("RightButton" in data)

    def test_vertical_scroll_up(self):
        mouse.click(coords=(self.__get_pos(50)))
        mouse.scroll(self.__get_pos(50), 1)
        data = self.__get_text()
        self.assertTrue("UP" in data)

    def test_vertical_scroll_down(self):
        mouse.click(coords=(self.__get_pos(50)))
        mouse.scroll(self.__get_pos(50), -1)
        data = self.__get_text()
        self.assertTrue("DOWN" in data)

    def test_wheel_click(self):
        mouse.wheel_click((self.__get_pos(50)))
        data = self.__get_text()
        self.assertTrue("Mouse Press" in data)
        self.assertTrue("Mouse Release" in data)
        self.assertTrue("MiddleButton" in data)

    if sys.platform == "win32":
        def test_mouse_can_move_cursor1(self):
            coord = (20, 10)
            mouse.move(coord)
            self.assertEqual(coord, win32api.GetCursorPos())

        def test_mouse_can_move_cursor2(self):
            coord = (0, 1)
            mouse.move(coord)
            self.assertEqual(coord, win32api.GetCursorPos())

        def test_mouse_can_move_cursor3(self):
            coord = (150, 150)
            mouse.move(coord)
            self.assertEqual(coord, win32api.GetCursorPos())

        def test_mouse_tween1(self):
            coord = (0, 0)
            mouse.move(coord, duration=0.5)
            self.assertEqual(coord, win32api.GetCursorPos())

        def test_mouse_tween2(self):
            coord = (0, 0)
            mouse.move(coord)
            mouse.move(coord, duration=0.5)
            self.assertEqual(coord, win32api.GetCursorPos())

    if sys.platform != 'win32':
        def test_swapped_buttons(self):
            current_map = self.display.get_pointer_mapping()
            swapped_map = copy.copy(current_map)
            swapped_map[0], swapped_map[2] = swapped_map[2], swapped_map[0]
            self.display.set_pointer_mapping(swapped_map)
            try:
                mouse.right_click((self.__get_pos(50)))
                data = self.__get_text()
                self.assertTrue("RightButton" in data)
            finally:
                self.display.set_pointer_mapping(current_map)

if __name__ == "__main__":
    unittest.main()
<|endoftext|>


<commit_before>"""
This is a DataSet class that will, given a root directory containing images
organised into folders with their class names, provide an interface to
pylearn2 to load the images at start time. Then, at runtime it will allow
custom preprocessing functions to be run for augmenting the data or
otherwise.

Developed as part of our entry to the National Data Science Bowl plankton
classification challenge.
"""
__authors__ = "Gavin Gray"
__copyright__ = "Copyright 2015 - University of Edinburgh"
__credits__ = ["Gavin Gray"]
__license__ = "3-clause BSD"
__maintainer__ = "Gavin Gray"
__email__ = "gavingray1729@gmail.com"

import numpy as np

import pylearn2.datasets.dataset
import neukrill_net.utils

# don't have to think too hard about how to write this:
# https://stackoverflow.com/questions/19151/build-a-basic-python-iterator
class FlyIterator(object):
    """
    Simple iterator class to take a dataset and iterate over
    it's contents applying a processing function. Assuming
    the dataset has a processing function to apply.

    It may have an issue of there being some leftover examples
    that will never be shown on any epoch. Can avoid this by
    seeding with sampled numbers from the dataset's own rng.
    """
    def __init__(self, dataset, batch_size, num_batches,
                 final_shape, seed=42):
        self.dataset = dataset
        self.batch_size = batch_size
        self.num_batches = num_batches
        self.final_shape = final_shape
        # initialise rng
        self.rng = np.random.RandomState(seed=seed)
        # shuffle indices of size equal to number of examples
        # in dataset
        N = self.dataset.get_num_examples()
        self.indices = range(N)
        self.rng.shuffle(self.indices)
        # have to add this for checks during training
        # bit of a lie
        self.stochastic = False
        # this is also required
        self.num_examples = batch_size*num_batches

    def __iter__(self):
        return self

    def next(self):
        # return one batch
        if len(self.indices) >= self.batch_size:
            batch_indices = [self.indices.pop() for i in range(self.batch_size)]
            # preallocate array
            if len(self.final_shape) == 2:
                Xbatch = np.zeros([self.batch_size]+list(self.final_shape)+[1])
            elif len(self.final_shape) == 3:
                Xbatch = np.zeros([self.batch_size]+list(self.final_shape))
            # iterate over indices, applying the dataset's processing function
            for i,j in enumerate(batch_indices):
                Xbatch[i] = self.dataset.fn(self.dataset.X[j]).reshape(Xbatch.shape[1:])
            # get the batch for y as well
            ybatch = self.dataset.y[batch_indices,:].astype(np.float32)
            Xbatch = Xbatch.astype(np.float32)
            return Xbatch,ybatch
        else:
            raise StopIteration

class ListDataset(pylearn2.datasets.dataset.Dataset):
    """
    Loads images as raw numpy arrays in a list, tries
    its best to respect the interface expected of a
    Pylearn2 Dataset.
    """
    def __init__(self, transformer, settings_path="settings.json",
                 run_settings_path="run_settings/alexnet_based.json",
                 training_set_mode="train",
                 verbose=False, force=False):
        """
        Loads the images as a list of differently shaped
        numpy arrays and loads the labels as a vector of
        integers, mapped deterministically.
        """
        self.fn = transformer
        # load settings
        self.settings = neukrill_net.utils.Settings(settings_path)
        self.run_settings = neukrill_net.utils.load_run_settings(run_settings_path,
                                                                 self.settings,
                                                                 force=force)
        # split train/test/validation
        self.settings.image_fnames["train"] = \
                neukrill_net.utils.train_test_split(
                        self.settings.image_fnames,
                        training_set_mode,
                        train_split=self.run_settings["train_split"])
        #load the data
        self.X, labels = neukrill_net.utils.load_rawdata(
                        self.settings.image_fnames,
                        classes=self.settings.classes,
                        verbose=verbose)
        self.N = len(self.X)
        # count the classes
        self.n_classes = len(self.settings.classes)
        # transform labels from strings to integers
        self.y = np.zeros((self.N,self.n_classes))
        class_dictionary = {}
        for i,c in enumerate(self.settings.classes):
            class_dictionary[c] = i
        for i,j in enumerate(map(lambda c: class_dictionary[c],labels)):
            self.y[i,j] = 1
        self.y = self.y.astype(np.float32)

        # set up the random state
        self.rng = np.random.RandomState(self.settings.r_seed)

        # shuffle a list of image indices
        self.indices = range(self.N)
        self.rng.shuffle(self.indices)

    def iterator(self, mode=None, batch_size=None, num_batches=None, rng=None,
                        data_specs=None, return_tuple=False):
        """
        Returns iterator object with standard Pythonic interface; iterates
        over the dataset over batches, popping off batches from a shuffled
        list of indices.
        Inputs:
            - mode: not actually used, just there so that Pylearn2 doesn't
        complain.
            - batch_size: required, size of the minibatches produced.
            - num_batches: supply if you want, the dataset will make as many
        as it can if you don't.
            - rng: not used, as above.
            - data_specs: not used, as above
            - return_tuple: not used, as above
        Outputs:
            - instance of FlyIterator, see above.
        """
        if not num_batches:
            # guess that we want to use all of them
            num_batches = int(len(self.X)/batch_size)
        iterator = FlyIterator(dataset=self, batch_size=batch_size,
                                num_batches=num_batches,
                                final_shape=self.run_settings["final_shape"],
                                seed=self.rng.random_integers(low=0, high=256))
        return iterator

    def adjust_to_be_viewed_with():
        raise NotImplementedError("Didn't think this was important, so didn't write it.")

    def get_batch_design(self, batch_size, include_labels=False):
        """
        Will return a list of the size batch_size of carefully raveled arrays.
        Optionally, will also include labels (using include_labels).
        """
        selection = self.rng.random_integers(0,high=self.N,size=batch_size)
        batch = [self.X[s].ravel() for s in selection]
        if include_labels:
            raise NotImplementedError
        return batch

    def get_batch_topo(self, batch_size, include_labels=False):
        """
        Will return a list of the size batch_size of raw, unfiltered, artisan
        numpy arrays. Optionally, will also include labels (using include_labels).

        Strongly discouraged to use this method for learning code, so I guess
        this isn't so important?
        """
        selection = self.rng.random_integers(0,high=self.N,size=batch_size)
        batch = [self.X[s] for s in selection]
        return batch

    def get_num_examples(self):
        return self.N

    def get_topological_view():
        raise NotImplementedError("Not written yet, not sure we need it")

    def get_weights_view():
        raise NotImplementedError("Not written yet, didn't think it was important")

    def has_targets(self):
        if self.y:
            return True
        else:
            return False
<commit_msg>argument checks<commit_after>"""
This is a DataSet class that will, given a root directory containing images
organised into folders with their class names, provide an interface to
pylearn2 to load the images at start time. Then, at runtime it will allow
custom preprocessing functions to be run for augmenting the data or
otherwise.

Developed as part of our entry to the National Data Science Bowl plankton
classification challenge.
"""
__authors__ = "Gavin Gray"
__copyright__ = "Copyright 2015 - University of Edinburgh"
__credits__ = ["Gavin Gray"]
__license__ = "3-clause BSD"
__maintainer__ = "Gavin Gray"
__email__ = "gavingray1729@gmail.com"

import numpy as np

import pylearn2.datasets.dataset
import neukrill_net.utils

# don't have to think too hard about how to write this:
# https://stackoverflow.com/questions/19151/build-a-basic-python-iterator
class FlyIterator(object):
    """
    Simple iterator class to take a dataset and iterate over
    it's contents applying a processing function. Assuming
    the dataset has a processing function to apply.

    It may have an issue of there being some leftover examples
    that will never be shown on any epoch. Can avoid this by
    seeding with sampled numbers from the dataset's own rng.
    """
    def __init__(self, dataset, batch_size, num_batches,
                 final_shape, seed=42):
        self.dataset = dataset
        self.batch_size = batch_size
        self.num_batches = num_batches
        self.final_shape = final_shape
        # initialise rng
        self.rng = np.random.RandomState(seed=seed)
        # shuffle indices of size equal to number of examples
        # in dataset
        N = self.dataset.get_num_examples()
        self.indices = range(N)
        self.rng.shuffle(self.indices)
        # have to add this for checks during training
        # bit of a lie
        self.stochastic = False
        # this is also required
        self.num_examples = batch_size*num_batches

    def __iter__(self):
        return self

    def next(self):
        # return one batch
        if len(self.indices) >= self.batch_size:
            batch_indices = [self.indices.pop() for i in range(self.batch_size)]
            # preallocate array
            if len(self.final_shape) == 2:
                Xbatch = np.zeros([self.batch_size]+list(self.final_shape)+[1])
            elif len(self.final_shape) == 3:
                Xbatch = np.zeros([self.batch_size]+list(self.final_shape))
            # iterate over indices, applying the dataset's processing function
            for i,j in enumerate(batch_indices):
                Xbatch[i] = self.dataset.fn(self.dataset.X[j]).reshape(Xbatch.shape[1:])
            # get the batch for y as well
            ybatch = self.dataset.y[batch_indices,:].astype(np.float32)
            Xbatch = Xbatch.astype(np.float32)
            return Xbatch,ybatch
        else:
            raise StopIteration

class ListDataset(pylearn2.datasets.dataset.Dataset):
    """
    Loads images as raw numpy arrays in a list, tries
    its best to respect the interface expected of a
    Pylearn2 Dataset.
    """
    def __init__(self, transformer, settings_path="settings.json",
                 run_settings_path="run_settings/alexnet_based.json",
                 training_set_mode="train",
                 verbose=False, force=False):
        """
        Loads the images as a list of differently shaped
        numpy arrays and loads the labels as a vector of
        integers, mapped deterministically.
        """
        self.fn = transformer
        # load settings
        self.settings = neukrill_net.utils.Settings(settings_path)
        self.run_settings = neukrill_net.utils.load_run_settings(run_settings_path,
                                                                 self.settings,
                                                                 force=force)
        # split train/test/validation
        self.settings.image_fnames["train"] = \
                neukrill_net.utils.train_test_split(
                        self.settings.image_fnames,
                        training_set_mode,
                        train_split=self.run_settings["train_split"])
        #load the data
        self.X, labels = neukrill_net.utils.load_rawdata(
                        self.settings.image_fnames,
                        classes=self.settings.classes,
                        verbose=verbose)
        self.N = len(self.X)
        # count the classes
        self.n_classes = len(self.settings.classes)
        # transform labels from strings to integers
        self.y = np.zeros((self.N,self.n_classes))
        class_dictionary = {}
        for i,c in enumerate(self.settings.classes):
            class_dictionary[c] = i
        for i,j in enumerate(map(lambda c: class_dictionary[c],labels)):
            self.y[i,j] = 1
        self.y = self.y.astype(np.float32)

        # set up the random state
        self.rng = np.random.RandomState(self.settings.r_seed)

        # shuffle a list of image indices
        self.indices = range(self.N)
        self.rng.shuffle(self.indices)

    def iterator(self, mode=None, batch_size=None, num_batches=None, rng=None,
                        data_specs=None, return_tuple=False):
        """
        Returns iterator object with standard Pythonic interface; iterates
        over the dataset over batches, popping off batches from a shuffled
        list of indices.
        Inputs:
            - mode: not actually used, just there so that Pylearn2 doesn't
        complain.
            - batch_size: required, size of the minibatches produced.
            - num_batches: supply if you want, the dataset will make as many
        as it can if you don't.
            - rng: not used, as above.
            - data_specs: not used, as above
            - return_tuple: not used, as above
        Outputs:
            - instance of FlyIterator, see above.
        """
        if not num_batches:
            # guess that we want to use all of them
            num_batches = int(len(self.X)/batch_size)
        iterator = FlyIterator(dataset=self, batch_size=batch_size,
                                num_batches=num_batches,
                                final_shape=self.run_settings["final_shape"],
                                seed=self.rng.random_integers(low=0, high=256))
        return iterator

    def adjust_to_be_viewed_with():
        raise NotImplementedError("Didn't think this was important, so didn't write it.")

    def get_batch_design(self, batch_size, include_labels=False):
        """
        Will return a list of the size batch_size of carefully raveled arrays.
        Optionally, will also include labels (using include_labels).
        """
        selection = self.rng.random_integers(0,high=self.N,size=batch_size)
        batch = [self.X[s].ravel() for s in selection]
        if include_labels:
            raise NotImplementedError
        return batch

    def get_batch_topo(self, batch_size, include_labels=False):
        """
        Will return a list of the size batch_size of raw, unfiltered, artisan
        numpy arrays. Optionally, will also include labels (using include_labels).

        Strongly discouraged to use this method for learning code, so I guess
        this isn't so important?
        """
        selection = self.rng.random_integers(0,high=self.N,size=batch_size)
        batch = [self.X[s] for s in selection]
        if include_labels:
            raise NotImplementedError
        return batch

    def get_num_examples(self):
        return self.N

    def get_topological_view():
        raise NotImplementedError("Not written yet, not sure we need it")

    def get_weights_view():
        raise NotImplementedError("Not written yet, didn't think it was important")

    def has_targets(self):
        if self.y:
            return True
        else:
            return False
<|endoftext|>


<commit_before>"""Abstract class for autoregressive decoding.

Either for the recurrent decoder, or for the transformer decoder.

The autoregressive decoder uses the while loop to get the outputs.
Descendants should only specify the initial state and the while loop body.
"""
from typing import (
    NamedTuple, Callable, Tuple, cast, Type, List, Optional, Any)

import numpy as np
import tensorflow as tf

from neuralmonkey.dataset import Dataset
from neuralmonkey.decorators import tensor
from neuralmonkey.model.model_part import ModelPart, FeedDict, InitializerSpecs
from neuralmonkey.logging import log, warn
from neuralmonkey.model.sequence import EmbeddedSequence
from neuralmonkey.nn.utils import dropout
from neuralmonkey.tf_utils import get_variable, get_state_shape_invariants
from neuralmonkey.vocabulary import Vocabulary, START_TOKEN, UNK_TOKEN_INDEX

def extend_namedtuple(name: str, parent: Type,
                      fields: List[Tuple[str, Type]]) -> Type:
    """Extend a named tuple to contain more elements."""
    # pylint: disable=protected-access
    ext_fields = [(k, parent._field_types[k]) for k in parent._fields] + fields
    # pylint: enable=protected-access
    return cast(Type, NamedTuple(name, ext_fields))

LoopState = NamedTuple(
    "LoopState",
    [("histories", Any),
     ("constants", Any),
     ("feedables", Any)])
# pylint: enable=invalid-name

# The LoopState is a structure that works with the tf.while_loop function
# the decoder loop state stores all the information that is not invariant
# for the decoder run.
# pylint: disable=invalid-name
DecoderHistories = NamedTuple(
    "DecoderHistories",
    [("logits", tf.Tensor),
     ("decoder_outputs", tf.Tensor),
     ("outputs", tf.Tensor),
     ("mask", tf.Tensor)])  # float matrix, 0s and 1s

DecoderConstants = NamedTuple(
    "DecoderConstants",
    [("train_inputs", Optional[tf.Tensor])])

DecoderFeedables = NamedTuple(
    "DecoderFeedables",
    [("step", tf.Tensor),  # 1D int, number of the step
     ("finished", tf.Tensor),  # batch-sized, bool
     ("input_symbol", tf.Tensor),
     ("prev_logits", tf.Tensor)])

# pylint: disable=too-many-public-methods,too-many-instance-attributes
class AutoregressiveDecoder(ModelPart):

    # pylint: disable=too-many-arguments,too-many-instance-attributes
    def __init__(self,
                 name: str,
                 vocabulary: Vocabulary,
                 data_id: str,
                 max_output_len: int,
                 dropout_keep_prob: float = 1.0,
                 embedding_size: int = None,
                 embeddings_source: EmbeddedSequence = None,
                 tie_embeddings: bool = False,
                 label_smoothing: float = None,
                 supress_unk: bool = False,
                 save_checkpoint: str = None,
                 load_checkpoint: str = None,
                 initializers: InitializerSpecs = None) -> None:
        """Initialize parameters common for all autoregressive decoders.

        Arguments:
            name: Name of the decoder. Should be unique accross all Neural
                Monkey objects.
            vocabulary: Target vocabulary.
            data_id: Target data series.
            max_output_len: Maximum length of an output sequence.
            dropout_keep_prob: Probability of keeping a value during dropout.
            embedding_size: Size of embedding vectors for target words.
            embeddings_source: Embedded sequence to take embeddings from.
            tie_embeddings: Use decoder.embedding_matrix also in place
                of the output decoding matrix.
            label_smoothing: Label smoothing parameter.
            supress_unk: If true, decoder will not produce symbols for unknown
                tokens.
        """
        ModelPart.__init__(self, name, save_checkpoint, load_checkpoint,
                           initializers)

        log("Initializing decoder, name: '{}'".format(name))

        self.vocabulary = vocabulary
        self.data_id = data_id
        self.max_output_len = max_output_len
        self.dropout_keep_prob = dropout_keep_prob
        self.embedding_size = embedding_size
        self.embeddings_source = embeddings_source
        self.label_smoothing = label_smoothing
        self.tie_embeddings = tie_embeddings
        self.supress_unk = supress_unk

        self.encoder_states = None
        self.encoder_mask = None

        # check the values of the parameters (max_output_len, ...)
        if max_output_len <= 0:
            raise ValueError("Maximum sequence length must be "
                             "a positive integer.")

        if dropout_keep_prob < 0.0 or dropout_keep_prob > 1.0:
            raise ValueError("Dropout keep probability must be"
                             "a real number in the interval [0,1].")

        if self.embedding_size is None and self.embeddings_source is None:
            raise ValueError("You must specify either embedding size or the "
                             "embedded sequence from which to reuse the "
                             "embeddings (e.g. set either 'embedding_size' or "
                             " 'embeddings_source' parameter)")

        if self.embeddings_source is not None:
            if self.embedding_size is not None:
                warn("Overriding the embedding_size parameter with the"
                     " size of the reused embeddings from the encoder.")

            self.embedding_size = (
                self.embeddings_source.embedding_matrix.get_shape()[1].value)

        with self.use_scope():
            self.go_symbols = tf.placeholder(tf.int32, [None], "go_symbols")

            self.train_inputs = tf.placeholder(
                tf.int32, [None, None], "train_inputs")
            self.train_mask = tf.placeholder(
                tf.float32, [None, None], "train_mask")
    # pylint: enable=too-many-arguments,too-many-instance-attributes

    @tensor
    def decoding_w(self) -> tf.Variable:
        if (self.tie_embeddings
                and self.embedding_size != self.output_dimension):
            raise ValueError(
                "`embedding_size must be equal to the output_projection "
                "size when using the `tie_embeddings` option")

        with tf.name_scope("output_projection"):
            if self.tie_embeddings:
                return tf.transpose(self.embedding_matrix)

            return get_variable(
                "state_to_word_W",
                [self.output_dimension, len(self.vocabulary)],
                initializer=tf.glorot_uniform_initializer())

    @tensor
    def decoding_b(self) -> Optional[tf.Variable]:
        if self.tie_embeddings:
            return tf.zeros(len(self.vocabulary))

        with tf.name_scope("output_projection"):
            return get_variable(
                "state_to_word_b",
                [len(self.vocabulary)],
                initializer=tf.zeros_initializer())

    @tensor
    def embedding_matrix(self) -> tf.Variable:
        """Variables and operations for embedding of input words.

        If we are reusing word embeddings, this function takes the embedding
        matrix from the first encoder
        """
        if self.embeddings_source is not None:
            return self.embeddings_source.embedding_matrix

        assert self.embedding_size is not None

        return get_variable(
            name="word_embeddings",
            shape=[len(self.vocabulary), self.embedding_size],
            initializer=tf.glorot_uniform_initializer())

    def get_logits(self, state: tf.Tensor) -> tf.Tensor:
        """Project the decoder's output layer to logits over the vocabulary."""
        state = dropout(state, self.dropout_keep_prob, self.train_mode)
        logits = tf.matmul(state, self.decoding_w) + self.decoding_b

        if self.supress_unk:
            unk_mask = tf.one_hot(
                UNK_TOKEN_INDEX, depth=len(self.vocabulary), on_value=-1e-9)
            logits += unk_mask

        return logits

    @tensor
    def train_loop_result(self) -> Tuple[tf.Tensor, tf.Tensor,
                                         tf.Tensor, tf.Tensor]:
        return self.decoding_loop(train_mode=True)

    @tensor
    def train_logits(self) -> tf.Tensor:
        # THE LAST TRAIN INPUT IS NOT USED IN DECODING FUNCTION
        # (just as a target)
        return tuple(self.train_loop_result)[0]

    @tensor
    def train_output_states(self) -> tf.Tensor:
        return tuple(self.train_loop_result)[1]

    @tensor
    def train_logprobs(self) -> tf.Tensor:
        return tf.nn.log_softmax(self.train_logits)

    @tensor
    def train_xents(self) -> tf.Tensor:
        train_targets = tf.transpose(self.train_inputs)
        softmax_function = None
        if self.label_smoothing:
            softmax_function = (
                lambda labels, logits: tf.losses.softmax_cross_entropy(
                    tf.one_hot(labels, len(self.vocabulary)),
                    logits, label_smoothing=self.label_smoothing))

        return tf.contrib.seq2seq.sequence_loss(
            tf.transpose(self.train_logits, perm=[1, 0, 2]),
            train_targets,
            tf.transpose(self.train_mask),
            average_across_batch=False,
            softmax_loss_function=softmax_function)

    @tensor
    def train_loss(self) -> tf.Tensor:
        return tf.reduce_mean(self.train_xents)

    @property
    def cost(self) -> tf.Tensor:
        return self.train_loss

    @tensor
    def runtime_loop_result(self) -> Tuple[tf.Tensor, tf.Tensor,
                                           tf.Tensor, tf.Tensor]:
        return self.decoding_loop(train_mode=False)

    @tensor
    def runtime_logits(self) -> tf.Tensor:
        return tuple(self.runtime_loop_result)[0]

    @tensor
    def runtime_output_states(self) -> tf.Tensor:
        return tuple(self.runtime_loop_result)[1]

    @tensor
    def runtime_mask(self) -> tf.Tensor:
        return tuple(self.runtime_loop_result)[2]

    @tensor
    def decoded(self) -> tf.Tensor:
        # We disable generating of <pad> tokens at index 0
        # (self.runtime_logits[:, :, 1:]). This shifts the indices
        # of the decoded tokens (therefore, we add +1 to the decoded
        # output indices).

        # self.runtime_logits is of size [batch, sentence_len, vocabulary_size]
        return tf.argmax(self.runtime_logits[:, :, 1:], -1) + 1

    @tensor
    def runtime_xents(self) -> tf.Tensor:
        train_targets = tf.transpose(self.train_inputs)
        batch_major_logits = tf.transpose(self.runtime_logits, [1, 0, 2])
        min_time = tf.minimum(tf.shape(train_targets)[1],
                              tf.shape(batch_major_logits)[1])

        # NOTE if done properly, there should be padding of the shorter
        # sequence instead of cropping to the length of the shorter one

        return tf.contrib.seq2seq.sequence_loss(
            logits=batch_major_logits[:, :min_time],
            targets=train_targets[:, :min_time],
            weights=tf.transpose(self.train_mask)[:, :min_time],
            average_across_batch=False)

    @tensor
    def runtime_loss(self) -> tf.Tensor:
        return tf.reduce_mean(self.runtime_xents)

    @tensor
    def runtime_logprobs(self) -> tf.Tensor:
        return tf.nn.log_softmax(self.runtime_logits)

    @property
    def output_dimension(self) -> int:
        raise NotImplementedError("Abstract property")

    def get_initial_loop_state(self) -> LoopState:

        dec_output = tf.zeros(
            shape=[0, self.batch_size, self.embedding_size],
            dtype=tf.float32,
            name="hist_decoder_outputs")

        logit = tf.zeros(
            shape=[0, self.batch_size, len(self.vocabulary)],
            dtype=tf.float32,
            name="hist_logits")

        mask = tf.zeros(
            shape=[0, self.batch_size],
            dtype=tf.bool,
            name="mask")

        outputs = tf.zeros(
            shape=[0, self.batch_size],
            dtype=tf.int32,
            name="outputs")

        feedables = DecoderFeedables(
            step=tf.constant(0, tf.int32),
            finished=tf.zeros([self.batch_size], dtype=tf.bool),
            input_symbol=self.go_symbols,
            prev_logits=tf.zeros([self.batch_size, len(self.vocabulary)]))

        histories = DecoderHistories(
            logits=logit,
            decoder_outputs=dec_output,
            mask=mask,
            outputs=outputs)

        constants = DecoderConstants(train_inputs=self.train_inputs)

        return LoopState(
            histories=histories,
            constants=constants,
            feedables=feedables)

    def loop_continue_criterion(self, *args) -> tf.Tensor:
        """Decide whether to break out of the while loop.

        Arguments:
            loop_state: ``LoopState`` instance (see the docs for this module).
                Represents current decoder loop state.
        """
        loop_state = LoopState(*args)
        finished = loop_state.feedables.finished
        not_all_done = tf.logical_not(tf.reduce_all(finished))
        before_max_len = tf.less(loop_state.feedables.step,
                                 self.max_output_len)
        return tf.logical_and(not_all_done, before_max_len)

    def get_body(self, train_mode: bool, sample: bool = False,
                 temperature: float = 1) -> Callable:
        """Return the while loop body function."""
        raise NotImplementedError("Abstract method")

    def finalize_loop(self, final_loop_state: LoopState,
                      train_mode: bool) -> None:
        """Execute post-while loop operations.

        Arguments:
            final_loop_state: Decoder loop state at the end
                of the decoding loop.
            train_mode: Boolean flag, telling whether this is
                a training run.
        """

    def decoding_loop(self, train_mode: bool, sample: bool = False,
                      temperature: float = 1) -> Tuple[tf.Tensor, tf.Tensor,
                                                       tf.Tensor, tf.Tensor]:
        """Run the decoding while loop.

        Calls get_initial_loop_state and constructs tf.while_loop
        with the continuation criterion returned from loop_continue_criterion,
        and body function returned from get_body.

        After finishing the tf.while_loop, it calls finalize_loop
        to further postprocess the final decoder loop state (usually
        by stacking Tensors containing decoding histories).

        Arguments:
            train_mode: Boolean flag, telling whether this is
                a training run.
            sample: Boolean flag, telling whether we should sample
                the output symbols from the output distribution instead
                of using argmax or gold data.
            temperature: float value specifying the softmax temperature
        """
        initial_loop_state = self.get_initial_loop_state()
        final_loop_state = tf.while_loop(
            self.loop_continue_criterion,
            self.get_body(train_mode, sample, temperature),
            initial_loop_state,
            shape_invariants=tf.contrib.framework.nest.map_structure(
                get_state_shape_invariants, initial_loop_state))

        self.finalize_loop(final_loop_state, train_mode)

        logits = final_loop_state.histories.logits
        decoder_outputs = final_loop_state.histories.decoder_outputs
        decoded = final_loop_state.histories.outputs

        # TODO mask should include also the end symbol
        mask = final_loop_state.histories.mask

        return logits, decoder_outputs, mask, decoded

    def feed_dict(self, dataset: Dataset, train: bool = False) -> FeedDict:
        """Populate the feed dictionary for the decoder object.

        Arguments:
            dataset: The dataset to use for the decoder.
            train: Boolean flag, telling whether this is a training run.
        """
        fd = ModelPart.feed_dict(self, dataset, train)

        sentences = dataset.maybe_get_series(self.data_id)

        if sentences is None and train:
            raise ValueError("When training, you must feed "
                             "reference sentences")

        go_symbol_idx = self.vocabulary.get_word_index(START_TOKEN)
        fd[self.go_symbols] = np.full([len(dataset)], go_symbol_idx,
                                      dtype=np.int32)

        if sentences is not None:
            sentences_list = list(sentences)
            # train_mode=False, since we don't want to <unk>ize target words!
            inputs, weights = self.vocabulary.sentences_to_tensor(
                sentences_list, self.max_output_len, train_mode=False,
                add_start_symbol=False, add_end_symbol=True,
                pad_to_max_len=False)

            fd[self.train_inputs] = inputs
            fd[self.train_mask] = weights

        return fd
<commit_msg>supress unk fix. closes #673<commit_after>"""Abstract class for autoregressive decoding.

Either for the recurrent decoder, or for the transformer decoder.

The autoregressive decoder uses the while loop to get the outputs.
Descendants should only specify the initial state and the while loop body.
"""
from typing import (
    NamedTuple, Callable, Tuple, cast, Type, List, Optional, Any)

import numpy as np
import tensorflow as tf

from neuralmonkey.dataset import Dataset
from neuralmonkey.decorators import tensor
from neuralmonkey.model.model_part import ModelPart, FeedDict, InitializerSpecs
from neuralmonkey.logging import log, warn
from neuralmonkey.model.sequence import EmbeddedSequence
from neuralmonkey.nn.utils import dropout
from neuralmonkey.tf_utils import get_variable, get_state_shape_invariants
from neuralmonkey.vocabulary import Vocabulary, START_TOKEN, UNK_TOKEN_INDEX

def extend_namedtuple(name: str, parent: Type,
                      fields: List[Tuple[str, Type]]) -> Type:
    """Extend a named tuple to contain more elements."""
    # pylint: disable=protected-access
    ext_fields = [(k, parent._field_types[k]) for k in parent._fields] + fields
    # pylint: enable=protected-access
    return cast(Type, NamedTuple(name, ext_fields))

LoopState = NamedTuple(
    "LoopState",
    [("histories", Any),
     ("constants", Any),
     ("feedables", Any)])
# pylint: enable=invalid-name

# The LoopState is a structure that works with the tf.while_loop function
# the decoder loop state stores all the information that is not invariant
# for the decoder run.
# pylint: disable=invalid-name
DecoderHistories = NamedTuple(
    "DecoderHistories",
    [("logits", tf.Tensor),
     ("decoder_outputs", tf.Tensor),
     ("outputs", tf.Tensor),
     ("mask", tf.Tensor)])  # float matrix, 0s and 1s

DecoderConstants = NamedTuple(
    "DecoderConstants",
    [("train_inputs", Optional[tf.Tensor])])

DecoderFeedables = NamedTuple(
    "DecoderFeedables",
    [("step", tf.Tensor),  # 1D int, number of the step
     ("finished", tf.Tensor),  # batch-sized, bool
     ("input_symbol", tf.Tensor),
     ("prev_logits", tf.Tensor)])

# pylint: disable=too-many-public-methods,too-many-instance-attributes
class AutoregressiveDecoder(ModelPart):

    # pylint: disable=too-many-arguments,too-many-instance-attributes
    def __init__(self,
                 name: str,
                 vocabulary: Vocabulary,
                 data_id: str,
                 max_output_len: int,
                 dropout_keep_prob: float = 1.0,
                 embedding_size: int = None,
                 embeddings_source: EmbeddedSequence = None,
                 tie_embeddings: bool = False,
                 label_smoothing: float = None,
                 supress_unk: bool = False,
                 save_checkpoint: str = None,
                 load_checkpoint: str = None,
                 initializers: InitializerSpecs = None) -> None:
        """Initialize parameters common for all autoregressive decoders.

        Arguments:
            name: Name of the decoder. Should be unique accross all Neural
                Monkey objects.
            vocabulary: Target vocabulary.
            data_id: Target data series.
            max_output_len: Maximum length of an output sequence.
            dropout_keep_prob: Probability of keeping a value during dropout.
            embedding_size: Size of embedding vectors for target words.
            embeddings_source: Embedded sequence to take embeddings from.
            tie_embeddings: Use decoder.embedding_matrix also in place
                of the output decoding matrix.
            label_smoothing: Label smoothing parameter.
            supress_unk: If true, decoder will not produce symbols for unknown
                tokens.
        """
        ModelPart.__init__(self, name, save_checkpoint, load_checkpoint,
                           initializers)

        log("Initializing decoder, name: '{}'".format(name))

        self.vocabulary = vocabulary
        self.data_id = data_id
        self.max_output_len = max_output_len
        self.dropout_keep_prob = dropout_keep_prob
        self.embedding_size = embedding_size
        self.embeddings_source = embeddings_source
        self.label_smoothing = label_smoothing
        self.tie_embeddings = tie_embeddings
        self.supress_unk = supress_unk

        self.encoder_states = None
        self.encoder_mask = None

        # check the values of the parameters (max_output_len, ...)
        if max_output_len <= 0:
            raise ValueError("Maximum sequence length must be "
                             "a positive integer.")

        if dropout_keep_prob < 0.0 or dropout_keep_prob > 1.0:
            raise ValueError("Dropout keep probability must be"
                             "a real number in the interval [0,1].")

        if self.embedding_size is None and self.embeddings_source is None:
            raise ValueError("You must specify either embedding size or the "
                             "embedded sequence from which to reuse the "
                             "embeddings (e.g. set either 'embedding_size' or "
                             " 'embeddings_source' parameter)")

        if self.embeddings_source is not None:
            if self.embedding_size is not None:
                warn("Overriding the embedding_size parameter with the"
                     " size of the reused embeddings from the encoder.")

            self.embedding_size = (
                self.embeddings_source.embedding_matrix.get_shape()[1].value)

        with self.use_scope():
            self.go_symbols = tf.placeholder(tf.int32, [None], "go_symbols")

            self.train_inputs = tf.placeholder(
                tf.int32, [None, None], "train_inputs")
            self.train_mask = tf.placeholder(
                tf.float32, [None, None], "train_mask")
    # pylint: enable=too-many-arguments,too-many-instance-attributes

    @tensor
    def decoding_w(self) -> tf.Variable:
        if (self.tie_embeddings
                and self.embedding_size != self.output_dimension):
            raise ValueError(
                "`embedding_size must be equal to the output_projection "
                "size when using the `tie_embeddings` option")

        with tf.name_scope("output_projection"):
            if self.tie_embeddings:
                return tf.transpose(self.embedding_matrix)

            return get_variable(
                "state_to_word_W",
                [self.output_dimension, len(self.vocabulary)],
                initializer=tf.glorot_uniform_initializer())

    @tensor
    def decoding_b(self) -> Optional[tf.Variable]:
        if self.tie_embeddings:
            return tf.zeros(len(self.vocabulary))

        with tf.name_scope("output_projection"):
            return get_variable(
                "state_to_word_b",
                [len(self.vocabulary)],
                initializer=tf.zeros_initializer())

    @tensor
    def embedding_matrix(self) -> tf.Variable:
        """Variables and operations for embedding of input words.

        If we are reusing word embeddings, this function takes the embedding
        matrix from the first encoder
        """
        if self.embeddings_source is not None:
            return self.embeddings_source.embedding_matrix

        assert self.embedding_size is not None

        return get_variable(
            name="word_embeddings",
            shape=[len(self.vocabulary), self.embedding_size],
            initializer=tf.glorot_uniform_initializer())

    def get_logits(self, state: tf.Tensor) -> tf.Tensor:
        """Project the decoder's output layer to logits over the vocabulary."""
        state = dropout(state, self.dropout_keep_prob, self.train_mode)
        logits = tf.matmul(state, self.decoding_w) + self.decoding_b

        if self.supress_unk:
            unk_mask = tf.one_hot(
                UNK_TOKEN_INDEX, depth=len(self.vocabulary), on_value=-1e9)
            logits += unk_mask

        return logits

    @tensor
    def train_loop_result(self) -> Tuple[tf.Tensor, tf.Tensor,
                                         tf.Tensor, tf.Tensor]:
        return self.decoding_loop(train_mode=True)

    @tensor
    def train_logits(self) -> tf.Tensor:
        # THE LAST TRAIN INPUT IS NOT USED IN DECODING FUNCTION
        # (just as a target)
        return tuple(self.train_loop_result)[0]

    @tensor
    def train_output_states(self) -> tf.Tensor:
        return tuple(self.train_loop_result)[1]

    @tensor
    def train_logprobs(self) -> tf.Tensor:
        return tf.nn.log_softmax(self.train_logits)

    @tensor
    def train_xents(self) -> tf.Tensor:
        train_targets = tf.transpose(self.train_inputs)
        softmax_function = None
        if self.label_smoothing:
            softmax_function = (
                lambda labels, logits: tf.losses.softmax_cross_entropy(
                    tf.one_hot(labels, len(self.vocabulary)),
                    logits, label_smoothing=self.label_smoothing))

        return tf.contrib.seq2seq.sequence_loss(
            tf.transpose(self.train_logits, perm=[1, 0, 2]),
            train_targets,
            tf.transpose(self.train_mask),
            average_across_batch=False,
            softmax_loss_function=softmax_function)

    @tensor
    def train_loss(self) -> tf.Tensor:
        return tf.reduce_mean(self.train_xents)

    @property
    def cost(self) -> tf.Tensor:
        return self.train_loss

    @tensor
    def runtime_loop_result(self) -> Tuple[tf.Tensor, tf.Tensor,
                                           tf.Tensor, tf.Tensor]:
        return self.decoding_loop(train_mode=False)

    @tensor
    def runtime_logits(self) -> tf.Tensor:
        return tuple(self.runtime_loop_result)[0]

    @tensor
    def runtime_output_states(self) -> tf.Tensor:
        return tuple(self.runtime_loop_result)[1]

    @tensor
    def runtime_mask(self) -> tf.Tensor:
        return tuple(self.runtime_loop_result)[2]

    @tensor
    def decoded(self) -> tf.Tensor:
        # We disable generating of <pad> tokens at index 0
        # (self.runtime_logits[:, :, 1:]). This shifts the indices
        # of the decoded tokens (therefore, we add +1 to the decoded
        # output indices).

        # self.runtime_logits is of size [batch, sentence_len, vocabulary_size]
        return tf.argmax(self.runtime_logits[:, :, 1:], -1) + 1

    @tensor
    def runtime_xents(self) -> tf.Tensor:
        train_targets = tf.transpose(self.train_inputs)
        batch_major_logits = tf.transpose(self.runtime_logits, [1, 0, 2])
        min_time = tf.minimum(tf.shape(train_targets)[1],
                              tf.shape(batch_major_logits)[1])

        # NOTE if done properly, there should be padding of the shorter
        # sequence instead of cropping to the length of the shorter one

        return tf.contrib.seq2seq.sequence_loss(
            logits=batch_major_logits[:, :min_time],
            targets=train_targets[:, :min_time],
            weights=tf.transpose(self.train_mask)[:, :min_time],
            average_across_batch=False)

    @tensor
    def runtime_loss(self) -> tf.Tensor:
        return tf.reduce_mean(self.runtime_xents)

    @tensor
    def runtime_logprobs(self) -> tf.Tensor:
        return tf.nn.log_softmax(self.runtime_logits)

    @property
    def output_dimension(self) -> int:
        raise NotImplementedError("Abstract property")

    def get_initial_loop_state(self) -> LoopState:

        dec_output = tf.zeros(
            shape=[0, self.batch_size, self.embedding_size],
            dtype=tf.float32,
            name="hist_decoder_outputs")

        logit = tf.zeros(
            shape=[0, self.batch_size, len(self.vocabulary)],
            dtype=tf.float32,
            name="hist_logits")

        mask = tf.zeros(
            shape=[0, self.batch_size],
            dtype=tf.bool,
            name="mask")

        outputs = tf.zeros(
            shape=[0, self.batch_size],
            dtype=tf.int32,
            name="outputs")

        feedables = DecoderFeedables(
            step=tf.constant(0, tf.int32),
            finished=tf.zeros([self.batch_size], dtype=tf.bool),
            input_symbol=self.go_symbols,
            prev_logits=tf.zeros([self.batch_size, len(self.vocabulary)]))

        histories = DecoderHistories(
            logits=logit,
            decoder_outputs=dec_output,
            mask=mask,
            outputs=outputs)

        constants = DecoderConstants(train_inputs=self.train_inputs)

        return LoopState(
            histories=histories,
            constants=constants,
            feedables=feedables)

    def loop_continue_criterion(self, *args) -> tf.Tensor:
        """Decide whether to break out of the while loop.

        Arguments:
            loop_state: ``LoopState`` instance (see the docs for this module).
                Represents current decoder loop state.
        """
        loop_state = LoopState(*args)
        finished = loop_state.feedables.finished
        not_all_done = tf.logical_not(tf.reduce_all(finished))
        before_max_len = tf.less(loop_state.feedables.step,
                                 self.max_output_len)
        return tf.logical_and(not_all_done, before_max_len)

    def get_body(self, train_mode: bool, sample: bool = False,
                 temperature: float = 1) -> Callable:
        """Return the while loop body function."""
        raise NotImplementedError("Abstract method")

    def finalize_loop(self, final_loop_state: LoopState,
                      train_mode: bool) -> None:
        """Execute post-while loop operations.

        Arguments:
            final_loop_state: Decoder loop state at the end
                of the decoding loop.
            train_mode: Boolean flag, telling whether this is
                a training run.
        """

    def decoding_loop(self, train_mode: bool, sample: bool = False,
                      temperature: float = 1) -> Tuple[tf.Tensor, tf.Tensor,
                                                       tf.Tensor, tf.Tensor]:
        """Run the decoding while loop.

        Calls get_initial_loop_state and constructs tf.while_loop
        with the continuation criterion returned from loop_continue_criterion,
        and body function returned from get_body.

        After finishing the tf.while_loop, it calls finalize_loop
        to further postprocess the final decoder loop state (usually
        by stacking Tensors containing decoding histories).

        Arguments:
            train_mode: Boolean flag, telling whether this is
                a training run.
            sample: Boolean flag, telling whether we should sample
                the output symbols from the output distribution instead
                of using argmax or gold data.
            temperature: float value specifying the softmax temperature
        """
        initial_loop_state = self.get_initial_loop_state()
        final_loop_state = tf.while_loop(
            self.loop_continue_criterion,
            self.get_body(train_mode, sample, temperature),
            initial_loop_state,
            shape_invariants=tf.contrib.framework.nest.map_structure(
                get_state_shape_invariants, initial_loop_state))

        self.finalize_loop(final_loop_state, train_mode)

        logits = final_loop_state.histories.logits
        decoder_outputs = final_loop_state.histories.decoder_outputs
        decoded = final_loop_state.histories.outputs

        # TODO mask should include also the end symbol
        mask = final_loop_state.histories.mask

        return logits, decoder_outputs, mask, decoded

    def feed_dict(self, dataset: Dataset, train: bool = False) -> FeedDict:
        """Populate the feed dictionary for the decoder object.

        Arguments:
            dataset: The dataset to use for the decoder.
            train: Boolean flag, telling whether this is a training run.
        """
        fd = ModelPart.feed_dict(self, dataset, train)

        sentences = dataset.maybe_get_series(self.data_id)

        if sentences is None and train:
            raise ValueError("When training, you must feed "
                             "reference sentences")

        go_symbol_idx = self.vocabulary.get_word_index(START_TOKEN)
        fd[self.go_symbols] = np.full([len(dataset)], go_symbol_idx,
                                      dtype=np.int32)

        if sentences is not None:
            sentences_list = list(sentences)
            # train_mode=False, since we don't want to <unk>ize target words!
            inputs, weights = self.vocabulary.sentences_to_tensor(
                sentences_list, self.max_output_len, train_mode=False,
                add_start_symbol=False, add_end_symbol=True,
                pad_to_max_len=False)

            fd[self.train_inputs] = inputs
            fd[self.train_mask] = weights

        return fd
<|endoftext|>


<commit_before># Copyright (c) 2015 Taturiello Consulting, Meh.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from oslo_config import cfg
from oslo_log import log
from oslo_utils import importutils
from pecan import request
from pecan import response

from neutron.api.v2 import attributes
from neutron.common import constants
from neutron.common import exceptions as n_exc
from neutron.pecan_wsgi.controllers import utils
from neutron.quota import resource_registry

LOG = log.getLogger(__name__)
RESOURCE_NAME = "quota"

class QuotasController(utils.NeutronPecanController):

    def __init__(self):
        self._driver = importutils.import_class(
            cfg.CONF.QUOTAS.quota_driver
        )
        super(QuotasController, self).__init__(
            "%ss" % RESOURCE_NAME, RESOURCE_NAME)

    def _check_admin(self, context,
                     reason=_("Only admin can view or configure quota")):
        if not context.is_admin:
            raise n_exc.AdminRequired(reason=reason)

    @utils.expose()
    def _lookup(self, tenant_id, *remainder):
        return QuotaController(self._driver, tenant_id), remainder

    @utils.expose()
    def index(self):
        neutron_context = request.context.get('neutron_context')
        # FIXME(salv-orlando): There shouldn't be any need to to this eplicit
        # check. However some behaviours from the "old" extension have
        # been temporarily carried over here
        self._check_admin(neutron_context)
        # TODO(salv-orlando): proper plurals management
        return {self.collection:
                self._driver.get_all_quotas(
                    neutron_context,
                    resource_registry.get_all_resources())}

class QuotaController(utils.NeutronPecanController):

    def __init__(self, _driver, tenant_id):
        self._driver = _driver
        self._tenant_id = tenant_id

        super(QuotaController, self).__init__(
            "%ss" % RESOURCE_NAME, RESOURCE_NAME)

        # Ensure limits for all registered resources are returned
        attr_dict = attributes.RESOURCE_ATTRIBUTE_MAP[self.collection]
        for quota_resource in resource_registry.get_all_resources().keys():
            attr_dict[quota_resource] = {
                'allow_post': False,
                'allow_put': True,
                'convert_to': attributes.convert_to_int,
                'validate': {
                    'type:range': [-1, constants.DB_INTEGER_MAX_VALUE]},
                'is_visible': True}

    @utils.expose(generic=True)
    def index(self):
        return get_tenant_quotas(self._tenant_id, self._driver)

    @utils.when(index, method='PUT')
    def put(self, *args, **kwargs):
        neutron_context = request.context.get('neutron_context')
        # For put requests there's always going to be a single element
        quota_data = request.context['resources'][0]
        for key, value in quota_data.items():
            self._driver.update_quota_limit(
                neutron_context, self._tenant_id, key, value)
        return get_tenant_quotas(self._tenant_id, self._driver)

    @utils.when(index, method='DELETE')
    def delete(self):
        neutron_context = request.context.get('neutron_context')
        self._driver.delete_tenant_quota(neutron_context,
                                         self._tenant_id)
        response.status = 204

def get_tenant_quotas(tenant_id, driver=None):
    if not driver:
        driver = importutils.import_class(cfg.CONF.QUOTAS.quota_driver)

    neutron_context = request.context.get('neutron_context')
    if tenant_id == 'tenant':
        # NOTE(salv-orlando): Read the following before the code in order
        # to avoid puking.
        # There is a weird undocumented behaviour of the Neutron quota API
        # as 'tenant' is used as an API action to return the identifier
        # of the tenant in the request context. This is used exclusively
        # for interaction with python-neutronclient and is a possibly
        # unnecessary 'whoami' API endpoint. Pending resolution of this
        # API issue, this controller will just treat the magic string
        # 'tenant' (and only that string) and return the response expected
        # by python-neutronclient
        return {'tenant': {'tenant_id': neutron_context.tenant_id}}
    tenant_quotas = driver.get_tenant_quotas(
        neutron_context,
        resource_registry.get_all_resources(),
        tenant_id)
    tenant_quotas['tenant_id'] = tenant_id
    return {RESOURCE_NAME: tenant_quotas}
<commit_msg>Pecan: remove deprecated warning<commit_after># Copyright (c) 2015 Taturiello Consulting, Meh.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from oslo_config import cfg
from oslo_log import log
from oslo_utils import importutils
from pecan import request
from pecan import response

from neutron._i18n import _
from neutron.api.v2 import attributes
from neutron.common import constants
from neutron.common import exceptions as n_exc
from neutron.pecan_wsgi.controllers import utils
from neutron.quota import resource_registry

LOG = log.getLogger(__name__)
RESOURCE_NAME = "quota"

class QuotasController(utils.NeutronPecanController):

    def __init__(self):
        self._driver = importutils.import_class(
            cfg.CONF.QUOTAS.quota_driver
        )
        super(QuotasController, self).__init__(
            "%ss" % RESOURCE_NAME, RESOURCE_NAME)

    def _check_admin(self, context,
                     reason=_("Only admin can view or configure quota")):
        if not context.is_admin:
            raise n_exc.AdminRequired(reason=reason)

    @utils.expose()
    def _lookup(self, tenant_id, *remainder):
        return QuotaController(self._driver, tenant_id), remainder

    @utils.expose()
    def index(self):
        neutron_context = request.context.get('neutron_context')
        # FIXME(salv-orlando): There shouldn't be any need to to this eplicit
        # check. However some behaviours from the "old" extension have
        # been temporarily carried over here
        self._check_admin(neutron_context)
        # TODO(salv-orlando): proper plurals management
        return {self.collection:
                self._driver.get_all_quotas(
                    neutron_context,
                    resource_registry.get_all_resources())}

class QuotaController(utils.NeutronPecanController):

    def __init__(self, _driver, tenant_id):
        self._driver = _driver
        self._tenant_id = tenant_id

        super(QuotaController, self).__init__(
            "%ss" % RESOURCE_NAME, RESOURCE_NAME)

        # Ensure limits for all registered resources are returned
        attr_dict = attributes.RESOURCE_ATTRIBUTE_MAP[self.collection]
        for quota_resource in resource_registry.get_all_resources().keys():
            attr_dict[quota_resource] = {
                'allow_post': False,
                'allow_put': True,
                'convert_to': attributes.convert_to_int,
                'validate': {
                    'type:range': [-1, constants.DB_INTEGER_MAX_VALUE]},
                'is_visible': True}

    @utils.expose(generic=True)
    def index(self):
        return get_tenant_quotas(self._tenant_id, self._driver)

    @utils.when(index, method='PUT')
    def put(self, *args, **kwargs):
        neutron_context = request.context.get('neutron_context')
        # For put requests there's always going to be a single element
        quota_data = request.context['resources'][0]
        for key, value in quota_data.items():
            self._driver.update_quota_limit(
                neutron_context, self._tenant_id, key, value)
        return get_tenant_quotas(self._tenant_id, self._driver)

    @utils.when(index, method='DELETE')
    def delete(self):
        neutron_context = request.context.get('neutron_context')
        self._driver.delete_tenant_quota(neutron_context,
                                         self._tenant_id)
        response.status = 204

def get_tenant_quotas(tenant_id, driver=None):
    if not driver:
        driver = importutils.import_class(cfg.CONF.QUOTAS.quota_driver)

    neutron_context = request.context.get('neutron_context')
    if tenant_id == 'tenant':
        # NOTE(salv-orlando): Read the following before the code in order
        # to avoid puking.
        # There is a weird undocumented behaviour of the Neutron quota API
        # as 'tenant' is used as an API action to return the identifier
        # of the tenant in the request context. This is used exclusively
        # for interaction with python-neutronclient and is a possibly
        # unnecessary 'whoami' API endpoint. Pending resolution of this
        # API issue, this controller will just treat the magic string
        # 'tenant' (and only that string) and return the response expected
        # by python-neutronclient
        return {'tenant': {'tenant_id': neutron_context.tenant_id}}
    tenant_quotas = driver.get_tenant_quotas(
        neutron_context,
        resource_registry.get_all_resources(),
        tenant_id)
    tenant_quotas['tenant_id'] = tenant_id
    return {RESOURCE_NAME: tenant_quotas}
<|endoftext|>


<commit_before># Copyright 2012 OpenStack Foundation.
# All Rights Reserved
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.
#

from neutronclient.neutron import v2_0 as cmd_base
from neutronclient.openstack.common.gettextutils import _

class ListExt(cmd_base.ListCommand):
    """List all extensions."""

    resource = 'extension'
    list_columns = ['alias', 'name']

class ShowExt(cmd_base.ShowCommand):
    """Show information of a given resource."""

    resource = "extension"
    allow_names = False

    def get_parser(self, prog_name):
        parser = super(cmd_base.ShowCommand, self).get_parser(prog_name)
        cmd_base.add_show_list_common_argument(parser)
        parser.add_argument(
            'id', metavar='EXT-ALIAS',
            help=_('The extension alias.'))
        return parser
<commit_msg>Remove incorrect super() call<commit_after># Copyright 2012 OpenStack Foundation.
# All Rights Reserved
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.
#

from neutronclient.neutron import v2_0 as cmd_base

class ListExt(cmd_base.ListCommand):
    """List all extensions."""

    resource = 'extension'
    list_columns = ['alias', 'name']

class ShowExt(cmd_base.ShowCommand):
    """Show information of a given resource."""

    resource = "extension"
    allow_names = False
<|endoftext|>


<commit_before>""" Compile jni C++ bindings and add them as resources
to the project

"""

import argparse
import os
import platform
import sys

from qisys import ui
import qisys.sh
import qibuild.find
import qibuild.parsers

def copy_jni_resources(build_worktree):
    """ Create symlinks (or copy), the jni library and its
    dependencies from the build dirs to the resource folder

    """
    android = "android" in (build_worktree.build_config.active_build_config.name or "")
    # FIXME: boost when using a toolchain on linux ?
    libs = list()
    proj_to_lib = [
            ("libqi",          "qi"),
            ("qimessaging-jni", "qimessagingjni")]
    if android:
        proj_to_lib.append(
            ("qimessaging-jni", "gnustl_shared"))

    for project_name, library_name in proj_to_lib:
        try:
            project = build_worktree.get_build_project(project_name, raises=True)
            library_path = qibuild.find.find_lib([project.sdk_directory], library_name)
            libs.append(library_path)
        except qibuild.find.NotFound as e:
            ui.error(e)
            ui.error("Make sure qimessaging-jni has been built")
            sys.exit(1)
    toolchain = build_worktree.toolchain
    boost = toolchain.get_package("boost")
    if platform.system() == "Windows":
        boost_lib = os.path.join(boost.path, "bin")
    else:
        boost_lib = os.path.join(boost.path, "lib")

    boost_lib_files = [os.path.join(boost_lib, f) for f in os.listdir(boost_lib)
        if os.path.isfile(os.path.join(boost_lib, f))]
    libs_to_keep = ("chrono", "system", "filesystem", "locale", "thread",
    "regex", "program_options")
    boost_lib_files = [f for f in boost_lib_files if any(lib in f for lib in libs_to_keep)]
    boost_lib_files = [f for f in boost_lib_files if f.endswith(".so") or f.endswith(".dylib") or f.endswith(".dll")]
    libs.extend(boost_lib_files)

    java_proj = build_worktree.worktree.get_project("sdk/libqi-java/qimessaging")
    if android:
        dest = os.path.join(java_proj.path, "native-android")
    else:
        dest = os.path.join(java_proj.path, "native")
    qisys.sh.mkdir(dest)
    for lib_src in libs:
        lib_dest = os.path.join(dest, os.path.basename(lib_src))
        copy_or_link(lib_src, lib_dest)

def copy_or_link(src, dest):
    """ Create a symlink from src to dest, or copy the file
    if the operating system does not support it

    """
    if os.name == 'nt':
        qisys.sh.install(src, dest)
    else:
        qisys.sh.rm(dest)
        os.symlink(src, dest)

def main():
    parser = argparse.ArgumentParser()
    qibuild.parsers.cmake_build_parser(parser)
    args = parser.parse_args()
    build_worktree = qibuild.parsers.get_build_worktree(args)
    ui.info("Copying jni libraries as resources ...")
    copy_jni_resources(build_worktree)
    ui.info("Done")

if __name__ == "__main__":
    main()
<commit_msg>remove copy-jni-resources<commit_after><|endoftext|>


<commit_before>""" Compile jni C++ bindings and add them as resources
to the project

"""

import argparse
import os
import sys

from qisys import ui
import qisys.sh
import qibuild.find
import qibuild.parsers

def copy_jni_resources(build_worktree):
    """ Create symlinks (or copy), the jni library and its
    dependencies from the build dirs to the resource folder

    """
    android = "android" in (build_worktree.build_config.active_config or "")
    # FIXME: boost when using a toolchain on linux ?
    libs = list()
    proj_to_lib = [
            ("libqi",          "qi"),
            ("qimessaging-jni", "qimessagingjni")]
    if android:
        proj_to_lib.append(
            ("qimessaging-jni", "gnustl_shared"))

    for project_name, library_name in proj_to_lib:
        try:
            project = build_worktree.get_build_project(project_name, raises=True)
            library_path = qibuild.find.find_lib([project.sdk_directory], library_name)
            libs.append(library_path)
        except qibuild.find.NotFound as e:
            ui.error(e)
            ui.error("Make sure qimessaging-jni has been built")
            sys.exit(1)

    jni_proj = build_worktree.get_build_project("qimessaging-jni")
    java_proj = build_worktree.worktree.get_project("sdk/libqi-java/qimessaging")
    if android:
        dest = os.path.join(java_proj.path, "native-android")
    else:
        dest = os.path.join(java_proj.path, "native")
    qisys.sh.mkdir(dest)
    for lib_src in libs:
        lib_dest = os.path.join(dest, os.path.basename(lib_src))
        copy_or_link(lib_src, lib_dest)

def copy_or_link(src, dest):
    """ Create a symlink from src to dest, or copy the file
    if the operating system does not support it

    """
    if os.name == 'nt':
        qisys.sh.install(src, dest)
    else:
        qisys.sh.rm(dest)
        os.symlink(src, dest)

def main():
    parser = argparse.ArgumentParser()
    qibuild.parsers.cmake_build_parser(parser)
    args = parser.parse_args()
    build_worktree = qibuild.parsers.get_build_worktree(args)
    ui.info("Copying jni libraries as resources ...")
    copy_jni_resources(build_worktree)
    ui.info("Done")

if __name__ == "__main__":
    main()
<commit_msg>copy-jni-resources.py: also copy boost libraries<commit_after>""" Compile jni C++ bindings and add them as resources
to the project

"""

import argparse
import os
import platform
import sys

from qisys import ui
import qisys.sh
import qibuild.find
import qibuild.parsers

def copy_jni_resources(build_worktree):
    """ Create symlinks (or copy), the jni library and its
    dependencies from the build dirs to the resource folder

    """
    android = "android" in (build_worktree.build_config.active_config or "")
    # FIXME: boost when using a toolchain on linux ?
    libs = list()
    proj_to_lib = [
            ("libqi",          "qi"),
            ("qimessaging-jni", "qimessagingjni")]
    if android:
        proj_to_lib.append(
            ("qimessaging-jni", "gnustl_shared"))

    for project_name, library_name in proj_to_lib:
        try:
            project = build_worktree.get_build_project(project_name, raises=True)
            library_path = qibuild.find.find_lib([project.sdk_directory], library_name)
            libs.append(library_path)
        except qibuild.find.NotFound as e:
            ui.error(e)
            ui.error("Make sure qimessaging-jni has been built")
            sys.exit(1)
    toolchain = build_worktree.toolchain
    boost = toolchain.get_package("boost")
    if platform.system() == "Windows":
        boost_lib = os.path.join(boost.path, "bin")
    else:
        boost_lib = os.path.join(boost.path, "lib")

    boost_lib_files = [os.path.join(boost_lib, f) for f in os.listdir(boost_lib)
        if os.path.isfile(os.path.join(boost_lib, f))]
    libs_to_keep = ("chrono", "filesystem", "locale" "thread", "regex",
        "program_options")
    boost_lib_files = [f for f in boost_lib_files if any(lib in f for lib in libs_to_keep)]
    libs.extend(boost_lib_files)

    java_proj = build_worktree.worktree.get_project("sdk/libqi-java/qimessaging")
    if android:
        dest = os.path.join(java_proj.path, "native-android")
    else:
        dest = os.path.join(java_proj.path, "native")
    qisys.sh.mkdir(dest)
    for lib_src in libs:
        lib_dest = os.path.join(dest, os.path.basename(lib_src))
        copy_or_link(lib_src, lib_dest)

def copy_or_link(src, dest):
    """ Create a symlink from src to dest, or copy the file
    if the operating system does not support it

    """
    if os.name == 'nt':
        qisys.sh.install(src, dest)
    else:
        qisys.sh.rm(dest)
        os.symlink(src, dest)

def main():
    parser = argparse.ArgumentParser()
    qibuild.parsers.cmake_build_parser(parser)
    args = parser.parse_args()
    build_worktree = qibuild.parsers.get_build_worktree(args)
    ui.info("Copying jni libraries as resources ...")
    copy_jni_resources(build_worktree)
    ui.info("Done")

if __name__ == "__main__":
    main()
<|endoftext|>