<commit_before># -*- coding: utf-8 -*-

import warnings

from influxdb.exceptions import InfluxDBClientError

_sentinel = object()

class ResultSet(object):
    """A wrapper around a single InfluxDB query result"""

    def __init__(self, series, raise_errors=True):
        self._raw = series
        self._error = self.raw.get('error', None)

        if self.error is not None and raise_errors is True:
            raise InfluxDBClientError(self.error)

    @property
    def raw(self):
        """Raw JSON from InfluxDB"""
        return self._raw

    @raw.setter
    def raw(self, value):
        self._raw = value

    @property
    def error(self):
        """Error returned by InfluxDB"""
        return self._error

    def __getitem__(self, key):
        """
        :param key: Either a serie name, or a tags_dict, or
                    a 2-tuple(serie_name, tags_dict).
                    If the serie name is None (or not given) then any serie
                    matching the eventual given tags will be given its points
                    one after the other.
                    To get the points of every serie in this resultset then
                    you have to provide None as key.
        :return: A generator yielding `Point`s matching the given key.
        NB:
        The order in which the points are yielded is actually undefined but
        it might change..
        """

        warnings.warn(
            ("ResultSet's ``__getitem__`` method will be deprecated. Use"
             "``get_points`` instead."),
            DeprecationWarning
        )

        if isinstance(key, tuple):
            if 2 != len(key):
                raise TypeError('only 2-tuples allowed')
            name = key[0]
            tags = key[1]
            if not isinstance(tags, dict) and tags is not None:
                raise TypeError('tags should be a dict')
        elif isinstance(key, dict):
            name = None
            tags = key
        else:
            name = key
            tags = None

        return self.get_points(name, tags)

    def get_points(self, measurement=None, tags=None):
        """
        Returns a generator for all the points that match the given filters.

        :param measurement: The measurement name
        :type measurement: str

        :param tags: Tags to look for
        :type tags: dict

        :return: Points generator
        """

        # Raise error if measurement is not str or bytes
        if not isinstance(measurement,
                          (bytes, type(b''.decode()), type(None))):
            raise TypeError('measurement must be an str or None')

        for serie in self._get_series():
            serie_name = serie.get('measurement', serie.get('name', 'results'))
            if serie_name is None:
                # this is a "system" query or a query which
                # doesn't return a name attribute.
                # like 'show retention policies' ..
                if tags is None:
                    for point in serie['values']:
                        yield self.point_from_cols_vals(
                            serie['columns'],
                            point
                        )

            elif measurement in (None, serie_name):
                # by default if no tags was provided then
                # we will matches every returned serie
                serie_tags = serie.get('tags', {})
                if tags is None or self._tag_matches(serie_tags, tags):
                    for point in serie.get('values', []):
                        yield self.point_from_cols_vals(
                            serie['columns'],
                            point
                        )

    def __repr__(self):
        items = []

        for item in self.items():
            items.append("'%s': %s" % (item[0], list(item[1])))

        return "ResultSet({%s})" % ", ".join(items)

    def __iter__(self):
        """ Iterating a ResultSet will yield one dict instance per serie result.
        """
        for key in self.keys():
            yield list(self.__getitem__(key))

    def _tag_matches(self, tags, filter):
        """Checks if all key/values in filter match in tags"""
        for tag_name, tag_value in filter.items():
            # using _sentinel as I'm not sure that "None"
            # could be used, because it could be a valid
            # serie_tags value : when a serie has no such tag
            # then I think it's set to /null/None/.. TBC..
            serie_tag_value = tags.get(tag_name, _sentinel)
            if serie_tag_value != tag_value:
                return False
        return True

    def _get_series(self):
        """Returns all series"""
        return self.raw.get('series', [])

    def __len__(self):
        return len(self.keys())

    def keys(self):
        """
        :return: List of keys. Keys are tuples (serie_name, tags)
        """
        keys = []
        for serie in self._get_series():
            keys.append(
                (serie.get('measurement',
                           serie.get('name', 'results')),
                 serie.get('tags', None))
            )
        return keys

    def items(self):
        """
        :return: List of tuples, (key, generator)
        """
        items = []
        for serie in self._get_series():
            serie_key = (serie.get('measurement',
                                   serie.get('name', 'results')),
                         serie.get('tags', None))
            items.append(
                (serie_key, self[serie_key])
            )
        return items

    @staticmethod
    def point_from_cols_vals(cols, vals):
        """ Creates a dict from columns and values lists

        :param cols: List of columns
        :param vals: List of values
        :return: Dict where keys are columns.
        """
        point = {}
        for col_index, col_name in enumerate(cols):
            point[col_name] = vals[col_index]
        return point
<commit_msg>Improve ResultSet.items() performance<commit_after># -*- coding: utf-8 -*-

import warnings

from influxdb.exceptions import InfluxDBClientError

_sentinel = object()

class ResultSet(object):
    """A wrapper around a single InfluxDB query result"""

    def __init__(self, series, raise_errors=True):
        self._raw = series
        self._error = self.raw.get('error', None)

        if self.error is not None and raise_errors is True:
            raise InfluxDBClientError(self.error)

    @property
    def raw(self):
        """Raw JSON from InfluxDB"""
        return self._raw

    @raw.setter
    def raw(self, value):
        self._raw = value

    @property
    def error(self):
        """Error returned by InfluxDB"""
        return self._error

    def __getitem__(self, key):
        """
        :param key: Either a serie name, or a tags_dict, or
                    a 2-tuple(serie_name, tags_dict).
                    If the serie name is None (or not given) then any serie
                    matching the eventual given tags will be given its points
                    one after the other.
                    To get the points of every serie in this resultset then
                    you have to provide None as key.
        :return: A generator yielding `Point`s matching the given key.
        NB:
        The order in which the points are yielded is actually undefined but
        it might change..
        """

        warnings.warn(
            ("ResultSet's ``__getitem__`` method will be deprecated. Use"
             "``get_points`` instead."),
            DeprecationWarning
        )

        if isinstance(key, tuple):
            if 2 != len(key):
                raise TypeError('only 2-tuples allowed')
            name = key[0]
            tags = key[1]
            if not isinstance(tags, dict) and tags is not None:
                raise TypeError('tags should be a dict')
        elif isinstance(key, dict):
            name = None
            tags = key
        else:
            name = key
            tags = None

        return self.get_points(name, tags)

    def get_points(self, measurement=None, tags=None):
        """
        Returns a generator for all the points that match the given filters.

        :param measurement: The measurement name
        :type measurement: str

        :param tags: Tags to look for
        :type tags: dict

        :return: Points generator
        """

        # Raise error if measurement is not str or bytes
        if not isinstance(measurement,
                          (bytes, type(b''.decode()), type(None))):
            raise TypeError('measurement must be an str or None')

        for serie in self._get_series():
            serie_name = serie.get('measurement', serie.get('name', 'results'))
            print serie_name
            if serie_name is None:
                # this is a "system" query or a query which
                # doesn't return a name attribute.
                # like 'show retention policies' ..
                if tags is None:
                    for item in self._get_points_for_serie(serie):
                        yield item

            elif measurement in (None, serie_name):
                # by default if no tags was provided then
                # we will matches every returned serie
                serie_tags = serie.get('tags', {})
                if tags is None or self._tag_matches(serie_tags, tags):
                    for item in self._get_points_for_serie(serie):
                        yield item

    def __repr__(self):
        items = []

        for item in self.items():
            items.append("'%s': %s" % (item[0], list(item[1])))

        return "ResultSet({%s})" % ", ".join(items)

    def __iter__(self):
        """ Iterating a ResultSet will yield one dict instance per serie result.
        """
        for key in self.keys():
            yield list(self.__getitem__(key))

    def _tag_matches(self, tags, filter):
        """Checks if all key/values in filter match in tags"""
        for tag_name, tag_value in filter.items():
            # using _sentinel as I'm not sure that "None"
            # could be used, because it could be a valid
            # serie_tags value : when a serie has no such tag
            # then I think it's set to /null/None/.. TBC..
            serie_tag_value = tags.get(tag_name, _sentinel)
            if serie_tag_value != tag_value:
                return False
        return True

    def _get_series(self):
        """Returns all series"""
        return self.raw.get('series', [])

    def __len__(self):
        return len(self.keys())

    def keys(self):
        """
        :return: List of keys. Keys are tuples (serie_name, tags)
        """
        keys = []
        for serie in self._get_series():
            keys.append(
                (serie.get('measurement',
                           serie.get('name', 'results')),
                 serie.get('tags', None))
            )
        return keys

    def items(self):
        """
        :return: List of tuples, (key, generator)
        """
        items = []
        for serie in self._get_series():
            serie_key = (serie.get('measurement',
                                   serie.get('name', 'results')),
                         serie.get('tags', None))
            items.append(
                (serie_key, self._get_points_for_serie(serie))
            )
        return items

    def _get_points_for_serie(self, serie):
        """ Return generator of dict from columns and values of a serie

        :param serie: One serie
        :return: Generator of dicts
        """
        for point in serie.get('values', []):
            yield self.point_from_cols_vals(
                serie['columns'],
                point
            )

    @staticmethod
    def point_from_cols_vals(cols, vals):
        """ Creates a dict from columns and values lists

        :param cols: List of columns
        :param vals: List of values
        :return: Dict where keys are columns.
        """
        point = {}
        for col_index, col_name in enumerate(cols):
            point[col_name] = vals[col_index]
        return point
<|endoftext|>


<commit_before>import pytest
import numpy as np

from rio_color.utils import to_math_type
from rio_color.operations import (
    sigmoidal, gamma, saturation,
    rgb2lch, lch2rgb, simple_atmo, parse_operations)

@pytest.fixture
def arr():

    return to_math_type(np.array([
        # red
        [[1, 2],
         [3, 4]],

        # green
        [[5, 6],
         [7, 8]],

        # blue
        [[9, 10],
         [11, 12]]
    ]).astype('uint8') * 10)

@pytest.fixture
def arr_rgba():
    return to_math_type(np.array([
        [[1, 2], [3, 4]],  # red
        [[5, 6], [7, 8]],  # green
        [[9, 10], [11, 12]],  # blue
        [[0, 0], [25.5, 25.5]]  # alpha
    ]).astype('uint8') * 10)

def test_sigmoidal(arr):
    x = sigmoidal(arr, 10, 0.15)
    assert x[0][0][0] - 0.08056034 < 1e-4

    # contrast < 0
    x = sigmoidal(arr, -10, 0.15)
    assert x[0][0][0] - 0.020186627 < 1e-4

    # bias zero, make it a tiny epsilon
    x = sigmoidal(arr, 10, 0)
    assert x[0][0][0] - 0.19362122 < 1e-4

    # contrast zzero, arrays are equal
    x = sigmoidal(arr, 0, 0.15)
    assert np.array_equal(x, arr)

    # output contains NaN
    with pytest.raises(ValueError):
        x = sigmoidal(arr, 100, -0.5)

    # output is not within the range of 0..1
    with pytest.raises(ValueError):
        arr[0][0][0] = 1.0
        arr[0][0][1] = 2.0
        x = sigmoidal(arr, 10, -0.5)

def test_gamma(arr):
    x = gamma(arr, 0.95)
    assert x[0][0][0] - 0.033069782 < 1e-4
    # output is not within the range of 0..1
    with pytest.raises(ValueError):
        x = gamma(arr, -2.0)
    # test output contains inf and is out of range 0..1
    with pytest.raises(ValueError):
        x = gamma(arr, -0.001)

def test_sat(arr):
    x = saturation(arr, 50)
    assert x[0][0][0] - 0.1513809257 < 1e-4

def test_sat_rgba_direct(arr_rgba):
    # Anything but 3-band RGB will fail
    with pytest.raises(ValueError):
        saturation(arr_rgba, 50)
    with pytest.raises(ValueError):
        saturation(arr_rgba[0:2], 50)

def test_atmo(arr):
    x = simple_atmo(arr, 0.03, 10, 15)
    assert x[0][0][0] - 0.080560341 < 1e-4

    # Gamma output is not within the range 0..1
    with pytest.raises(ValueError):
        x = simple_atmo(arr, 2.0, 10, 15)

    # Sigmoidal contrast output contains NaN
    with pytest.raises(ValueError):
        x = simple_atmo(arr, 0.03, 1000, -5)

def test_parse_gamma(arr):
    f = list(parse_operations(["gamma 1,2,3 0.95", ]))[0]
    assert np.array_equal(f(arr), gamma(arr, 0.95))

def test_parse_sigmoidal(arr):
    f = list(parse_operations(["sigmoidal 1,2,3 5 0.53", ]))[0]
    assert np.array_equal(
        f(arr),
        sigmoidal(arr, contrast=5, bias=0.53))

def test_parse_multi(arr):
    f1, f2 = list(parse_operations([
        "gamma 1,2,3 0.95", "sigmoidal 1,2,3 35 0.13"]))
    assert np.array_equal(
        f2(f1(arr)),
        sigmoidal(gamma(arr, g=0.95), contrast=35, bias=0.13))

def test_parse_rgb(arr):
    f = list(parse_operations(["saturation 125", ]))[0]
    assert np.allclose(f(arr), saturation(arr, 125))

def test_parse_rgba(arr, arr_rgba):
    f = list(parse_operations(["gamma r,g 0.95", ]))[0]
    rgb = f(arr)
    assert rgb.shape[0] == 3

    rgba = f(arr_rgba)
    assert rgba.shape[0] == 4
    # rgb bands are same
    assert np.allclose(rgba[0:3], rgb[0:3])
    # alpha unaltered
    assert np.array_equal(rgba[3], arr_rgba[3])

def test_saturation_rgba(arr, arr_rgba):
    f = list(parse_operations(["saturation 125", ]))[0]

    satrgb = f(arr)
    assert satrgb.shape[0] == 3

    satrgba = f(arr_rgba)
    # Still rgba
    assert satrgba.shape[0] == 4
    # alpha is unaltered
    assert np.array_equal(satrgba[3], arr_rgba[3])
    # first 3 bands are same b/t rgb and rgba
    assert np.allclose(satrgba[0:3], satrgb[0:3])

def test_parse_bad_op():
    with pytest.raises(ValueError):
        list(parse_operations(["foob 123"]))

def test_parse_bands(arr):
    fa = list(parse_operations(["gamma 1,2 0.95", ]))[0]
    fb = list(parse_operations(["gamma R,g 0.95", ]))[0]
    assert np.array_equal(fa(arr), fb(arr))

    with pytest.raises(ValueError):
        list(parse_operations(["gamma 7,8,9 1.05"]))
<commit_msg>add edge case nan<commit_after>import pytest
import numpy as np

from rio_color.utils import to_math_type
from rio_color.operations import (
    sigmoidal, gamma, saturation,
    rgb2lch, lch2rgb, simple_atmo, parse_operations)

@pytest.fixture
def arr():

    return to_math_type(np.array([
        # red
        [[1, 2],
         [3, 4]],

        # green
        [[5, 6],
         [7, 8]],

        # blue
        [[9, 10],
         [11, 12]]
    ]).astype('uint8') * 10)

@pytest.fixture
def arr_rgba():
    return to_math_type(np.array([
        [[1, 2], [3, 4]],  # red
        [[5, 6], [7, 8]],  # green
        [[9, 10], [11, 12]],  # blue
        [[0, 0], [25.5, 25.5]]  # alpha
    ]).astype('uint8') * 10)

def test_sigmoidal(arr):
    x = sigmoidal(arr, 10, 0.15)
    assert x[0][0][0] - 0.08056034 < 1e-4

    # contrast < 0
    x = sigmoidal(arr, -10, 0.15)
    assert x[0][0][0] - 0.020186627 < 1e-4

    # bias zero, make it a tiny epsilon
    x = sigmoidal(arr, 10, 0)
    assert x[0][0][0] - 0.19362122 < 1e-4

    # contrast zzero, arrays are equal
    x = sigmoidal(arr, 0, 0.15)
    assert np.array_equal(x, arr)

    # output contains NaN
    with pytest.raises(ValueError):
        x = sigmoidal(arr, 100, -0.5)

    # output is not within the range of 0..1
    with pytest.raises(ValueError):
        arr[0][0][0] = 1.0
        arr[0][0][1] = 2.0
        x = sigmoidal(arr, 10, -0.5)

def test_gamma(arr):
    x = gamma(arr, 0.95)
    assert x[0][0][0] - 0.033069782 < 1e-4
    # output is not within the range of 0..1
    with pytest.raises(ValueError):
        x = gamma(arr, -2.0)
    # test output contains inf and is out of range 0..1
    with pytest.raises(ValueError):
        x = gamma(arr, -0.001)
    # test output contains NaN
    with pytest.raises(ValueError):
        x = gamma(arr, np.nan)

def test_sat(arr):
    x = saturation(arr, 50)
    assert x[0][0][0] - 0.1513809257 < 1e-4

def test_sat_rgba_direct(arr_rgba):
    # Anything but 3-band RGB will fail
    with pytest.raises(ValueError):
        saturation(arr_rgba, 50)
    with pytest.raises(ValueError):
        saturation(arr_rgba[0:2], 50)

def test_atmo(arr):
    x = simple_atmo(arr, 0.03, 10, 15)
    assert x[0][0][0] - 0.080560341 < 1e-4

    # Gamma output is not within the range 0..1
    with pytest.raises(ValueError):
        x = simple_atmo(arr, 2.0, 10, 15)

    # Sigmoidal contrast output contains NaN
    with pytest.raises(ValueError):
        x = simple_atmo(arr, 0.03, 1000, -5)

def test_parse_gamma(arr):
    f = list(parse_operations(["gamma 1,2,3 0.95", ]))[0]
    assert np.array_equal(f(arr), gamma(arr, 0.95))

def test_parse_sigmoidal(arr):
    f = list(parse_operations(["sigmoidal 1,2,3 5 0.53", ]))[0]
    assert np.array_equal(
        f(arr),
        sigmoidal(arr, contrast=5, bias=0.53))

def test_parse_multi(arr):
    f1, f2 = list(parse_operations([
        "gamma 1,2,3 0.95", "sigmoidal 1,2,3 35 0.13"]))
    assert np.array_equal(
        f2(f1(arr)),
        sigmoidal(gamma(arr, g=0.95), contrast=35, bias=0.13))

def test_parse_rgb(arr):
    f = list(parse_operations(["saturation 125", ]))[0]
    assert np.allclose(f(arr), saturation(arr, 125))

def test_parse_rgba(arr, arr_rgba):
    f = list(parse_operations(["gamma r,g 0.95", ]))[0]
    rgb = f(arr)
    assert rgb.shape[0] == 3

    rgba = f(arr_rgba)
    assert rgba.shape[0] == 4
    # rgb bands are same
    assert np.allclose(rgba[0:3], rgb[0:3])
    # alpha unaltered
    assert np.array_equal(rgba[3], arr_rgba[3])

def test_saturation_rgba(arr, arr_rgba):
    f = list(parse_operations(["saturation 125", ]))[0]

    satrgb = f(arr)
    assert satrgb.shape[0] == 3

    satrgba = f(arr_rgba)
    # Still rgba
    assert satrgba.shape[0] == 4
    # alpha is unaltered
    assert np.array_equal(satrgba[3], arr_rgba[3])
    # first 3 bands are same b/t rgb and rgba
    assert np.allclose(satrgba[0:3], satrgb[0:3])

def test_parse_bad_op():
    with pytest.raises(ValueError):
        list(parse_operations(["foob 123"]))

def test_parse_bands(arr):
    fa = list(parse_operations(["gamma 1,2 0.95", ]))[0]
    fb = list(parse_operations(["gamma R,g 0.95", ]))[0]
    assert np.array_equal(fa(arr), fb(arr))

    with pytest.raises(ValueError):
        list(parse_operations(["gamma 7,8,9 1.05"]))
<|endoftext|>


<commit_before>from ..Script import Script
class BQ_PauseAtHeight(Script):
    def __init__(self):
        super().__init__()

    def getSettingData(self):
        return {
            "label":"Pause at height (BQ Printers)",
            "key": "BQ_PauseAtHeight",
            "settings":
            {
                "pause_height":
                {
                    "label": "Pause height",
                    "description": "At what height should the pause occur",
                    "unit": "mm",
                    "type": "float",
                    "default": 5.0,
                    "visible": True
                }
            }
        }

    def execute(self, data):
        x = 0.
        y = 0.
        current_z = 0.
        pause_z = self.getSettingValueByKey("pause_height")
        for layer in data:
            lines = layer.split("\n")
            for line in lines:
                if self.getValue(line, 'G') == 1 or self.getValue(line, 'G') == 0:
                    current_z = self.getValue(line, 'Z')
                    if current_z != None:
                        if current_z >= pause_z:
                            prepend_gcode = ";TYPE:CUSTOM\n"
                            prepend_gcode = "; -- Pause at height (%.2f mm) --\n" % pause_z

                            # Insert Pause gcode
                            prepend_gcode += "M25        ; Pauses the print and waits for the user to resume it\n"

                            index = data.index(layer)
                            layer = prepend_gcode + layer
                            data[index] = layer # Override the data of this layer with the modified data
                            return data
                        break
        return data
<commit_msg>Small fix<commit_after>from ..Script import Script
class BQ_PauseAtHeight(Script):
    def __init__(self):
        super().__init__()

    def getSettingData(self):
        return {
            "label":"Pause at height (BQ Printers)",
            "key": "BQ_PauseAtHeight",
            "settings":
            {
                "pause_height":
                {
                    "label": "Pause height",
                    "description": "At what height should the pause occur",
                    "unit": "mm",
                    "type": "float",
                    "default": 5.0,
                    "visible": True
                }
            }
        }

    def execute(self, data):
        x = 0.
        y = 0.
        current_z = 0.
        pause_z = self.getSettingValueByKey("pause_height")
        for layer in data:
            lines = layer.split("\n")
            for line in lines:
                if self.getValue(line, 'G') == 1 or self.getValue(line, 'G') == 0:
                    current_z = self.getValue(line, 'Z')
                    if current_z != None:
                        if current_z >= pause_z:
                            prepend_gcode = ";TYPE:CUSTOM\n"
                            prepend_gcode += "; -- Pause at height (%.2f mm) --\n" % pause_z

                            # Insert Pause gcode
                            prepend_gcode += "M25        ; Pauses the print and waits for the user to resume it\n"

                            index = data.index(layer)
                            layer = prepend_gcode + layer
                            data[index] = layer # Override the data of this layer with the modified data
                            return data
                        break
        return data
<|endoftext|>


<commit_before>from __future__ import print_function

import json
import platform
import warnings

from importlib import import_module
from os import listdir, path
from unittest import TestCase

import six
from mock import patch

from pyinfra.api import (
    FileDownloadCommand,
    FileUploadCommand,
    FunctionCommand,
    StringCommand,
)
from pyinfra.api.util import unroll_generators
from pyinfra_cli.util import json_encode

from .util import create_host, FakeState, get_command_string, JsonTest, patch_files

PLATFORM_NAME = platform.system()

def parse_commands(commands):
    json_commands = []

    for command in commands:
        if isinstance(command, six.string_types):  # matches pyinfra/api/operation.py
            command = StringCommand(command.strip())

        if isinstance(command, StringCommand):
            json_command = get_command_string(command)

        elif isinstance(command, dict):
            command['command'] = get_command_string(command['command']).strip()
            json_command = command

        elif isinstance(command, FunctionCommand):
            func_name = (
                command.function if command.function == '__func__'
                else command.function.__name__
            )
            json_command = [
                func_name, list(command.args), command.kwargs,
            ]

        elif isinstance(command, FileUploadCommand):
            if hasattr(command.src, 'read'):
                command.src.seek(0)
                data = command.src.read()
            else:
                data = command.src
            json_command = ['upload', data, command.dest]

        elif isinstance(command, FileDownloadCommand):
            json_command = ['download', command.src, command.dest]

        else:
            raise Exception('{0} is not a valid command!'.format(command))

        if command.executor_kwargs:
            command.executor_kwargs['command'] = json_command
            json_command = command.executor_kwargs

        json_commands.append(json_command)
    return json_commands

def assert_commands(commands, wanted_commands):
    try:
        assert commands == wanted_commands
    except AssertionError as e:
        print()
        print('--> COMMANDS OUTPUT:')
        print(json.dumps(commands, indent=4, default=json_encode))

        print('--> TEST WANTS:')
        print(json.dumps(
            wanted_commands, indent=4, default=json_encode,
        ))

        raise e

def make_operation_tests(arg):
    # Get the operation we're testing against
    module_name, op_name = arg.split('.')
    module = import_module('pyinfra.operations.{0}'.format(module_name))
    op = getattr(module, op_name)

    # Generate a test class
    @patch('pyinfra.operations.files.get_timestamp', lambda: 'a-timestamp')
    @patch('pyinfra.operations.util.files.get_timestamp', lambda: 'a-timestamp')
    @six.add_metaclass(JsonTest)
    class TestTests(TestCase):
        jsontest_files = path.join('tests', 'operations', arg)
        jsontest_prefix = 'test_{0}_{1}_'.format(module_name, op_name)

        @classmethod
        def setUpClass(cls):
            # Create a global fake state that attach to pseudo state
            cls.state = FakeState()

        def jsontest_function(self, test_name, test_data):
            if (
                'require_platform' in test_data
                and PLATFORM_NAME not in test_data['require_platform']
            ):
                return

            op_test_name = '{0}/{1}.json'.format(arg, test_name)

            # Create a host with this tests facts and attach to pseudo host
            host = create_host(facts=test_data.get('facts', {}))

            allowed_exception = test_data.get('exception')

            kwargs = test_data.get('kwargs', {})
            kwargs['state'] = self.state
            kwargs['host'] = host

            with patch_files(test_data.get('local_files', {})):
                try:
                    output_commands = unroll_generators(op._pyinfra_op(
                        *test_data.get('args', []),
                        **kwargs
                    ))
                except Exception as e:
                    if allowed_exception:
                        allowed_exception_names = allowed_exception.get('names')
                        if not allowed_exception_names:
                            allowed_exception_names = [allowed_exception['name']]

                        if e.__class__.__name__ not in allowed_exception_names:
                            print('Wrong execption raised!')
                            raise

                        assert e.args[0] == allowed_exception['message']
                        return

                    raise

                op_is_idempotent = getattr(op._pyinfra_op, 'is_idempotent', True)
                test_second_output_commands = 'second_output_commands' in test_data

                if op_is_idempotent or test_second_output_commands:
                    second_output_commands = unroll_generators(op._pyinfra_op(
                        *test_data.get('args', []),
                        **kwargs
                    ))

                    if op_is_idempotent:
                        if test_data.get('idempotent', True):
                            if second_output_commands:
                                raise Exception((
                                    'Operation not idempotent, second output commands: {0}'
                                ).format(second_output_commands))
                        else:
                            if not test_data.get('disable_itempotent_warning_reason'):
                                warnings.warn((
                                    'This operation should be idempotent, but the test has disabled'
                                    ' this check without reason: {0}'
                                ).format(op_test_name))

            commands = parse_commands(output_commands)
            assert_commands(commands, test_data['commands'])

            if test_second_output_commands:
                assert_commands(
                    parse_commands(second_output_commands),
                    test_data['second_output_commands'],
                )

            noop_description = test_data.get('noop_description')
            if len(commands) == 0 or noop_description:
                if noop_description is not None:
                    assert host.noop_description == noop_description
                else:
                    assert host.noop_description is not None, 'no noop description was set'
                    warnings.warn('No noop_description set for test: {0} (got "{1}")'.format(
                        op_test_name, host.noop_description,
                    ))

    return TestTests

# Find available operation tests
operations = sorted([
    filename
    for filename in listdir(path.join('tests', 'operations'))
    if path.isdir(path.join('tests', 'operations', filename))
])

# Generate the classes, attaching to locals so nose picks them up
for operation_name in operations:
    locals()[operation_name] = make_operation_tests(operation_name)
<commit_msg>Raise an exception when ops test idempotent but the test says otherwise.<commit_after>from __future__ import print_function

import json
import platform
import warnings

from importlib import import_module
from os import listdir, path
from unittest import TestCase

import six
from mock import patch

from pyinfra.api import (
    FileDownloadCommand,
    FileUploadCommand,
    FunctionCommand,
    StringCommand,
)
from pyinfra.api.util import unroll_generators
from pyinfra_cli.util import json_encode

from .util import create_host, FakeState, get_command_string, JsonTest, patch_files

PLATFORM_NAME = platform.system()

def parse_commands(commands):
    json_commands = []

    for command in commands:
        if isinstance(command, six.string_types):  # matches pyinfra/api/operation.py
            command = StringCommand(command.strip())

        if isinstance(command, StringCommand):
            json_command = get_command_string(command)

        elif isinstance(command, dict):
            command['command'] = get_command_string(command['command']).strip()
            json_command = command

        elif isinstance(command, FunctionCommand):
            func_name = (
                command.function if command.function == '__func__'
                else command.function.__name__
            )
            json_command = [
                func_name, list(command.args), command.kwargs,
            ]

        elif isinstance(command, FileUploadCommand):
            if hasattr(command.src, 'read'):
                command.src.seek(0)
                data = command.src.read()
            else:
                data = command.src
            json_command = ['upload', data, command.dest]

        elif isinstance(command, FileDownloadCommand):
            json_command = ['download', command.src, command.dest]

        else:
            raise Exception('{0} is not a valid command!'.format(command))

        if command.executor_kwargs:
            command.executor_kwargs['command'] = json_command
            json_command = command.executor_kwargs

        json_commands.append(json_command)
    return json_commands

def assert_commands(commands, wanted_commands):
    try:
        assert commands == wanted_commands
    except AssertionError as e:
        print()
        print('--> COMMANDS OUTPUT:')
        print(json.dumps(commands, indent=4, default=json_encode))

        print('--> TEST WANTS:')
        print(json.dumps(
            wanted_commands, indent=4, default=json_encode,
        ))

        raise e

def make_operation_tests(arg):
    # Get the operation we're testing against
    module_name, op_name = arg.split('.')
    module = import_module('pyinfra.operations.{0}'.format(module_name))
    op = getattr(module, op_name)

    # Generate a test class
    @patch('pyinfra.operations.files.get_timestamp', lambda: 'a-timestamp')
    @patch('pyinfra.operations.util.files.get_timestamp', lambda: 'a-timestamp')
    @six.add_metaclass(JsonTest)
    class TestTests(TestCase):
        jsontest_files = path.join('tests', 'operations', arg)
        jsontest_prefix = 'test_{0}_{1}_'.format(module_name, op_name)

        @classmethod
        def setUpClass(cls):
            # Create a global fake state that attach to pseudo state
            cls.state = FakeState()

        def jsontest_function(self, test_name, test_data):
            if (
                'require_platform' in test_data
                and PLATFORM_NAME not in test_data['require_platform']
            ):
                return

            op_test_name = '{0}/{1}.json'.format(arg, test_name)

            # Create a host with this tests facts and attach to pseudo host
            host = create_host(facts=test_data.get('facts', {}))

            allowed_exception = test_data.get('exception')

            kwargs = test_data.get('kwargs', {})
            kwargs['state'] = self.state
            kwargs['host'] = host

            with patch_files(test_data.get('local_files', {})):
                try:
                    output_commands = unroll_generators(op._pyinfra_op(
                        *test_data.get('args', []),
                        **kwargs
                    ))
                except Exception as e:
                    if allowed_exception:
                        allowed_exception_names = allowed_exception.get('names')
                        if not allowed_exception_names:
                            allowed_exception_names = [allowed_exception['name']]

                        if e.__class__.__name__ not in allowed_exception_names:
                            print('Wrong execption raised!')
                            raise

                        assert e.args[0] == allowed_exception['message']
                        return

                    raise

                op_is_idempotent = getattr(op._pyinfra_op, 'is_idempotent', True)
                test_second_output_commands = 'second_output_commands' in test_data

                if op_is_idempotent or test_second_output_commands:
                    second_output_commands = unroll_generators(op._pyinfra_op(
                        *test_data.get('args', []),
                        **kwargs
                    ))

                    if op_is_idempotent:
                        if test_data.get('idempotent', True):
                            if second_output_commands:
                                raise Exception((
                                    'Operation not idempotent, second output commands: {0}'
                                ).format(second_output_commands))
                        else:
                            if not second_output_commands:
                                raise Exception((
                                    'Operation tests as idempotent but test '
                                    'says it is not: {0}'
                                ).format(op_test_name))
                            if not test_data.get('disable_idempotent_warning_reason'):
                                warnings.warn((
                                    'This operation should be idempotent, but the test has disabled'
                                    ' this check without reason: {0}'
                                ).format(op_test_name))

            commands = parse_commands(output_commands)
            assert_commands(commands, test_data['commands'])

            if test_second_output_commands:
                assert_commands(
                    parse_commands(second_output_commands),
                    test_data['second_output_commands'],
                )

            noop_description = test_data.get('noop_description')
            if len(commands) == 0 or noop_description:
                if noop_description is not None:
                    assert host.noop_description == noop_description
                else:
                    assert host.noop_description is not None, 'no noop description was set'
                    warnings.warn('No noop_description set for test: {0} (got "{1}")'.format(
                        op_test_name, host.noop_description,
                    ))

    return TestTests

# Find available operation tests
operations = sorted([
    filename
    for filename in listdir(path.join('tests', 'operations'))
    if path.isdir(path.join('tests', 'operations', filename))
])

# Generate the classes, attaching to locals so nose picks them up
for operation_name in operations:
    locals()[operation_name] = make_operation_tests(operation_name)
<|endoftext|>


<commit_before># -*- coding: utf8 -*-
# This file is part of PyBossa.
#
# Copyright (C) 2014 SF Isle of Man Limited
#
# PyBossa is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# PyBossa is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with PyBossa.  If not, see <http://www.gnu.org/licenses/>.
"""This module tests the Uploader class."""

from default import Test
from pybossa.uploader.rackspace import RackspaceUploader
from mock import patch, PropertyMock, call, MagicMock
from werkzeug.datastructures import FileStorage
from pyrax.fakes import FakeContainer
from pyrax.exceptions import NoSuchObject, NoSuchContainer
from test_uploader import cloudfiles_mock, fake_container

class TestRackspaceUploader(Test):

    """Test PyBossa Rackspace Uploader module."""

    @patch('pybossa.uploader.rackspace.pyrax.set_credentials',
           return_value=True)
    def test_rackspace_uploader_init(self, Mock):
        """Test RACKSPACE UPLOADER init works."""
        new_extensions = ['pdf', 'doe']
        with patch('pybossa.uploader.rackspace.pyrax.cloudfiles',
                   return_value=cloudfiles_mock):
            with patch.dict(self.flask_app.config,
                            {'ALLOWED_EXTENSIONS': new_extensions}):

                with patch('pybossa.uploader.rackspace.pyrax.cloudfiles') as mycf:
                    mycf.get_container.return_value = True
                    u = RackspaceUploader()
                    res = u.init_app(self.flask_app, cont_name='mycontainer')
                    err_msg = "It should return the container."
                    assert res is True, err_msg
                    err_msg = "The container name should be updated."
                    assert u.cont_name == 'mycontainer', err_msg
                    for ext in new_extensions:
                        err_msg = "The .%s extension should be allowed" % ext
                        assert ext in u.allowed_extensions, err_msg

    @patch('pybossa.uploader.rackspace.pyrax.set_credentials',
           return_value=True)
    @patch('pybossa.uploader.rackspace.pyrax.utils.get_checksum',
           return_value="1234abcd")
    def test_rackspace_uploader_creates_container(self, mock, mock2):
        """Test RACKSPACE UPLOADER creates container works."""
        with patch('pybossa.uploader.rackspace.pyrax.cloudfiles') as mycf:
            mycf.get_container.side_effect = NoSuchContainer
            mycf.create_container.return_value = True
            mycf.make_container_public.return_value = True
            u = RackspaceUploader()
            res = u.init_app(self.flask_app)
            err_msg = "Init app should return the container."
            assert res is True, err_msg

    @patch('pybossa.uploader.rackspace.pyrax.set_credentials',
           return_value=True)
    @patch('pybossa.uploader.rackspace.pyrax.utils.get_checksum',
           return_value="1234abcd")
    def test_rackspace_uploader_upload_correct_file(self, mock, mock2):
        """Test RACKSPACE UPLOADER upload file works."""
        with patch('pybossa.uploader.rackspace.pyrax.cloudfiles') as mycf:
            mycf.upload_file.return_value=True
            mycf.get_object.side_effect = NoSuchObject
            u = RackspaceUploader()
            u.init_app(self.flask_app)
            file = FileStorage(filename='test.jpg')
            err_msg = "Upload file should return True"
            assert u.upload_file(file, container='user_3') is True, err_msg
            calls = [call.get_container('user_3'),
                     call.get_container().get_object('test.jpg')]
            mycf.assert_has_calls(calls, any_order=True)

    @patch('pybossa.uploader.rackspace.pyrax.set_credentials',
           return_value=True)
    @patch('pybossa.uploader.rackspace.pyrax.utils.get_checksum',
           return_value="1234abcd")
    def test_rackspace_uploader_upload_correct_purgin_first_file(self, mock, mock2):
        """Test RACKSPACE UPLOADER upload file purging first file works."""
        with patch('pybossa.uploader.rackspace.pyrax.cloudfiles') as mycf:
            mycf.upload_file.return_value=True
            mycf.get_object.side_effect = True
            u = RackspaceUploader()
            u.init_app(self.flask_app)
            file = FileStorage(filename='test.jpg')
            err_msg = "Upload file should return True"
            assert u.upload_file(file, container='user_3') is True, err_msg
            calls = [call.get_container('user_3'),
                     call.get_container().get_object().delete(),
                     call.get_container().get_object('test.jpg')]
            print mycf.mock_calls
            mycf.assert_has_calls(calls, any_order=True)

    @patch('pybossa.uploader.rackspace.pyrax.set_credentials',
           return_value=True)
    @patch('pybossa.uploader.rackspace.pyrax.utils.get_checksum',
           return_value="1234abcd")
    def test_rackspace_uploader_upload_file_fails(self, mock, mock2):
        """Test RACKSPACE UPLOADER upload file fail works."""
        with patch('pybossa.uploader.rackspace.pyrax.cloudfiles') as mycf:
            from pyrax.exceptions import UploadFailed
            mycf.upload_file.side_effect = UploadFailed
            u = RackspaceUploader()
            u.init_app(self.flask_app)
            file = FileStorage(filename='test.jpg')
            err_msg = "Upload file should return False"
            assert u.upload_file(file, container='user_3') is False, err_msg

    @patch('pybossa.uploader.rackspace.pyrax.set_credentials',
           return_value=True)
    @patch('pybossa.uploader.rackspace.pyrax.utils.get_checksum',
           return_value="1234abcd")
    def test_rackspace_uploader_upload_wrong_file(self, mock, mock2):
        """Test RACKSPACE UPLOADER upload wrong file extension works."""
        with patch('pybossa.uploader.rackspace.pyrax.cloudfiles') as mycf:
            mycf.upload_file.return_value = True
            u = RackspaceUploader()
            u.init_app(self.flask_app)
            file = FileStorage(filename='test.docs')
            err_msg = "Upload file should return False"
            res = u.upload_file(file, container='user_3')
            assert res is False, err_msg

    @patch('pybossa.uploader.rackspace.pyrax.set_credentials',
           return_value=True)
    def test_rackspace_uploader_lookup_url(self, mock1):
        """Test RACKSPACE UPLOADER lookup returns a valid link."""
        uri = 'http://rackspace.com'
        filename = 'test.jpg'
        with patch('pybossa.uploader.rackspace.pyrax.cloudfiles') as mycf:
            cdn_enabled_mock = PropertyMock(return_value=True)
            type(fake_container).cdn_enabled = cdn_enabled_mock
            mycf.get_container.return_value = fake_container

            u = RackspaceUploader()
            u.init_app(self.flask_app)
            res = u._lookup_url('rackspace', {'filename': filename,
                                              'container': 'user_3'})
            expected_url = "%s/%s" % (uri, filename)
            print res
            err_msg = "We should get the following URL: %s" % expected_url
            assert res == expected_url, err_msg

    @patch('pybossa.uploader.rackspace.pyrax.set_credentials',
           return_value=True)
    def test_rackspace_uploader_lookup_url_none(self, mock1):
        """Test RACKSPACE UPLOADER lookup returns None for non enabled CDN."""
        filename = 'test.jpg'
        with patch('pybossa.uploader.rackspace.pyrax.cloudfiles') as mycf:
            cdn_enabled_mock = PropertyMock(return_value=False)
            type(fake_container).cdn_enabled = cdn_enabled_mock
            mycf.get_container.return_value = fake_container

            u = RackspaceUploader()
            u.init_app(self.flask_app)
            res = u._lookup_url('rackspace', {'filename': filename,
                                              'container': 'user_3'})
            err_msg = "We should get the None"
            assert res is None, err_msg

    @patch('pybossa.uploader.rackspace.pyrax.set_credentials',
           return_value=True)
    def test_rackspace_uploader_get_container(self, mock1):
        """Test RACKSPACE UPLOADER get_container method works."""
        with patch('pybossa.uploader.rackspace.pyrax.cloudfiles') as mycf:
            cdn_enabled_mock = PropertyMock(return_value=False)
            type(fake_container).cdn_enabled = cdn_enabled_mock
            mycf.get_container.side_effect = NoSuchContainer

            calls = [call.get_container('user_3'),
                     call.create_container('user_3'),
                     call.make_container_public('user_3')
                     ]
            u = RackspaceUploader()
            u.init_app(self.flask_app)
            assert u.get_container('user_3')
            mycf.assert_has_calls(calls, any_order=True)

    @patch('pybossa.uploader.rackspace.pyrax.set_credentials',
           return_value=True)
    def test_rackspace_uploader_delete(self, mock1):
        """Test RACKSPACE UPLOADER delete method works."""
        with patch('pybossa.uploader.rackspace.pyrax.cloudfiles') as mycf:
            #cdn_enabled_mock = PropertyMock(return_value=False)
            #type(fake_container).cdn_enabled = cdn_enabled_mock
            #mycf.get_container.side_effect = NoSuchContainer

            calls = [call.get_container('container'),
                     call.get_container().get_object('file'),
                     call.get_container().get_object().delete()
                     ]
            u = RackspaceUploader()
            u.init_app(self.flask_app)
            err_msg = "It should return True"
            assert u.delete_file('file', 'container') is True, err_msg
            mycf.assert_has_calls(calls, any_order=True)

    @patch('pybossa.uploader.rackspace.pyrax.set_credentials',
           return_value=True)
    def test_rackspace_uploader_delete_fails(self, mock1):
        """Test RACKSPACE UPLOADER delete fails method works."""
        with patch('pybossa.uploader.rackspace.pyrax.cloudfiles') as mycf:
            container = MagicMock()
            #container.get_object.return_value = 'file'
            container.get_object.side_effect = NoSuchObject
            mycf.get_container.return_value = container

            calls = [call.get_container('container'),
                     ]
            u = RackspaceUploader()
            u.init_app(self.flask_app)
            err_msg = "It should return False"
            assert u.delete_file('file', 'container') is False, err_msg
            mycf.assert_has_calls(calls, any_order=True)
<commit_msg>Minor improvements<commit_after># -*- coding: utf8 -*-
# This file is part of PyBossa.
#
# Copyright (C) 2014 SF Isle of Man Limited
#
# PyBossa is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# PyBossa is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with PyBossa.  If not, see <http://www.gnu.org/licenses/>.
"""This module tests the Uploader class."""

from default import Test
from pybossa.uploader.rackspace import RackspaceUploader
from mock import patch, PropertyMock, call, MagicMock
from werkzeug.datastructures import FileStorage
from pyrax.fakes import FakeContainer
from pyrax.exceptions import NoSuchObject, NoSuchContainer
from test_uploader import cloudfiles_mock, fake_container

class TestRackspaceUploader(Test):

    """Test PyBossa Rackspace Uploader module."""

    @patch('pybossa.uploader.rackspace.pyrax.set_credentials',
           return_value=True)
    def test_rackspace_uploader_init(self, Mock):
        """Test RACKSPACE UPLOADER init works."""
        new_extensions = ['pdf', 'doe']
        with patch('pybossa.uploader.rackspace.pyrax.cloudfiles',
                   return_value=cloudfiles_mock):
            with patch.dict(self.flask_app.config,
                            {'ALLOWED_EXTENSIONS': new_extensions}):

                with patch('pybossa.uploader.rackspace.pyrax.cloudfiles') as mycf:
                    mycf.get_container.return_value = True
                    u = RackspaceUploader()
                    res = u.init_app(self.flask_app, cont_name='mycontainer')
                    err_msg = "It should return the container."
                    assert res is True, err_msg
                    err_msg = "The container name should be updated."
                    assert u.cont_name == 'mycontainer', err_msg
                    for ext in new_extensions:
                        err_msg = "The .%s extension should be allowed" % ext
                        assert ext in u.allowed_extensions, err_msg

    @patch('pybossa.uploader.rackspace.pyrax.set_credentials',
           return_value=True)
    @patch('pybossa.uploader.rackspace.pyrax.utils.get_checksum',
           return_value="1234abcd")
    def test_rackspace_uploader_creates_container(self, mock, mock2):
        """Test RACKSPACE UPLOADER creates container works."""
        with patch('pybossa.uploader.rackspace.pyrax.cloudfiles') as mycf:
            mycf.get_container.side_effect = NoSuchContainer
            mycf.create_container.return_value = True
            mycf.make_container_public.return_value = True
            u = RackspaceUploader()
            res = u.init_app(self.flask_app)
            err_msg = "Init app should return the container."
            assert res is True, err_msg

    @patch('pybossa.uploader.rackspace.pyrax.set_credentials',
           return_value=True)
    @patch('pybossa.uploader.rackspace.pyrax.utils.get_checksum',
           return_value="1234abcd")
    def test_rackspace_uploader_upload_correct_file(self, mock, mock2):
        """Test RACKSPACE UPLOADER upload file works."""
        with patch('pybossa.uploader.rackspace.pyrax.cloudfiles') as mycf:
            mycf.upload_file.return_value=True
            mycf.get_object.side_effect = NoSuchObject
            u = RackspaceUploader()
            u.init_app(self.flask_app)
            file = FileStorage(filename='test.jpg')
            err_msg = "Upload file should return True"
            assert u.upload_file(file, container='user_3') is True, err_msg
            calls = [call.get_container('user_3'),
                     call.get_container().get_object('test.jpg')]
            mycf.assert_has_calls(calls, any_order=True)

    @patch('pybossa.uploader.rackspace.pyrax.set_credentials',
           return_value=True)
    @patch('pybossa.uploader.rackspace.pyrax.utils.get_checksum',
           return_value="1234abcd")
    def test_rackspace_uploader_upload_correct_purgin_first_file(self, mock, mock2):
        """Test RACKSPACE UPLOADER upload file purging first file works."""
        with patch('pybossa.uploader.rackspace.pyrax.cloudfiles') as mycf:
            mycf.upload_file.return_value=True
            mycf.get_object.side_effect = True
            u = RackspaceUploader()
            u.init_app(self.flask_app)
            file = FileStorage(filename='test.jpg')
            err_msg = "Upload file should return True"
            assert u.upload_file(file, container='user_3') is True, err_msg
            calls = [call.get_container('user_3'),
                     call.get_container().get_object().delete(),
                     call.get_container().get_object('test.jpg')]
            print mycf.mock_calls
            mycf.assert_has_calls(calls, any_order=True)

    @patch('pybossa.uploader.rackspace.pyrax.set_credentials',
           return_value=True)
    @patch('pybossa.uploader.rackspace.pyrax.utils.get_checksum',
           return_value="1234abcd")
    def test_rackspace_uploader_upload_file_fails(self, mock, mock2):
        """Test RACKSPACE UPLOADER upload file fail works."""
        with patch('pybossa.uploader.rackspace.pyrax.cloudfiles') as mycf:
            from pyrax.exceptions import UploadFailed
            mycf.upload_file.side_effect = UploadFailed
            u = RackspaceUploader()
            u.init_app(self.flask_app)
            file = FileStorage(filename='test.jpg')
            err_msg = "Upload file should return False"
            assert u.upload_file(file, container='user_3') is False, err_msg

    @patch('pybossa.uploader.rackspace.pyrax.set_credentials',
           return_value=True)
    @patch('pybossa.uploader.rackspace.pyrax.utils.get_checksum',
           return_value="1234abcd")
    def test_rackspace_uploader_upload_wrong_file(self, mock, mock2):
        """Test RACKSPACE UPLOADER upload wrong file extension works."""
        with patch('pybossa.uploader.rackspace.pyrax.cloudfiles') as mycf:
            mycf.upload_file.return_value = True
            u = RackspaceUploader()
            u.init_app(self.flask_app)
            file = FileStorage(filename='test.docs')
            err_msg = "Upload file should return False"
            res = u.upload_file(file, container='user_3')
            assert res is False, err_msg

    @patch('pybossa.uploader.rackspace.pyrax.set_credentials',
           return_value=True)
    def test_rackspace_uploader_lookup_url(self, mock1):
        """Test RACKSPACE UPLOADER lookup returns a valid link."""
        uri = 'http://rackspace.com'
        filename = 'test.jpg'
        with patch('pybossa.uploader.rackspace.pyrax.cloudfiles') as mycf:
            cdn_enabled_mock = PropertyMock(return_value=True)
            type(fake_container).cdn_enabled = cdn_enabled_mock
            mycf.get_container.return_value = fake_container

            u = RackspaceUploader()
            u.init_app(self.flask_app)
            res = u._lookup_url('rackspace', {'filename': filename,
                                              'container': 'user_3'})
            expected_url = "%s/%s" % (uri, filename)
            print res
            err_msg = "We should get the following URL: %s" % expected_url
            assert res == expected_url, err_msg

    @patch('pybossa.uploader.rackspace.pyrax.set_credentials',
           return_value=True)
    def test_rackspace_uploader_lookup_url_none(self, mock1):
        """Test RACKSPACE UPLOADER lookup returns None for non enabled CDN."""
        filename = 'test.jpg'
        with patch('pybossa.uploader.rackspace.pyrax.cloudfiles') as mycf:
            cdn_enabled_mock = PropertyMock(return_value=False)
            type(fake_container).cdn_enabled = cdn_enabled_mock
            mycf.get_container.return_value = fake_container

            u = RackspaceUploader()
            u.init_app(self.flask_app)
            res = u._lookup_url('rackspace', {'filename': filename,
                                              'container': 'user_3'})
            err_msg = "We should get the None"
            assert res is None, err_msg

    @patch('pybossa.uploader.rackspace.pyrax.set_credentials',
           return_value=True)
    def test_rackspace_uploader_get_container(self, mock1):
        """Test RACKSPACE UPLOADER get_container method works."""
        with patch('pybossa.uploader.rackspace.pyrax.cloudfiles') as mycf:
            cdn_enabled_mock = PropertyMock(return_value=False)
            type(fake_container).cdn_enabled = cdn_enabled_mock
            mycf.get_container.side_effect = NoSuchContainer

            calls = [call.get_container('user_3'),
                     call.create_container('user_3'),
                     call.make_container_public('user_3')
                     ]
            u = RackspaceUploader()
            u.init_app(self.flask_app)
            assert u.get_container('user_3')
            mycf.assert_has_calls(calls, any_order=True)

    @patch('pybossa.uploader.rackspace.pyrax.set_credentials',
           return_value=True)
    def test_rackspace_uploader_delete(self, mock1):
        """Test RACKSPACE UPLOADER delete method works."""
        with patch('pybossa.uploader.rackspace.pyrax.cloudfiles') as mycf:
            #cdn_enabled_mock = PropertyMock(return_value=False)
            #type(fake_container).cdn_enabled = cdn_enabled_mock
            #mycf.get_container.side_effect = NoSuchContainer

            calls = [call.get_container('container'),
                     call.get_container().get_object('file'),
                     call.get_container().get_object().delete()
                     ]
            u = RackspaceUploader()
            u.init_app(self.flask_app)
            err_msg = "It should return True"
            assert u.delete_file('file', 'container') is True, err_msg
            mycf.assert_has_calls(calls, any_order=True)

    @patch('pybossa.uploader.rackspace.pyrax.set_credentials',
           return_value=True)
    def test_rackspace_uploader_delete_fails(self, mock1):
        """Test RACKSPACE UPLOADER delete fails method works."""
        with patch('pybossa.uploader.rackspace.pyrax.cloudfiles') as mycf:
            container = MagicMock()
            container.get_object.side_effect = NoSuchObject
            mycf.get_container.return_value = container

            calls = [call.get_container('container'),
                     ]
            u = RackspaceUploader()
            u.init_app(self.flask_app)
            err_msg = "It should return False"
            assert u.delete_file('file', 'container') is False, err_msg
            mycf.assert_has_calls(calls, any_order=True)
<|endoftext|>


<commit_before># Copyright 2017 the GPflow authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import numpy as np
import tensorflow as tf
import pytest

import gpflow
from gpflow.test_util import session_tf
from gpflow.test_util import GPflowTestCase
from gpflow.training.natgrad_optimizer import NatGradOptimizer
from gpflow.training import GradientDescentOptimizer
from gpflow.training.optimizer import Optimizer

from numpy.testing import assert_allclose

class Empty(gpflow.models.Model):
    @gpflow.params_as_tensors
    def _build_likelihood(self):
        return tf.constant(0.0, dtype=gpflow.settings.float_type)

class Demo(gpflow.models.Model):
    def __init__(self, add_to_inits=[], add_to_trainables=[], name=None):
        super().__init__(name=name)
        data = np.random.randn(10, 10)
        self.a = gpflow.Param(data, dtype=gpflow.settings.float_type)
        self.init_vars = add_to_inits
        self.trainable_vars = add_to_trainables

    @property
    def initializables(self):
        return super().initializables + self.init_vars

    @property
    def trainable_tensors(self):
        return super().trainable_tensors + self.trainable_vars

    @gpflow.params_as_tensors
    def _build_likelihood(self):
        return tf.reduce_sum(self.a) + sum(map(tf.reduce_prod, self.trainable_vars))

class NonOptimizer(gpflow.training.tensorflow_optimizer._TensorFlowOptimizer):
    pass

class TestSimpleOptimizerInterface(GPflowTestCase):
    def test_non_existing_optimizer(self):
        with self.assertRaises(TypeError):
            _ = NonOptimizer()

class OptimizerCase:
    optimizer = None

    # pylint: disable=E1101,E1102

    def test_different_sessions(self):
        with self.test_context() as session:
            demo = Demo()

        # Force initialization.
        with self.test_context() as session1:
            gpflow.reset_default_session()
            opt = self.optimizer()
            demo.initialize(session1, force=True)
            opt.minimize(demo, maxiter=1)

        # Mild initialization requirement: default changed session case.
        with self.test_context() as session2:
            self.assertFalse(session1 == session2)
            gpflow.reset_default_session()
            opt = self.optimizer()
            opt.minimize(demo, maxiter=1, initialize=True)

        # Mild initialization requirement: pass session case.
        with self.test_context() as session3:
            opt = self.optimizer()
            opt.minimize(demo, maxiter=1, session=session3, initialize=True)

    def test_optimizer_with_var_list(self):
        with self.test_context():
            demo = Demo()
            dfloat = gpflow.settings.float_type
            var1 = tf.get_variable('var_a1', shape=(), dtype=dfloat)
            var2 = tf.get_variable('var_b2', shape=(), trainable=False, dtype=dfloat)
            var3 = tf.get_variable('var_c3', shape=(), dtype=dfloat)

        with self.test_context() as session:
            opt = self.optimizer()
            demo.initialize(session)

            # No var list variables and empty feed_dict
            opt.minimize(demo, maxiter=1, initialize=False, anchor=False)

            # Var list is empty
            with self.assertRaises(ValueError):
                opt.minimize(Empty(), var_list=[], maxiter=1)

            # Var list variable
            session.run(var1.initializer)
            placeholder = tf.placeholder(gpflow.settings.float_type)
            opt.minimize(demo,
                         var_list=[var1],
                         feed_dict={placeholder: [1., 2]},
                         maxiter=5, initialize=False, anchor=False)

            # Var list variable is not trainable
            session.run(var2.initializer)
            opt.minimize(demo, var_list=[var2], maxiter=1, initialize=False)

            # NOTE(@awav): TensorFlow optimizer skips uninitialized values which
            # are not present in objective.
            demo._objective += var3
            with self.assertRaises(tf.errors.FailedPreconditionError):
                opt.minimize(demo, session=session, var_list=[var3], maxiter=1,
                             initialize=False, anchor=False)

    def test_optimizer_tensors(self):
        with self.test_context():
            opt = self.optimizer()
            if not isinstance(opt, gpflow.train.ScipyOptimizer):
                demo = Demo()
                opt.minimize(demo, maxiter=0)
                self.assertEqual(opt.model, demo)

                opt.model = Demo()
                self.assertNotEqual(opt.model, demo)
                self.assertEqual(opt.minimize_operation, None)
                self.assertEqual(opt.optimizer, None)

    def test_non_gpflow_model(self):
        with self.test_context():
            opt = self.optimizer()
            with self.assertRaises(ValueError):
                opt.minimize(None, maxiter=0)

    def test_external_variables_in_model(self):
        with self.test_context():
            dfloat = gpflow.settings.float_type

            var1_init = np.array(1.0, dtype=dfloat)
            var2_init = np.array(2.0, dtype=dfloat)
            var3_init = np.array(3.0, dtype=dfloat)

            var1 = tf.get_variable('var1', initializer=var1_init, dtype=dfloat)
            var2 = tf.get_variable('var2', initializer=var2_init, dtype=dfloat)
            var3 = tf.get_variable('var3', initializer=var3_init, trainable=False, dtype=dfloat)

            opt = self.optimizer()

        # Just initialize variable, but do not use it in training
        with self.test_context() as session:
            gpflow.reset_default_session()
            demo1 = Demo(add_to_inits=[var1], add_to_trainables=[], name="demo1")
            opt.minimize(demo1, maxiter=10)
            self.assertEqual(session.run(var1), var1_init)

        with self.test_context() as session:
            gpflow.reset_default_session()
            with self.assertRaises(tf.errors.FailedPreconditionError):
                demo2 = Demo(add_to_inits=[], add_to_trainables=[var2], name="demo2")
                opt.minimize(demo2, maxiter=10)

        with self.test_context() as session:
            gpflow.reset_default_session()
            demo3 = Demo(add_to_inits=[var1, var2, var3],
                         add_to_trainables=[var2, var3], name="demo3")
            opt.minimize(demo3, maxiter=10)

            self.assertEqual(session.run(var1), var1_init)
            self.assertFalse(session.run(var3) == var3_init)
            self.assertFalse(session.run(var2) == var2_init)

class TestScipyOptimizer(GPflowTestCase, OptimizerCase):
    optimizer = gpflow.train.ScipyOptimizer

class TestGradientDescentOptimizer(GPflowTestCase, OptimizerCase):
    optimizer = lambda _self: gpflow.train.GradientDescentOptimizer(0.1)

class TestAdamOptimizer(GPflowTestCase, OptimizerCase):
    optimizer = lambda _self: gpflow.train.AdamOptimizer(0.1)

class TestMomentumOptimizer(GPflowTestCase, OptimizerCase):
    optimizer = lambda _self: gpflow.train.MomentumOptimizer(0.1, 0.9)

class TestAdadeltaOptimizer(GPflowTestCase, OptimizerCase):
    optimizer = lambda _self: gpflow.train.AdadeltaOptimizer(0.1)

class TestRMSPropOptimizer(GPflowTestCase, OptimizerCase):
    optimizer = lambda _self: gpflow.train.RMSPropOptimizer(0.1)

class TestFtrlOptimizer(GPflowTestCase, OptimizerCase):
    optimizer = lambda _self: gpflow.train.FtrlOptimizer(0.1)

# class TestProximalAdagradOptimizer(GPflowTestCase, OptimizerCase):
#     optimizer = lambda _self: gpflow.train.ProximalAdagradOptimizer(0.1)
#
#                 raise TypeError(
#                     "%s type %s of argument '%s'." %
#                     (prefix, dtypes.as_dtype(attrs[input_arg.type_attr]).name,
# >                    inferred_from[input_arg.type_attr]))
# E               TypeError: Input 'lr' of 'ApplyProximalAdagrad' Op has type float32 that does not match type float64 of argument 'var'.
#
# ../../anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:546: TypeError

def test_VGP_vs_GPR(session_tf):
    N, D = 3, 2
    X = np.random.randn(N, D)
    Y = np.random.randn(N, 1)
    kern = gpflow.kernels.RBF(D)
    lik_var = 0.1
    lik = gpflow.likelihoods.Gaussian()
    lik.variance = lik_var

    m_vgp = gpflow.models.VGP(X, Y, kern, lik)
    m_gpr = gpflow.models.GPR(X, Y, kern)
    m_gpr.likelihood.variance = lik_var

    m_vgp.set_trainable(False)
    m_vgp.q_mu.set_trainable(True)
    m_vgp.q_sqrt.set_trainable(True)
    NatGradOptimizer(1.).minimize(m_vgp, [[m_vgp.q_mu, m_vgp.q_sqrt]], maxiter=1)

    assert_allclose(m_gpr.compute_log_likelihood(),
                    m_vgp.compute_log_likelihood(), atol=1e-5)

def test_SVGP_vs_SGPR(session_tf):
    N, M, D = 4, 3, 2
    X = np.random.randn(N, D)
    Z = np.random.randn(M, D)
    Y = np.random.randn(N, 1)
    kern = gpflow.kernels.RBF(D)
    lik_var = 0.1
    lik = gpflow.likelihoods.Gaussian()
    lik.variance = lik_var

    m_svgp = gpflow.models.SVGP(X, Y, kern, lik, Z=Z)
    m_sgpr = gpflow.models.SGPR(X, Y, kern, Z=Z)
    m_sgpr.likelihood.variance = lik_var

    m_svgp.set_trainable(False)
    m_svgp.q_mu.set_trainable(True)
    m_svgp.q_sqrt.set_trainable(True)
    NatGradOptimizer(1.).minimize(m_svgp, [[m_svgp.q_mu, m_svgp.q_sqrt]], maxiter=1)

    assert_allclose(m_sgpr.compute_log_likelihood(),
                    m_svgp.compute_log_likelihood(), atol=1e-5)

class CombinationOptimizer(Optimizer):
    """
    A class that applies one step of each of multiple optimizers in a loop
    """
    def __init__(self, optimizers_with_kwargs):
        self.name = None
        super().__init__()
        self.optimizers_with_kwargs = optimizers_with_kwargs

    def minimize(self, model, session=None, maxiter=1000, feed_dict=None, anchor=True):
        session = model.enquire_session(session)

        minimize_ops = []
        for optimizer, kwargs in self.optimizers_with_kwargs:
            # hack to init the optimizer operation
            optimizer.minimize(model, maxiter=0, **kwargs)

            # this is what we will call in the loop
            minimize_ops.append(optimizer.minimize_operation)

        with session.graph.as_default(), tf.name_scope(self.name):
            feed_dict = self._gen_feed_dict(model, feed_dict)

            for _i in range(maxiter):
                for op in minimize_ops:
                    session.run(op, feed_dict=feed_dict)

        if anchor:
            model.anchor(session)

# TODO(@hugh) this needs to be fixed
def test_hypers_SVGP_vs_SGPR(session_tf):
    N, M, D = 4, 3, 2
    X = np.random.randn(N, D)
    Z = np.random.randn(M, D)
    Y = np.random.randn(N, 1)
    kern = gpflow.kernels.RBF(D)
    lik_var = 0.1
    lik = gpflow.likelihoods.Gaussian()
    lik.variance = lik_var

    m_svgp = gpflow.models.SVGP(X, Y, kern, lik, Z=Z)
    m_sgpr = gpflow.models.SGPR(X, Y, kern, Z=Z)
    m_sgpr.likelihood.variance = lik_var

    m_svgp.q_mu.set_trainable(False)
    m_svgp.q_sqrt.set_trainable(False)

    NatGradOptimizer(1.).minimize(m_svgp, [[m_svgp.q_mu, m_svgp.q_sqrt]], maxiter=1)

    # this is fine
    assert_allclose(m_sgpr.compute_log_likelihood(),
                        m_svgp.compute_log_likelihood(), atol=1e-5)

    # combination (doing GD first as we've already done the nat grad step
    o1 = [GradientDescentOptimizer(0.01), {}]
    o2 = [NatGradOptimizer(1.), {'var_list':[[m_svgp.q_mu, m_svgp.q_sqrt]]}]
    nag_grads_with_gd_optimizer = CombinationOptimizer([o1, o2])
    nag_grads_with_gd_optimizer.minimize(m_svgp,  maxiter=1)

    # this should be the same as
    GradientDescentOptimizer(0.01).minimize(m_svgp, maxiter=1)

    # but isn't...
    with pytest.raises(AssertionError):
        assert_allclose(m_sgpr.compute_log_likelihood(),
                        m_svgp.compute_log_likelihood(), atol=1e-5)
    # the trouble seems to be that the [q_mu, q_sqrt] haven't updated as far as o1 is concerned
<commit_msg>added test with excluded vars<commit_after># Copyright 2017 the GPflow authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import numpy as np
import tensorflow as tf
import pytest

import gpflow
from gpflow.test_util import session_tf
from gpflow.test_util import GPflowTestCase
from gpflow.training.natgrad_optimizer import NatGradOptimizer
from gpflow.training import GradientDescentOptimizer
from gpflow.training.optimizer import Optimizer
from gpflow.training.tensorflow_optimizer import _TensorFlowOptimizer

from numpy.testing import assert_allclose

class Empty(gpflow.models.Model):
    @gpflow.params_as_tensors
    def _build_likelihood(self):
        return tf.constant(0.0, dtype=gpflow.settings.float_type)

class Demo(gpflow.models.Model):
    def __init__(self, add_to_inits=[], add_to_trainables=[], name=None):
        super().__init__(name=name)
        data = np.random.randn(10, 10)
        self.a = gpflow.Param(data, dtype=gpflow.settings.float_type)
        self.init_vars = add_to_inits
        self.trainable_vars = add_to_trainables

    @property
    def initializables(self):
        return super().initializables + self.init_vars

    @property
    def trainable_tensors(self):
        return super().trainable_tensors + self.trainable_vars

    @gpflow.params_as_tensors
    def _build_likelihood(self):
        return tf.reduce_sum(self.a) + sum(map(tf.reduce_prod, self.trainable_vars))

class NonOptimizer(gpflow.training.tensorflow_optimizer._TensorFlowOptimizer):
    pass

class TestSimpleOptimizerInterface(GPflowTestCase):
    def test_non_existing_optimizer(self):
        with self.assertRaises(TypeError):
            _ = NonOptimizer()

class OptimizerCase:
    optimizer = None

    # pylint: disable=E1101,E1102

    def test_different_sessions(self):
        with self.test_context() as session:
            demo = Demo()

        # Force initialization.
        with self.test_context() as session1:
            gpflow.reset_default_session()
            opt = self.optimizer()
            demo.initialize(session1, force=True)
            opt.minimize(demo, maxiter=1)

        # Mild initialization requirement: default changed session case.
        with self.test_context() as session2:
            self.assertFalse(session1 == session2)
            gpflow.reset_default_session()
            opt = self.optimizer()
            opt.minimize(demo, maxiter=1, initialize=True)

        # Mild initialization requirement: pass session case.
        with self.test_context() as session3:
            opt = self.optimizer()
            opt.minimize(demo, maxiter=1, session=session3, initialize=True)

    def test_optimizer_with_var_list(self):
        with self.test_context():
            demo = Demo()
            dfloat = gpflow.settings.float_type
            var1 = tf.get_variable('var_a1', shape=(), dtype=dfloat)
            var2 = tf.get_variable('var_b2', shape=(), trainable=False, dtype=dfloat)
            var3 = tf.get_variable('var_c3', shape=(), dtype=dfloat)

        with self.test_context() as session:
            opt = self.optimizer()
            demo.initialize(session)

            # No var list variables and empty feed_dict
            opt.minimize(demo, maxiter=1, initialize=False, anchor=False)

            # Var list is empty
            with self.assertRaises(ValueError):
                opt.minimize(Empty(), var_list=[], maxiter=1)

            # Var list variable
            session.run(var1.initializer)
            placeholder = tf.placeholder(gpflow.settings.float_type)
            opt.minimize(demo,
                         var_list=[var1],
                         feed_dict={placeholder: [1., 2]},
                         maxiter=5, initialize=False, anchor=False)

            # Var list variable is not trainable
            session.run(var2.initializer)
            opt.minimize(demo, var_list=[var2], maxiter=1, initialize=False)

            # NOTE(@awav): TensorFlow optimizer skips uninitialized values which
            # are not present in objective.
            demo._objective += var3
            with self.assertRaises(tf.errors.FailedPreconditionError):
                opt.minimize(demo, session=session, var_list=[var3], maxiter=1,
                             initialize=False, anchor=False)

    def test_optimizer_tensors(self):
        with self.test_context():
            opt = self.optimizer()
            if not isinstance(opt, gpflow.train.ScipyOptimizer):
                demo = Demo()
                opt.minimize(demo, maxiter=0)
                self.assertEqual(opt.model, demo)

                opt.model = Demo()
                self.assertNotEqual(opt.model, demo)
                self.assertEqual(opt.minimize_operation, None)
                self.assertEqual(opt.optimizer, None)

    def test_non_gpflow_model(self):
        with self.test_context():
            opt = self.optimizer()
            with self.assertRaises(ValueError):
                opt.minimize(None, maxiter=0)

    def test_external_variables_in_model(self):
        with self.test_context():
            dfloat = gpflow.settings.float_type

            var1_init = np.array(1.0, dtype=dfloat)
            var2_init = np.array(2.0, dtype=dfloat)
            var3_init = np.array(3.0, dtype=dfloat)

            var1 = tf.get_variable('var1', initializer=var1_init, dtype=dfloat)
            var2 = tf.get_variable('var2', initializer=var2_init, dtype=dfloat)
            var3 = tf.get_variable('var3', initializer=var3_init, trainable=False, dtype=dfloat)

            opt = self.optimizer()

        # Just initialize variable, but do not use it in training
        with self.test_context() as session:
            gpflow.reset_default_session()
            demo1 = Demo(add_to_inits=[var1], add_to_trainables=[], name="demo1")
            opt.minimize(demo1, maxiter=10)
            self.assertEqual(session.run(var1), var1_init)

        with self.test_context() as session:
            gpflow.reset_default_session()
            with self.assertRaises(tf.errors.FailedPreconditionError):
                demo2 = Demo(add_to_inits=[], add_to_trainables=[var2], name="demo2")
                opt.minimize(demo2, maxiter=10)

        with self.test_context() as session:
            gpflow.reset_default_session()
            demo3 = Demo(add_to_inits=[var1, var2, var3],
                         add_to_trainables=[var2, var3], name="demo3")
            opt.minimize(demo3, maxiter=10)

            self.assertEqual(session.run(var1), var1_init)
            self.assertFalse(session.run(var3) == var3_init)
            self.assertFalse(session.run(var2) == var2_init)

class TestScipyOptimizer(GPflowTestCase, OptimizerCase):
    optimizer = gpflow.train.ScipyOptimizer

class TestGradientDescentOptimizer(GPflowTestCase, OptimizerCase):
    optimizer = lambda _self: gpflow.train.GradientDescentOptimizer(0.1)

class TestAdamOptimizer(GPflowTestCase, OptimizerCase):
    optimizer = lambda _self: gpflow.train.AdamOptimizer(0.1)

class TestMomentumOptimizer(GPflowTestCase, OptimizerCase):
    optimizer = lambda _self: gpflow.train.MomentumOptimizer(0.1, 0.9)

class TestAdadeltaOptimizer(GPflowTestCase, OptimizerCase):
    optimizer = lambda _self: gpflow.train.AdadeltaOptimizer(0.1)

class TestRMSPropOptimizer(GPflowTestCase, OptimizerCase):
    optimizer = lambda _self: gpflow.train.RMSPropOptimizer(0.1)

class TestFtrlOptimizer(GPflowTestCase, OptimizerCase):
    optimizer = lambda _self: gpflow.train.FtrlOptimizer(0.1)

# class TestProximalAdagradOptimizer(GPflowTestCase, OptimizerCase):
#     optimizer = lambda _self: gpflow.train.ProximalAdagradOptimizer(0.1)
#
#                 raise TypeError(
#                     "%s type %s of argument '%s'." %
#                     (prefix, dtypes.as_dtype(attrs[input_arg.type_attr]).name,
# >                    inferred_from[input_arg.type_attr]))
# E               TypeError: Input 'lr' of 'ApplyProximalAdagrad' Op has type float32 that does not match type float64 of argument 'var'.
#
# ../../anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:546: TypeError

def test_VGP_vs_GPR(session_tf):
    N, D = 3, 2
    X = np.random.randn(N, D)
    Y = np.random.randn(N, 1)
    kern = gpflow.kernels.RBF(D)
    lik_var = 0.1
    lik = gpflow.likelihoods.Gaussian()
    lik.variance = lik_var

    m_vgp = gpflow.models.VGP(X, Y, kern, lik)
    m_gpr = gpflow.models.GPR(X, Y, kern)
    m_gpr.likelihood.variance = lik_var

    m_vgp.set_trainable(False)
    m_vgp.q_mu.set_trainable(True)
    m_vgp.q_sqrt.set_trainable(True)
    NatGradOptimizer(1.).minimize(m_vgp, [[m_vgp.q_mu, m_vgp.q_sqrt]], maxiter=1)

    assert_allclose(m_gpr.compute_log_likelihood(),
                    m_vgp.compute_log_likelihood(), atol=1e-5)

def test_SVGP_vs_SGPR(session_tf):
    N, M, D = 4, 3, 2
    X = np.random.randn(N, D)
    Z = np.random.randn(M, D)
    Y = np.random.randn(N, 1)
    kern = gpflow.kernels.RBF(D)
    lik_var = 0.1
    lik = gpflow.likelihoods.Gaussian()
    lik.variance = lik_var

    m_svgp = gpflow.models.SVGP(X, Y, kern, lik, Z=Z)
    m_sgpr = gpflow.models.SGPR(X, Y, kern, Z=Z)
    m_sgpr.likelihood.variance = lik_var

    m_svgp.set_trainable(False)
    m_svgp.q_mu.set_trainable(True)
    m_svgp.q_sqrt.set_trainable(True)
    NatGradOptimizer(1.).minimize(m_svgp, [[m_svgp.q_mu, m_svgp.q_sqrt]], maxiter=1)

    assert_allclose(m_sgpr.compute_log_likelihood(),
                    m_svgp.compute_log_likelihood(), atol=1e-5)

class CombinationOptimizer(Optimizer):
    """
    A class that applies one step of each of multiple optimizers in a loop
    """
    def __init__(self, optimizers_with_kwargs):
        self.name = None
        super().__init__()
        self.optimizers_with_kwargs = optimizers_with_kwargs

    def minimize(self, model, session=None, maxiter=1000, feed_dict=None, anchor=True):
        session = model.enquire_session(session)

        minimize_ops = []
        for optimizer, kwargs in self.optimizers_with_kwargs:
            # hack to init the optimizer operation
            optimizer.minimize(model, maxiter=0, **kwargs)

            # this is what we will call in the loop
            minimize_ops.append(optimizer.minimize_operation)

        with session.graph.as_default(), tf.name_scope(self.name):
            feed_dict = self._gen_feed_dict(model, feed_dict)

            for _i in range(maxiter):
                for op in minimize_ops:
                    session.run(op, feed_dict=feed_dict)

        if anchor:
            model.anchor(session)

# # TODO(@hugh) this needs to be fixed
def test_hypers_SVGP_vs_SGPR(session_tf):
    N, M, D = 4, 3, 2
    X = np.random.randn(N, D)
    Z = np.random.randn(M, D)
    Y = np.random.randn(N, 1)
    kern = gpflow.kernels.RBF(D)
    lik_var = 0.1
    lik = gpflow.likelihoods.Gaussian()
    lik.variance = lik_var

    m_svgp = gpflow.models.SVGP(X, Y, kern, lik, Z=Z)
    m_sgpr = gpflow.models.SGPR(X, Y, kern, Z=Z)
    m_sgpr.likelihood.variance = lik_var

    m_svgp.q_mu.set_trainable(False)
    m_svgp.q_sqrt.set_trainable(False)

    NatGradOptimizer(1.).minimize(m_svgp, [[m_svgp.q_mu, m_svgp.q_sqrt]], maxiter=1)

    # this is fine
    assert_allclose(m_sgpr.compute_log_likelihood(),
                        m_svgp.compute_log_likelihood(), atol=1e-5)

    # combination (doing GD first as we've already done the nat grad step
    o1 = [GradientDescentOptimizer(0.01), {}]
    o2 = [NatGradOptimizer(1.), {'var_list':[[m_svgp.q_mu, m_svgp.q_sqrt]]}]
    nag_grads_with_gd_optimizer = CombinationOptimizer([o1, o2])
    nag_grads_with_gd_optimizer.minimize(m_svgp,  maxiter=1)

    # this should be the same as
    GradientDescentOptimizer(0.01).minimize(m_sgpr, maxiter=1)

    # but isn't...
    with pytest.raises(AssertionError):
        assert_allclose(m_sgpr.compute_log_likelihood(),
                        m_svgp.compute_log_likelihood(), atol=1e-5)
    # the trouble seems to be that the [q_mu, q_sqrt] haven't updated as far as o1 is concerned

class ExcludedGradientDescentOptimizer(_TensorFlowOptimizer):
    def __init__(self, *args, excluded_params=[], **kwargs):
        Optimizer.__init__(self)
        self._model = None
        self._optimizer = tf.train.GradientDescentOptimizer(*args, **kwargs)
        self._minimize_operation = None
        self.excluded_params = excluded_params

    def _gen_var_list(self, model, var_list):
        var_list = var_list or []
        p = set(model.trainable_tensors)
        p -= set([t.unconstrained_tensor for t in self.excluded_params])
        return list(p.union(var_list))

def test_hypers_SVGP_vs_SGPR_with_excluded_vars(session_tf):
    N, M, D = 4, 3, 2
    X = np.random.randn(N, D)
    Z = np.random.randn(M, D)
    Y = np.random.randn(N, 1)

    lik_var = 0.1
    lik = gpflow.likelihoods.Gaussian()
    lik.variance = lik_var

    m_svgp = gpflow.models.SVGP(X, Y, gpflow.kernels.RBF(D), lik, Z=Z)

    m_sgpr = gpflow.models.SGPR(X, Y, gpflow.kernels.RBF(D), Z=Z)
    m_sgpr.likelihood.variance = lik_var

    lr = 0.1

    # combination (doing GD first as we've already done the nat grad step
    p = [[m_svgp.q_mu, m_svgp.q_sqrt]]
    o1 = [NatGradOptimizer(1.), {'var_list':p}]
    o2 = [ExcludedGradientDescentOptimizer(lr, excluded_params=p[0]), {}]
    o3 = [NatGradOptimizer(1.), {'var_list':p}]

    nag_grads_with_gd_optimizer = CombinationOptimizer([o1, o2, o3])
    nag_grads_with_gd_optimizer.minimize(m_svgp, maxiter=1)

    # this should be the same as
    GradientDescentOptimizer(lr).minimize(m_sgpr, maxiter=1)

    assert_allclose(m_sgpr.compute_log_likelihood(),
                    m_svgp.compute_log_likelihood(), atol=1e-5)
    # now it works...<|endoftext|>


<commit_before>import unittest
import time
from app import create_app, db
from app.models import Resource, Descriptor, OptionAssociation, TextAssociation

class UserModelTestCase(unittest.TestCase):
    def setUp(self):
        self.app = create_app('testing')
        self.app_context = self.app.app_context()
        self.app_context.push()
        db.create_all()

    def tearDown(self):
        db.session.remove()
        db.drop_all()
        self.app_context.pop()

    def test_create_resource(self):
        r = Resource(name='test')
        a = OptionAssociation(option=0)
        options = ["True", "False"]
        a.descriptor = Descriptor(name="Open", values=options)
        r.option_descriptors.append(a)

        for option_assoc in r.option_descriptors:
        	print option_assoc.option
        	print option_assoc.descriptor

        db.session.add(r)
        db.session.commit()

    def test_random(self):
        Resource.generate_fake()
        Resource.print_users()

        <commit_msg>renamed print_random test<commit_after>import unittest
import time
from app import create_app, db
from app.models import Resource, Descriptor, OptionAssociation, TextAssociation

class UserModelTestCase(unittest.TestCase):
    def setUp(self):
        self.app = create_app('testing')
        self.app_context = self.app.app_context()
        self.app_context.push()
        db.create_all()

    def tearDown(self):
        db.session.remove()
        db.drop_all()
        self.app_context.pop()

    def test_create_resource(self):
        r = Resource(name='test')
        a = OptionAssociation(option=0)
        options = ["True", "False"]
        a.descriptor = Descriptor(name="Open", values=options)
        r.option_descriptors.append(a)

        for option_assoc in r.option_descriptors:
        	print option_assoc.option
        	print option_assoc.descriptor

        db.session.add(r)
        db.session.commit()

    def test_print_random(self):
        Resource.generate_fake()
        Resource.print_users()

        <|endoftext|>